From 876f10f9d8f45efa2b446840b4b5ee80ca0ade25 Mon Sep 17 00:00:00 2001 From: soypat Date: Thu, 22 May 2025 23:24:22 -0300 Subject: [PATCH 1/8] begin working on httpx --- httpx/definitions.go | 270 +++++++++++++++ httpx/header_parse.go | 768 ++++++++++++++++++++++++++++++++++++++++++ httpx/tokenizer.go | 622 ++++++++++++++++++++++++++++++++++ 3 files changed, 1660 insertions(+) create mode 100644 httpx/definitions.go create mode 100644 httpx/header_parse.go create mode 100644 httpx/tokenizer.go diff --git a/httpx/definitions.go b/httpx/definitions.go new file mode 100644 index 0000000..ce73fca --- /dev/null +++ b/httpx/definitions.go @@ -0,0 +1,270 @@ +package httpx + +const ( + slashChar = '/' + rChar = '\r' + nChar = '\n' + defaultServerName = "fasthttp" + defaultUserAgent = "fasthttp" + defaultContentType = "text/plain; charset=utf-8" +) + +const ( + strSlashSlash = "//" + strSlashDotDot = "/.." + strSlashDotSlash = "/./" + strSlashDotDotSlash = "/../" + strBackSlashDotDot = `\..` + strBackSlashDotBackSlash = `\.\` + strSlashDotDotBackSlash = `/..\` + strBackSlashDotDotBackSlash = `\..\` + strCRLF = "\r\n" + strHTTP = "http" + strHTTPS = "https" + strHTTP10 = "HTTP/1.0" + strHTTP11 = "HTTP/1.1" + strColon = ":" + strColonSlashSlash = "://" + strColonSpace = ": " + strCommaSpace = ", " + strGMT = "GMT" + + strResponseContinue = "HTTP/1.1 100 Continue\r\n\r\n" + + strExpect = HeaderExpect + strConnection = HeaderConnection + strContentLength = HeaderContentLength + strContentType = HeaderContentType + strDate = HeaderDate + strHost = HeaderHost + strReferer = HeaderReferer + strServer = HeaderServer + strTransferEncoding = HeaderTransferEncoding + strContentEncoding = HeaderContentEncoding + strAcceptEncoding = HeaderAcceptEncoding + strUserAgent = HeaderUserAgent + strCookie = HeaderCookie + strSetCookie = HeaderSetCookie + strLocation = HeaderLocation + strIfModifiedSince = HeaderIfModifiedSince + strLastModified = HeaderLastModified + strAcceptRanges = HeaderAcceptRanges + strRange = HeaderRange + strContentRange = HeaderContentRange + strAuthorization = HeaderAuthorization + strTE = HeaderTE + strTrailer = HeaderTrailer + strMaxForwards = HeaderMaxForwards + strProxyConnection = HeaderProxyConnection + strProxyAuthenticate = HeaderProxyAuthenticate + strProxyAuthorization = HeaderProxyAuthorization + strWWWAuthenticate = HeaderWWWAuthenticate + strVary = HeaderVary + + strCookieExpires = "expires" + strCookieDomain = "domain" + strCookiePath = "path" + strCookieHTTPOnly = "HttpOnly" + strCookieSecure = "secure" + strCookieMaxAge = "max-age" + strCookieSameSite = "SameSite" + strCookieSameSiteLax = "Lax" + strCookieSameSiteStrict = "Strict" + strCookieSameSiteNone = "None" + + strClose = "close" + strGzip = "gzip" + strBr = "br" + strDeflate = "deflate" + strKeepAlive = "keep-alive" + strUpgrade = "Upgrade" + strChunked = "chunked" + strIdentity = "identity" + str100Continue = "100-continue" + strPostArgsContentType = "application/x-www-form-urlencoded" + strDefaultContentType = "application/octet-stream" + strMultipartFormData = "multipart/form-data" + strBoundary = "boundary" + strBytes = "bytes" + strBasicSpace = "Basic " + + strApplicationSlash = "application/" + strImageSVG = "image/svg" + strImageIcon = "image/x-icon" + strFontSlash = "font/" + strMultipartSlash = "multipart/" + strTextSlash = "text/" +) + +// Headers. +const ( + // Authentication. + HeaderAuthorization = "Authorization" + HeaderProxyAuthenticate = "Proxy-Authenticate" + HeaderProxyAuthorization = "Proxy-Authorization" + HeaderWWWAuthenticate = "WWW-Authenticate" + + // Caching. + HeaderAge = "Age" + HeaderCacheControl = "Cache-Control" + HeaderClearSiteData = "Clear-Site-Data" + HeaderExpires = "Expires" + HeaderPragma = "Pragma" + HeaderWarning = "Warning" + + // Client hints. + HeaderAcceptCH = "Accept-CH" + HeaderAcceptCHLifetime = "Accept-CH-Lifetime" + HeaderContentDPR = "Content-DPR" + HeaderDPR = "DPR" + HeaderEarlyData = "Early-Data" + HeaderSaveData = "Save-Data" + HeaderViewportWidth = "Viewport-Width" + HeaderWidth = "Width" + + // Conditionals. + HeaderETag = "ETag" + HeaderIfMatch = "If-Match" + HeaderIfModifiedSince = "If-Modified-Since" + HeaderIfNoneMatch = "If-None-Match" + HeaderIfUnmodifiedSince = "If-Unmodified-Since" + HeaderLastModified = "Last-Modified" + HeaderVary = "Vary" + + // Connection management. + HeaderConnection = "Connection" + HeaderKeepAlive = "Keep-Alive" + HeaderProxyConnection = "Proxy-Connection" + + // Content negotiation. + HeaderAccept = "Accept" + HeaderAcceptCharset = "Accept-Charset" + HeaderAcceptEncoding = "Accept-Encoding" + HeaderAcceptLanguage = "Accept-Language" + + // Controls. + HeaderCookie = "Cookie" + HeaderExpect = "Expect" + HeaderMaxForwards = "Max-Forwards" + HeaderSetCookie = "Set-Cookie" + + // CORS. + HeaderAccessControlAllowCredentials = "Access-Control-Allow-Credentials" + HeaderAccessControlAllowHeaders = "Access-Control-Allow-Headers" + HeaderAccessControlAllowMethods = "Access-Control-Allow-Methods" + HeaderAccessControlAllowOrigin = "Access-Control-Allow-Origin" + HeaderAccessControlExposeHeaders = "Access-Control-Expose-Headers" + HeaderAccessControlMaxAge = "Access-Control-Max-Age" + HeaderAccessControlRequestHeaders = "Access-Control-Request-Headers" + HeaderAccessControlRequestMethod = "Access-Control-Request-Method" + HeaderOrigin = "Origin" + HeaderTimingAllowOrigin = "Timing-Allow-Origin" + HeaderXPermittedCrossDomainPolicies = "X-Permitted-Cross-Domain-Policies" + + // Do Not Track. + HeaderDNT = "DNT" + HeaderTk = "Tk" + + // Downloads. + HeaderContentDisposition = "Content-Disposition" + + // Message body information. + HeaderContentEncoding = "Content-Encoding" + HeaderContentLanguage = "Content-Language" + HeaderContentLength = "Content-Length" + HeaderContentLocation = "Content-Location" + HeaderContentType = "Content-Type" + + // Proxies. + HeaderForwarded = "Forwarded" + HeaderVia = "Via" + HeaderXForwardedFor = "X-Forwarded-For" + HeaderXForwardedHost = "X-Forwarded-Host" + HeaderXForwardedProto = "X-Forwarded-Proto" + + // Redirects. + HeaderLocation = "Location" + + // Request context. + HeaderFrom = "From" + HeaderHost = "Host" + HeaderReferer = "Referer" + HeaderReferrerPolicy = "Referrer-Policy" + HeaderUserAgent = "User-Agent" + + // Response context. + HeaderAllow = "Allow" + HeaderServer = "Server" + + // Range requests. + HeaderAcceptRanges = "Accept-Ranges" + HeaderContentRange = "Content-Range" + HeaderIfRange = "If-Range" + HeaderRange = "Range" + + // Security. + HeaderContentSecurityPolicy = "Content-Security-Policy" + HeaderContentSecurityPolicyReportOnly = "Content-Security-Policy-Report-Only" + HeaderCrossOriginResourcePolicy = "Cross-Origin-Resource-Policy" + HeaderExpectCT = "Expect-CT" + HeaderFeaturePolicy = "Feature-Policy" + HeaderPublicKeyPins = "Public-Key-Pins" + HeaderPublicKeyPinsReportOnly = "Public-Key-Pins-Report-Only" + HeaderStrictTransportSecurity = "Strict-Transport-Security" + HeaderUpgradeInsecureRequests = "Upgrade-Insecure-Requests" + HeaderXContentTypeOptions = "X-Content-Type-Options" + HeaderXDownloadOptions = "X-Download-Options" + HeaderXFrameOptions = "X-Frame-Options" + HeaderXPoweredBy = "X-Powered-By" + HeaderXXSSProtection = "X-XSS-Protection" + + // Server-sent event. + HeaderLastEventID = "Last-Event-ID" + HeaderNEL = "NEL" + HeaderPingFrom = "Ping-From" + HeaderPingTo = "Ping-To" + HeaderReportTo = "Report-To" + + // Transfer coding. + HeaderTE = "TE" + HeaderTrailer = "Trailer" + HeaderTransferEncoding = "Transfer-Encoding" + + // WebSockets. + HeaderSecWebSocketAccept = "Sec-WebSocket-Accept" + HeaderSecWebSocketExtensions = "Sec-WebSocket-Extensions" /* #nosec G101 */ + HeaderSecWebSocketKey = "Sec-WebSocket-Key" + HeaderSecWebSocketProtocol = "Sec-WebSocket-Protocol" + HeaderSecWebSocketVersion = "Sec-WebSocket-Version" + + // Other. + HeaderAcceptPatch = "Accept-Patch" + HeaderAcceptPushPolicy = "Accept-Push-Policy" + HeaderAcceptSignature = "Accept-Signature" + HeaderAltSvc = "Alt-Svc" + HeaderDate = "Date" + HeaderIndex = "Index" + HeaderLargeAllocation = "Large-Allocation" + HeaderLink = "Link" + HeaderPushPolicy = "Push-Policy" + HeaderRetryAfter = "Retry-After" + HeaderServerTiming = "Server-Timing" + HeaderSignature = "Signature" + HeaderSignedHeaders = "Signed-Headers" + HeaderSourceMap = "SourceMap" + HeaderUpgrade = "Upgrade" + HeaderXDNSPrefetchControl = "X-DNS-Prefetch-Control" + HeaderXPingback = "X-Pingback" + HeaderXRequestedWith = "X-Requested-With" + HeaderXRobotsTag = "X-Robots-Tag" + HeaderXUACompatible = "X-UA-Compatible" +) + +// Probably replace these with short functions to take up less program memory +const ( + hex2intTable = "\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x00\x01\x02\x03\x04\x05\x06\a\b\t\x10\x10\x10\x10\x10\x10\x10\n\v\f\r\x0e\x0f\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\n\v\f\r\x0e\x0f\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10" + toLowerTable = "\x00\x01\x02\x03\x04\x05\x06\a\b\t\n\v\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@abcdefghijklmnopqrstuvwxyz[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + toUpperTable = "\x00\x01\x02\x03\x04\x05\x06\a\b\t\n\v\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~\u007f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + quotedArgShouldEscapeTable = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + quotedPathShouldEscapeTable = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x01\x00\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" +) diff --git a/httpx/header_parse.go b/httpx/header_parse.go new file mode 100644 index 0000000..915541c --- /dev/null +++ b/httpx/header_parse.go @@ -0,0 +1,768 @@ +package httpx + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "strings" + "unsafe" +) + +var ( + errNeedMore = errors.New("need more data: cannot find trailing lf") + errInvalidName = errors.New("invalid header name") + errSmallBuffer = errors.New("small read buffer. Increase ReadBufferSize") + errNonNumericChars = errors.New("non-numeric chars found") +) + +type headerScanner struct { + b []byte + key []byte + value []byte + err error + + // hLen stores header subslice len + hLen int + + disableNormalizing bool + + // by checking whether the next line contains a colon or not to tell + // it's a header entry or a multi line value of current header entry. + // the side effect of this operation is that we know the index of the + // next colon and new line, so this can be used during next iteration, + // instead of find them again. + nextColon int + nextNewLine int + + initialized bool +} + +type headerValueScanner struct { + b string + value string +} + +func (h *header) parse(buf []byte) (int, error) { + m, err := h.parseFirstLine(buf) + if err != nil { + return 0, err + } + + h.rawHeaders, _, err = readRawHeaders(h.rawHeaders[:0], b2s(buf[m:])) + if err != nil { + return 0, err + } + var n int + n, err = h.parseHeaders(buf[m:]) + if err != nil { + return 0, err + } + return m + n, nil +} + +func (hb *headerBuf) offBuf() []byte { + return hb.buf[hb.off:] +} + +func (hb *headerBuf) scanLine() []byte { + buf := hb.scanUntilByte('\n') + if len(buf) > 0 && buf[len(buf)-1] == '\r' { + buf = buf[:len(buf)-1] // exclude carriage return. + } + if hb.off < len(hb.buf) { + hb.off++ // consume newline. + } + return buf +} + +func (hb *headerBuf) scanUntilByte(c byte) []byte { + buf := hb.offBuf() + idx := bytes.IndexByte(buf, c) + if idx >= 0 { + buf = buf[:idx] + } + hb.off += len(buf) + return buf +} + +func (hb *headerBuf) parseFirstLine() (method, uri, proto headerSlice, flags flags, err error) { + var b []byte + for len(b) == 0 { + b = hb.scanLine() + } + if len(b) < 5 { + return method, uri, proto, 0, errors.New("too short first HTTP line") + } + + methodEnd := max(0, bytes.IndexByte(b, ' ')) + reqURIEnd := bytes.IndexByte(b[methodEnd+1:], ' ') + switch { + case reqURIEnd < 0: + flags |= noHTTP11 + reqURIEnd = methodEnd + 1 + case reqURIEnd == 0: + return method, uri, proto, flags, errors.New("empty URI") + case b2s(b[reqURIEnd+1:]) != strHTTP11: + flags |= noHTTP11 + fallthrough + default: + proto = hb.slice(b[reqURIEnd+1:]) + } + uri = hb.slice(b[methodEnd+1 : reqURIEnd]) + method = hb.slice(b[:methodEnd]) + return method, uri, proto, flags, nil +} + +func readRawHeaders(dst []byte, buf string) ([]byte, int, error) { + n := strings.IndexByte(buf, nChar) + if n < 0 { + return dst[:0], 0, errNeedMore + } + if (n == 1 && buf[0] == rChar) || n == 0 { + // empty headers + return dst, n + 1, nil + } + + n++ + b := buf + m := n + for { + b = b[m:] + m = strings.IndexByte(b, nChar) + if m < 0 { + return dst, 0, errNeedMore + } + m++ + n += m + if (m == 2 && b[0] == rChar) || m == 1 { + dst = append(dst, buf[:n]...) + return dst, n, nil + } + } +} + +func (h *header) parseHeaders(buf []byte) (int, error) { + h.contentLength = -2 + h.scanner = headerScanner{} + s := &h.scanner + s.b = buf + s.disableNormalizing = h.disableNormalizing + var err error + for s.next() { + key := b2s(s.key) + value := b2s(s.value) + if len(key) > 0 { + // Spaces between the header key and colon are not allowed. + // See RFC 7230, Section 3.2.4. + + if strings.IndexByte(key, ' ') != -1 || strings.IndexByte(key, '\t') != -1 { + err = fmt.Errorf("invalid header key %q", s.key) + continue + } + + if h.disableSpecialHeader { + h.h = appendArg(h.h, key, value, argsHasValue) + continue + } + + switch s.key[0] | 0x20 { + case 'h': + if caseInsensitiveCompare(key, strHost) { + h.host = append(h.host[:0], value...) + continue + } + case 'u': + if caseInsensitiveCompare(key, strUserAgent) { + h.userAgent = append(h.userAgent[:0], value...) + continue + } + case 'c': + if caseInsensitiveCompare(key, strContentType) { + h.contentType = append(h.contentType[:0], value...) + continue + } + if caseInsensitiveCompare(key, strContentLength) { + if h.contentLength != -1 { + var nerr error + if h.contentLength, nerr = parseContentLength(b2s(s.value)); nerr != nil { + if err == nil { + err = nerr + } + h.contentLength = -2 + } else { + h.contentLengthBytes = append(h.contentLengthBytes[:0], value...) + } + } + continue + } + if caseInsensitiveCompare(key, strConnection) { + if b2s(s.value) == strClose { + h.connectionClose = true + } else { + h.connectionClose = false + h.h = appendArg(h.h, key, value, argsHasValue) + } + continue + } + case 't': + if caseInsensitiveCompare(key, strTransferEncoding) { + if value != strIdentity { + h.contentLength = -1 + h.h = setArg(h.h, strTransferEncoding, strChunked, argsHasValue) + } + continue + } + if caseInsensitiveCompare(key, strTrailer) { + if nerr := h.SetTrailer(value); nerr != nil { + if err == nil { + err = nerr + } + } + continue + } + } + } + h.h = appendArg(h.h, key, value, argsHasValue) + } + if s.err != nil && err == nil { + err = s.err + } + if err != nil { + h.connectionClose = true + return 0, err + } + + if h.contentLength < 0 { + h.contentLengthBytes = h.contentLengthBytes[:0] + } + if h.noHTTP11 && !h.connectionClose { + // close connection for non-http/1.1 request unless 'Connection: keep-alive' is set. + v := peekArgStr(h.h, strConnection) + h.connectionClose = !hasHeaderValue(b2s(v), strKeepAlive) + } + return s.hLen, nil +} + +func (s *headerScanner) next() bool { + if !s.initialized { + s.nextColon = -1 + s.nextNewLine = -1 + s.initialized = true + } + bLen := len(s.b) + if bLen >= 2 && s.b[0] == rChar && s.b[1] == nChar { + s.b = s.b[2:] + s.hLen += 2 + return false + } + if bLen >= 1 && s.b[0] == nChar { + s.b = s.b[1:] + s.hLen++ + return false + } + + var n int + if s.nextColon >= 0 { + n = s.nextColon + s.nextColon = -1 + } else { + n = bytes.IndexByte(s.b, ':') + + // There can't be a \n inside the header name, check for this. + x := bytes.IndexByte(s.b, nChar) + if x < 0 { + // A header name should always at some point be followed by a \n + // even if it's the one that terminates the header block. + s.err = errNeedMore + return false + } + if x < n { + // There was a \n before the : + s.err = errInvalidName + return false + } + } + if n < 0 { + s.err = errNeedMore + return false + } + s.key = s.b[:n] + normalizeHeaderKey(s.key, s.disableNormalizing) + n++ + for len(s.b) > n && s.b[n] == ' ' { + n++ + // the newline index is a relative index, and lines below trimmed `s.b` by `n`, + // so the relative newline index also shifted forward. it's safe to decrease + // to a minus value, it means it's invalid, and will find the newline again. + s.nextNewLine-- + } + s.hLen += n + s.b = s.b[n:] + if s.nextNewLine >= 0 { + n = s.nextNewLine + s.nextNewLine = -1 + } else { + n = bytes.IndexByte(s.b, nChar) + } + if n < 0 { + s.err = errNeedMore + return false + } + isMultiLineValue := false + for { + if n+1 >= len(s.b) { + break + } + if s.b[n+1] != ' ' && s.b[n+1] != '\t' { + break + } + d := bytes.IndexByte(s.b[n+1:], nChar) + if d <= 0 { + break + } else if d == 1 && s.b[n+1] == rChar { + break + } + e := n + d + 1 + if c := bytes.IndexByte(s.b[n+1:e], ':'); c >= 0 { + s.nextColon = c + s.nextNewLine = d - c - 1 + break + } + isMultiLineValue = true + n = e + } + if n >= len(s.b) { + s.err = errNeedMore + return false + } + oldB := s.b + s.value = s.b[:n] + s.hLen += n + 1 + s.b = s.b[n+1:] + + if n > 0 && s.value[n-1] == rChar { + n-- + } + for n > 0 && s.value[n-1] == ' ' { + n-- + } + s.value = s.value[:n] + if isMultiLineValue { + s.value, s.b, s.hLen = normalizeHeaderValue(s.value, oldB, s.hLen) + } + return true +} + +func normalizeHeaderKey(b []byte, disableNormalizing bool) { + if disableNormalizing { + return + } + + n := len(b) + if n == 0 { + return + } + + b[0] = toUpperTable[b[0]] + for i := 1; i < n; i++ { + p := &b[i] + if *p == '-' { + i++ + if i < n { + b[i] = toUpperTable[b[i]] + } + continue + } + *p = toLowerTable[*p] + } +} + +func normalizeHeaderValue(ov, ob []byte, headerLength int) (nv, nb []byte, nhl int) { + nv = ov + length := len(ov) + if length <= 0 { + return + } + write := 0 + shrunk := 0 + lineStart := false + for read := 0; read < length; read++ { + c := ov[read] + switch { + case c == rChar || c == nChar: + shrunk++ + if c == nChar { + lineStart = true + } + continue + case lineStart && c == '\t': + c = ' ' + default: + lineStart = false + } + nv[write] = c + write++ + } + + nv = nv[:write] + copy(ob[write:], ob[write+shrunk:]) + + // Check if we need to skip \r\n or just \n + skip := 0 + if ob[write] == rChar { + if ob[write+1] == nChar { + skip += 2 + } else { + skip++ + } + } else if ob[write] == nChar { + skip++ + } + + nb = ob[write+skip : len(ob)-shrunk] + nhl = headerLength - shrunk + return +} + +func parseContentLength(b string) (int, error) { + v, n, err := parseUintBuf(b) + if err != nil { + return -1, fmt.Errorf("cannot parse Content-Length: %w", err) + } + if n != len(b) { + return -1, fmt.Errorf("cannot parse Content-Length: %w", errNonNumericChars) + } + return v, nil +} + +func hasHeaderValue(s, value string) bool { + var vs headerValueScanner + vs.b = s + for vs.next() { + if caseInsensitiveCompare(vs.value, value) { + return true + } + } + return false +} + +func (s *headerValueScanner) next() bool { + b := s.b + if len(b) == 0 { + return false + } + n := strings.IndexByte(b, ',') + if n < 0 { + s.value = stripSpace(b) + s.b = b[len(b):] + return true + } + s.value = stripSpace(b[:n]) + s.b = b[n+1:] + return true +} + +func nextLine(b []byte) ([]byte, []byte, error) { + nNext := bytes.IndexByte(b, nChar) + if nNext < 0 { + return nil, nil, errNeedMore + } + n := nNext + if n > 0 && b[n-1] == rChar { + n-- + } + return b[:n], b[nNext+1:], nil +} + +func stripSpace(b string) string { + for len(b) > 0 && b[0] == ' ' { + b = b[1:] + } + for len(b) > 0 && b[len(b)-1] == ' ' { + b = b[:len(b)-1] + } + return b +} + +var ( + errEmptyInt = errors.New("empty integer") + errUnexpectedFirstChar = errors.New("unexpected first char found. Expecting 0-9") + errUnexpectedTrailingChar = errors.New("unexpected trailing char found. Expecting 0-9") + errTooLongInt = errors.New("too long int") +) + +func parseUintBuf(b string) (int, int, error) { + n := len(b) + if n == 0 { + return -1, 0, errEmptyInt + } + v := 0 + for i := 0; i < n; i++ { + c := b[i] + k := c - '0' + if k > 9 { + if i == 0 { + return -1, i, errUnexpectedFirstChar + } + return v, i, nil + } + vNew := 10*v + int(k) + // Test for overflow. + if vNew < v { + return -1, i, errTooLongInt + } + v = vNew + } + return v, n, nil +} + +/* + +Request Parsing + +*/ + +// Read reads request header from r. +// +// io.EOF is returned if r is closed before reading the first header byte. +func (h *header) Read(r *bufio.Reader) error { + return h.readLoop(r, true) +} + +// readLoop reads request header from r optionally loops until it has enough data. +// +// io.EOF is returned if r is closed before reading the first header byte. +func (h *header) readLoop(r *bufio.Reader, waitForMore bool) error { + n := 1 + for { + err := h.tryRead(r, n) + if err == nil { + return nil + } + if !waitForMore || err != errNeedMore { + h.resetSkipNormalize() + return err + } + n = r.Buffered() + 1 + } +} + +func (h *header) tryRead(r *bufio.Reader, n int) error { + h.resetSkipNormalize() + b, err := r.Peek(n) + if len(b) == 0 { + if err == io.EOF { + return err + } + + if err == nil { + panic("bufio.Reader.Peek() returned nil, nil") + } + + // This is for go 1.6 bug. See https://github.com/golang/go/issues/14121 . + if err == bufio.ErrBufferFull { + return &ErrSmallBuffer{ + error: fmt.Errorf("error when reading request headers: %w (n=%d, r.Buffered()=%d)", errSmallBuffer, n, r.Buffered()), + } + } + + // n == 1 on the first read for the request. + if n == 1 { + // We didn't read a single byte. + return ErrNothingRead{err} + } + + return fmt.Errorf("error when reading request headers: %w", err) + } + b = mustPeekBuffered(r) + headersLen, errParse := h.parse(b) + if errParse != nil { + return headerError("request", err, errParse, b, false) + } + mustDiscard(r, headersLen) + return nil +} + +func (h *header) resetSkipNormalize() { + h.noHTTP11 = false + h.connectionClose = false + + h.contentLength = 0 + h.contentLengthBytes = h.contentLengthBytes[:0] + + h.method = h.method[:0] + h.proto = h.proto[:0] + h.requestURI = h.requestURI[:0] + h.host = h.host[:0] + h.contentType = h.contentType[:0] + h.userAgent = h.userAgent[:0] + h.trailer = h.trailer[:0] + h.mulHeader = h.mulHeader[:0] + + h.h = h.h[:0] + h.cookies = h.cookies[:0] + h.cookiesCollected = false + + h.rawHeaders = h.rawHeaders[:0] +} + +func headerError(typ string, err, errParse error, b []byte, secureErrorLogMessage bool) error { + if errParse != errNeedMore { + return headerErrorMsg(typ, errParse, b, secureErrorLogMessage) + } + if err == nil { + return errNeedMore + } + + // Buggy servers may leave trailing CRLFs after http body. + // Treat this case as EOF. + if isOnlyCRLF(b) { + return io.EOF + } + + if err != bufio.ErrBufferFull { + return headerErrorMsg(typ, err, b, secureErrorLogMessage) + } + return &ErrSmallBuffer{ + error: headerErrorMsg(typ, errSmallBuffer, b, secureErrorLogMessage), + } +} + +func isOnlyCRLF(b []byte) bool { + for _, ch := range b { + if ch != rChar && ch != nChar { + return false + } + } + return true +} +func headerErrorMsg(typ string, err error, b []byte, secureErrorLogMessage bool) error { + return fmt.Errorf("error when reading %s headers: %w. Buffer size=%d", typ, err, len(b)) +} + +// ErrNothingRead is returned when a keep-alive connection is closed, +// either because the remote closed it or because of a read timeout. +type ErrNothingRead struct { + error +} + +// ErrSmallBuffer is returned when the provided buffer size is too small +// for reading request and/or response headers. +// +// ReadBufferSize value from Server or clients should reduce the number +// of such errors. +type ErrSmallBuffer struct { + error +} + +func mustPeekBuffered(r *bufio.Reader) []byte { + buf, err := r.Peek(r.Buffered()) + if len(buf) == 0 || err != nil { + panic(fmt.Sprintf("bufio.Reader.Peek() returned unexpected data (%q, %v)", buf, err)) + } + return buf +} + +func mustDiscard(r *bufio.Reader, n int) { + if _, err := r.Discard(n); err != nil { + panic(fmt.Sprintf("bufio.Reader.Discard(%d) failed: %v", n, err)) + } +} + +// Peek returns header value for the given key. +// +// The returned value is valid until the request is released, +// either though ReleaseRequest or your request handler returning. +// Do not store references to returned value. Make copies instead. +func (h *header) Peek(key string) []byte { + k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) + return h.peek(b2s(k)) +} + +// Host returns Host header value. +func (h *header) Host() []byte { + if h.disableSpecialHeader { + return peekArg(h.h, HeaderHost) + } + return h.host +} + +func getHeaderKeyBytes(kv *argsKV, key string, disableNormalizing bool) []byte { + kv.key = append(kv.key[:0], key...) + normalizeHeaderKey(kv.key, disableNormalizing) + return kv.key +} + +func peekArg(h []argsKV, k string) []byte { + for i, n := 0, len(h); i < n; i++ { + kv := &h[i] + if b2s(kv.key) == k { + return kv.value + } + } + return nil +} + +func (h *header) peek(key string) []byte { + switch key { + case HeaderHost: + return h.Host() + case HeaderContentType: + return h.ContentType() + case HeaderUserAgent: + return h.UserAgent() + case HeaderConnection: + if h.ConnectionClose() { + return []byte(strClose) + } + return peekArg(h.h, key) + case HeaderContentLength: + return h.contentLengthBytes + case HeaderCookie: + if h.cookiesCollected { + return appendRequestCookieBytes(nil, h.cookies) + } + return peekArg(h.h, key) + case HeaderTrailer: + return appendArgsKey(nil, h.trailer, strCommaSpace) + default: + return peekArg(h.h, key) + } +} + +// ConnectionClose returns true if 'Connection: close' header is set. +func (h *header) ConnectionClose() bool { + return h.connectionClose +} + +// UserAgent returns User-Agent header value. +func (h *header) UserAgent() []byte { + if h.disableSpecialHeader { + return peekArg(h.h, HeaderUserAgent) + } + return h.userAgent +} + +func appendArgsKey(dst []byte, args []argsKV, sep string) []byte { + for i, n := 0, len(args); i < n; i++ { + kv := &args[i] + dst = append(dst, kv.key...) + if i+1 < n { + dst = append(dst, sep...) + } + } + return dst +} + +// b2s converts byte slice to a string without memory allocation. +// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . +func b2s(b []byte) string { + return unsafe.String(unsafe.SliceData(b), len(b)) +} + +// s2b converts string to a byte slice without memory allocation. +func s2b(s string) []byte { + return unsafe.Slice(unsafe.StringData(s), len(s)) +} diff --git a/httpx/tokenizer.go b/httpx/tokenizer.go new file mode 100644 index 0000000..667f53c --- /dev/null +++ b/httpx/tokenizer.go @@ -0,0 +1,622 @@ +package httpx + +import ( + "errors" + "log/slog" + "net/http" + "strconv" + "strings" + "unsafe" + + "github.com/soypat/lneto/internal" +) + +type headerBuf struct { + buf []byte + off int // offset into buf for parsing. + // args contains key-value store. + args []argsKV +} + +type tokint = uint16 + +type headerSlice struct { + start tokint + len tokint +} + +type argSlice struct { + start tokint + len tokint +} + +type argsKV struct { + key headerSlice + value headerSlice // value start >0 means value is present. +} + +func (tb headerBuf) musttoken(slice headerSlice) []byte { + return tb.buf[slice.start : slice.start+slice.len] +} +func (tb headerBuf) mustargs(slice argSlice) []argsKV { + return tb.args[slice.start : slice.start+slice.len] +} + +func (tb headerBuf) visitArgs(args argSlice, f func(k, v []byte)) { + a := tb.mustargs(args) + for _, arg := range a { + k := tb.musttoken(arg.key) + v := tb.musttoken(arg.value) + f(k, v) + } +} + +func (tb headerBuf) visitArgsKey(args argSlice, f func(k []byte)) { + a := tb.mustargs(args) + for _, arg := range a { + f(tb.musttoken(arg.key)) + } +} + +func (tb headerBuf) slice(b []byte) headerSlice { + base := uintptr(unsafe.Pointer(&tb.buf[0])) + off := uintptr(unsafe.Pointer(&b[0])) + if off < base || off > base+uintptr(len(tb.buf)) { + panic("httpx: argument buffer does not alias header buffer") + } + return headerSlice{ + start: tokint(off - base), + len: tokint(len(b)), + } +} + +func (kv argsKV) HasValue() bool { return kv.value.start > 0 } + +type flags uint8 + +const ( + disableNormalizing flags = 1 << iota + disableSpecialHeader + noDefaultContentType + connectionClose + noHTTP11 + cookiesCollected +) + +func (f flags) hasAll(checkThese flags) bool { + return f&checkThese == checkThese +} + +type header struct { + buf headerBuf + contentLength int + h argSlice + cookies argSlice + trailer argSlice + + host headerSlice + contentLengthBytes headerSlice + contentType headerSlice + userAgent headerSlice + method headerSlice + proto headerSlice + requestURI headerSlice + rawHeaders headerSlice + mulHeader headerSlice + + flags flags + logger *slog.Logger +} + +func (h *header) Set(key, value string) { + h.bufKV.key = append(h.bufKV.key[:0], key...) + normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) + h.SetCanonical(b2s(h.bufKV.key), value) +} + +func (h *header) Add(key, value string) { + if h.setSpecialHeader(key, value) { + return + } + k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) + h.h = appendArg(h.h, b2s(k), value, argsHasValue) +} + +// ContentEncoding returns Content-Encoding header value. +func (h *header) ContentEncoding() []byte { + return peekArg(h.h, strContentEncoding) +} + +// SetContentEncoding sets Content-Encoding header value. +func (h *header) SetContentEncoding(contentEncoding string) { + h.Set(strContentEncoding, contentEncoding) +} + +// SetContentType sets Content-Type header value. +func (h *header) SetContentType(contentType string) { + h.contentType = append(h.contentType[:0], contentType...) +} + +// ContentType returns Content-Type header value. +func (h *header) ContentType() []byte { + contentType := h.contentType + if !h.noDefaultContentType && len(h.contentType) == 0 { + contentType = append(contentType, defaultContentType...) + } + return contentType +} + +// SetCanonical sets the given 'key: value' header assuming that +// key is in canonical form. +// +// If the header is set as a Trailer (forbidden trailers will not be set, see SetTrailer for more details), +// it will be sent after the chunked request body. +func (h *header) SetCanonical(key, value string) { + if h.setSpecialHeader(key, value) { + return + } + h.setNonSpecial(key, value) +} + +// setSpecialHeader handles special headers and return true when a header is processed. +func (h *header) setSpecialHeader(key, value string) bool { + if len(key) == 0 || h.disableSpecialHeader { + return false + } + h.trace("setSpecialHeader", slog.String("key", key), slog.String("value", value)) + switch key[0] | 0x20 { + case 'c': + switch { + case caseInsensitiveCompare(strContentType, key): + h.SetContentType(value) + return true + case caseInsensitiveCompare(strContentLength, key): + if contentLength, err := parseContentLength(value); err == nil { + h.contentLength = contentLength + h.contentLengthBytes = append(h.contentLengthBytes[:0], value...) + } + return true + case caseInsensitiveCompare(strConnection, key): + if strClose == value { + h.SetConnectionClose() + } else { + h.ResetConnectionClose() + h.setNonSpecial(key, value) + } + return true + case caseInsensitiveCompare(strCookie, key): + h.collectCookies() + h.cookies = parseRequestCookies(h.cookies, value) + return true + } + case 't': // OK + if caseInsensitiveCompare(strTransferEncoding, key) { + // Transfer-Encoding is managed automatically. + return true + } else if caseInsensitiveCompare(strTrailer, key) { + _ = h.SetTrailer(value) + return true + } + case 'h': + if caseInsensitiveCompare(strHost, key) { + h.SetHost(value) + return true + } + case 'u': + if caseInsensitiveCompare(strUserAgent, key) { + h.SetUserAgent(value) + return true + } + } + + return false +} + +// SetHost sets Host header value. +func (h *header) SetHost(host string) { + h.host = append(h.host[:0], host...) +} + +// SetUserAgent sets User-Agent header value. +func (h *header) SetUserAgent(userAgent string) { + h.userAgent = append(h.userAgent[:0], userAgent...) +} + +// SetConnectionClose sets 'Connection: close' header. +func (h *header) SetConnectionClose() { + h.connectionClose = true +} + +// ResetConnectionClose clears 'Connection: close' header if it exists. +func (h *header) ResetConnectionClose() { + if h.connectionClose { + h.connectionClose = false + h.h = delAllArgs(h.h, strConnection) + } +} + +func (h *header) SetContentRange(startPos, endPos, contentLength int) { + b := h.bufKV.value[:0] + b = append(b, strBytes...) + b = append(b, ' ') + b = appendUint(b, startPos) + b = append(b, '-') + b = appendUint(b, endPos) + b = append(b, '/') + b = appendUint(b, contentLength) + h.bufKV.value = b + + h.setNonSpecial(strContentRange, b2s(h.bufKV.value)) +} + +// setNonSpecial directly put into map i.e. not a basic header. +func (h *header) setNonSpecial(key string, value string) { + h.trace("httpx:setNonSpecial", slog.String("key", key), slog.String("value", value)) + h.h = setArg(h.h, key, value, argsHasValue) +} + +func appendUint(b []byte, v int) []byte { + if v < 0 { + panic("negative uint") + } + return strconv.AppendUint(b, uint64(v), 10) +} + +// ContentLength returns Content-Length header value. +// +// It may be negative: +// -1 means Transfer-Encoding: chunked. +// -2 means Transfer-Encoding: identity. +func (h *header) ContentLength() int { + return h.contentLength +} + +// SetContentLength sets Content-Length header value. +// +// Content-Length may be negative: +// -1 means Transfer-Encoding: chunked. +// -2 means Transfer-Encoding: identity. +func (h *header) SetContentLength(contentLength int) { + h.contentLength = contentLength + if contentLength >= 0 { + h.contentLengthBytes = appendUint(h.contentLengthBytes[:0], contentLength) + h.h = delAllArgs(h.h, strTransferEncoding) + } else { + h.contentLengthBytes = h.contentLengthBytes[:0] + h.h = setArg(h.h, strTransferEncoding, strChunked, argsHasValue) + } +} + +var ErrBadTrailer = errors.New("contain forbidden trailer") + +// SetTrailer sets Trailer header value for chunked request +// to indicate which headers will be sent after the body. +// +// Use Set to set the trailer header later. +// +// Trailers are only supported with chunked transfer. +// Trailers allow the sender to include additional headers at the end of chunked messages. +// +// The following trailers are forbidden: +// 1. necessary for message framing (e.g., Transfer-Encoding and Content-Length), +// 2. routing (e.g., Host), +// 3. request modifiers (e.g., controls and conditionals in Section 5 of [RFC7231]), +// 4. authentication (e.g., see [RFC7235] and [RFC6265]), +// 5. response control data (e.g., see Section 7.1 of [RFC7231]), +// 6. determining how to process the payload (e.g., Content-Encoding, Content-Type, Content-Range, and Trailer) +// +// Return ErrBadTrailer if contain any forbidden trailers. +func (h *header) SetTrailer(trailer string) error { + h.trailer = h.trailer[:0] + return h.AddTrailer(trailer) +} + +// AddTrailerBytes add Trailer header value for chunked response +// to indicate which headers will be sent after the body. +// +// Use Set to set the trailer header later. +// +// Trailers are only supported with chunked transfer. +// Trailers allow the sender to include additional headers at the end of chunked messages. +// +// The following trailers are forbidden: +// 1. necessary for message framing (e.g., Transfer-Encoding and Content-Length), +// 2. routing (e.g., Host), +// 3. request modifiers (e.g., controls and conditionals in Section 5 of [RFC7231]), +// 4. authentication (e.g., see [RFC7235] and [RFC6265]), +// 5. response control data (e.g., see Section 7.1 of [RFC7231]), +// 6. determining how to process the payload (e.g., Content-Encoding, Content-Type, Content-Range, and Trailer) +// +// Return ErrBadTrailer if contain any forbidden trailers. +func (h *header) AddTrailer(trailer string) error { + h.trace("httpx:AddTrailer", slog.String("trailer", trailer)) + var err error + for i := -1; i+1 < len(trailer); { + trailer = trailer[i+1:] + i = strings.IndexByte(trailer, ',') + if i < 0 { + i = len(trailer) + } + key := stripSpace(trailer[:i]) + // Forbidden by RFC 7230, section 4.1.2 + if isBadTrailer(key) { + err = ErrBadTrailer + continue + } + h.bufKV.key = append(h.bufKV.key[:0], key...) + normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) + h.trailer = appendArg(h.trailer, b2s(h.bufKV.key), "", argsNoValue) + } + + return err +} + +func isBadTrailer(key string) bool { + if len(key) == 0 { + return true + } + + switch key[0] | 0x20 { + case 'a': + return caseInsensitiveCompare(key, strAuthorization) + case 'c': + if len(key) > len(HeaderContentType) && caseInsensitiveCompare(key[:8], strContentType[:8]) { + // skip compare prefix 'Content-' + return caseInsensitiveCompare(key[8:], strContentEncoding[8:]) || + caseInsensitiveCompare(key[8:], strContentLength[8:]) || + caseInsensitiveCompare(key[8:], strContentType[8:]) || + caseInsensitiveCompare(key[8:], strContentRange[8:]) + } + return caseInsensitiveCompare(key, strConnection) + case 'e': + return caseInsensitiveCompare(key, strExpect) + case 'h': + return caseInsensitiveCompare(key, strHost) + case 'k': + return caseInsensitiveCompare(key, strKeepAlive) + case 'm': + return caseInsensitiveCompare(key, strMaxForwards) + case 'p': + if len(key) > len(HeaderProxyConnection) && caseInsensitiveCompare(key[:6], strProxyConnection[:6]) { + // skip compare prefix 'Proxy-' + return caseInsensitiveCompare(key[6:], strProxyConnection[6:]) || + caseInsensitiveCompare(key[6:], strProxyAuthenticate[6:]) || + caseInsensitiveCompare(key[6:], strProxyAuthorization[6:]) + } + case 'r': + return caseInsensitiveCompare(key, strRange) + case 't': + return caseInsensitiveCompare(key, strTE) || + caseInsensitiveCompare(key, strTrailer) || + caseInsensitiveCompare(key, strTransferEncoding) + case 'w': + return caseInsensitiveCompare(key, strWWWAuthenticate) + } + return false +} + +// RawHeaders returns raw header key/value bytes. +// +// Depending on server configuration, header keys may be normalized to +// capital-case in place. +// +// This copy is set aside during parsing, so empty slice is returned for all +// cases where parsing did not happen. Similarly, request line is not stored +// during parsing and can not be returned. +// +// The slice is not safe to use after the handler returns. +func (h *header) RawHeaders() []byte { + return h.rawHeaders +} + +// DisableNormalizing disables header names' normalization. +// +// By default all the header names are normalized by uppercasing +// the first letter and all the first letters following dashes, +// while lowercasing all the other letters. +// Examples: +// +// - CONNECTION -> Connection +// - conteNT-tYPE -> Content-Type +// - foo-bar-baz -> Foo-Bar-Baz +// +// Disable header names' normalization only if know what are you doing. +func (h *header) DisableNormalizing() { + h.disableNormalizing = true +} + +// DisableSpecialHeader disables special header processing. +// fasthttp will not set any special headers for you, such as Host, Content-Type, User-Agent, etc. +// You must set everything yourself. +// If RequestHeader.Read() is called, special headers will be ignored. +// This can be used to control case and order of special headers. +// This is generally not recommended. +func (h *header) DisableSpecialHeader() { + h.disableSpecialHeader = true +} + +// Method returns HTTP request method. +func (h *header) Method() []byte { + if len(h.method) == 0 { + h.method = append(h.method, http.MethodGet...) + } + return h.method +} + +func (h *header) SetMethod(method string) { + h.method = append(h.method[:0], method...) +} + +// SetRequestURI sets RequestURI for the first HTTP request line. +func (h *header) SetRequestURI(requestURI string) { + h.requestURI = append(h.requestURI[:0], requestURI...) +} + +// RequestURI returns RequestURI from the first HTTP request line. +func (h *header) RequestURI() []byte { + requestURI := h.requestURI + if len(requestURI) == 0 { + requestURI = append(requestURI, '/') + } + return requestURI +} + +// Protocol returns HTTP protocol. +func (h *header) Protocol() []byte { + if len(h.proto) == 0 { + h.proto = append(h.proto, strHTTP11...) + } + return h.proto +} + +func (h *header) SetProtocol(protocol string) { + h.proto = append(h.proto[:0], protocol...) +} + +// AppendReqRespCommon appends request/response common header representation to dst and returns the extended buffer. +func (h *header) AppendReqRespCommon(dst []byte) []byte { + for i, n := 0, len(h.h); i < n; i++ { + kv := &h.h[i] + // Exclude trailer from header + exclude := false + for _, t := range h.trailer { + if b2s(kv.key) == b2s(t.key) { + exclude = true + break + } + } + if !exclude { + dst = appendHeaderLine(dst, b2s(kv.key), b2s(kv.value)) + } + } + + if len(h.trailer) > 0 { + aux := appendArgsKey(nil, h.trailer, strCommaSpace) + dst = appendHeaderLine(dst, strTrailer, b2s(aux)) + } + + // there is no need in h.collectCookies() here, since if cookies aren't collected yet, + // they all are located in h.h. + n := len(h.cookies) + if n > 0 && !h.disableSpecialHeader { + dst = append(dst, strCookie...) + dst = append(dst, strColonSpace...) + dst = appendRequestCookieBytes(dst, h.cookies) + dst = append(dst, strCRLF...) + } + + if h.ConnectionClose() && !h.disableSpecialHeader { + dst = appendHeaderLine(dst, strConnection, strClose) + } + + return append(dst, strCRLF...) +} + +func appendHeaderLine(dst []byte, key, value string) []byte { + dst = append(dst, key...) + dst = append(dst, strColonSpace...) + dst = append(dst, value...) + return append(dst, strCRLF...) +} + +func (h *header) ignoreBody() bool { + return h.IsGet() || h.IsHead() +} + +func (h *header) collectCookies() { + if h.cookiesCollected { + return + } + + for i, n := 0, len(h.h); i < n; i++ { + kv := &h.h[i] + if caseInsensitiveCompare(b2s(kv.key), strCookie) { + h.cookies = parseRequestCookies(h.cookies, b2s(kv.value)) + tmp := *kv + copy(h.h[i:], h.h[i+1:]) + n-- + i-- + h.h[n] = tmp + h.h = h.h[:n] + } + } + h.cookiesCollected = true +} + +func (h *header) MethodIs(method string) bool { + return b2s(h.method) == method +} + +// IsGet returns true if request method is GET. +func (h *header) IsGet() bool { return len(h.method) == 0 || h.MethodIs(http.MethodGet) } + +// IsHead returns true if request method is HEAD. +func (h *header) IsHead() bool { return h.MethodIs(http.MethodHead) } + +// IsPost returns true if request method is POST. +func (h *header) IsPost() bool { return h.MethodIs(http.MethodPost) } + +// IsPut returns true if request method is PUT. +func (h *header) IsPut() bool { return h.MethodIs(http.MethodPut) } + +// IsDelete returns true if request method is DELETE. +func (h *header) IsDelete() bool { return h.MethodIs(http.MethodDelete) } + +// IsConnect returns true if request method is CONNECT. +func (h *header) IsConnect() bool { return h.MethodIs(http.MethodConnect) } + +// IsOptions returns true if request method is OPTIONS. +func (h *header) IsOptions() bool { return h.MethodIs(http.MethodOptions) } + +// IsTrace returns true if request method is TRACE. +func (h *header) IsTrace() bool { return h.MethodIs(http.MethodTrace) } + +// IsPatch returns true if request method is PATCH. +func (h *header) IsPatch() bool { return h.MethodIs(http.MethodPatch) } + +// IsHTTP11 returns true if the request is HTTP/1.1. +func (h *header) IsHTTP11() bool { return !h.noHTTP11 } + +// Embed this type into a struct, which mustn't be copied, +// so `go vet` gives a warning if this struct is copied. +// +// See https://github.com/golang/go/issues/8005#issuecomment-190753527 for details. +// and also: https://stackoverflow.com/questions/52494458/nocopy-minimal-example +type noCopy struct{} + +func (*noCopy) Lock() {} +func (*noCopy) Unlock() {} + +func (h *header) trace(msg string, attrs ...slog.Attr) { + internal.LogAttrs(h.logger, internal.LevelTrace, msg, attrs...) +} +func (h *header) debug(msg string, attrs ...slog.Attr) { + internal.LogAttrs(h.logger, slog.LevelDebug, msg, attrs...) +} +func (h *header) info(msg string, attrs ...slog.Attr) { + internal.LogAttrs(h.logger, slog.LevelInfo, msg, attrs...) +} + +func normalizeHeaderKey(b []byte, disableNormalizing bool) { + if disableNormalizing { + return + } + + n := len(b) + if n == 0 { + return + } + + b[0] = toUpperTable[b[0]] + for i := 1; i < n; i++ { + p := &b[i] + if *p == '-' { + i++ + if i < n { + b[i] = toUpperTable[b[i]] + } + continue + } + *p = toLowerTable[*p] + } +} From 095c4e1b57618658daf4a9cfe5a1fedb6ca35067 Mon Sep 17 00:00:00 2001 From: soypat Date: Sat, 24 May 2025 16:09:35 -0300 Subject: [PATCH 2/8] omg almost done with httpx port to super smol http --- httpx/cookie.go | 700 ++++++++++++++++++++++++++++++++++++++++++ httpx/header_parse.go | 565 +++++++++++++++------------------- httpx/tokenizer.go | 461 +++++----------------------- 3 files changed, 1034 insertions(+), 692 deletions(-) create mode 100644 httpx/cookie.go diff --git a/httpx/cookie.go b/httpx/cookie.go new file mode 100644 index 0000000..611af48 --- /dev/null +++ b/httpx/cookie.go @@ -0,0 +1,700 @@ +package httpx + +import ( + "bytes" + "errors" + "io" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +var zeroTime time.Time + +var ( + // cookieExpireDelete may be set on Cookie.Expire for expiring the given cookie. + cookieExpireDelete = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) + + // cookieExpireUnlimited indicates that the cookie doesn't expire. + cookieExpireUnlimited = zeroTime +) + +// CookieSameSite is an enum for the mode in which the SameSite flag should be set for the given cookie. +// See https://tools.ietf.org/html/draft-ietf-httpbis-cookie-same-site-00 for details. +type CookieSameSite int + +const ( + // CookieSameSiteDisabled removes the SameSite flag. + CookieSameSiteDisabled CookieSameSite = iota + // CookieSameSiteDefaultMode sets the SameSite flag. + CookieSameSiteDefaultMode + // CookieSameSiteLaxMode sets the SameSite flag with the "Lax" parameter. + CookieSameSiteLaxMode + // CookieSameSiteStrictMode sets the SameSite flag with the "Strict" parameter. + CookieSameSiteStrictMode + // CookieSameSiteNoneMode sets the SameSite flag with the "None" parameter. + // See https://tools.ietf.org/html/draft-west-cookie-incrementalism-00 + CookieSameSiteNoneMode +) + +// acquireCookie returns an empty Cookie object from the pool. +// +// The returned object may be returned back to the pool with ReleaseCookie. +// This allows reducing GC load. +func acquireCookie() *Cookie { + return cookiePool.Get().(*Cookie) +} + +// releaseCookie returns the Cookie object acquired with AcquireCookie back +// to the pool. +// +// Do not access released Cookie object, otherwise data races may occur. +func releaseCookie(c *Cookie) { + c.Reset() + cookiePool.Put(c) +} + +var cookiePool = &sync.Pool{ + New: func() any { + return &Cookie{} + }, +} + +// Cookie represents HTTP response cookie. +// +// Do not copy Cookie objects. Create new object and use CopyTo instead. +// +// Cookie instance MUST NOT be used from concurrently running goroutines. +type Cookie struct { + noCopy noCopy + + key []byte + value []byte + expire time.Time + maxAge int + domain []byte + path []byte + + httpOnly bool + secure bool + sameSite CookieSameSite + + bufKV argsKV + buf []byte +} + +// CopyTo copies src cookie to c. +func (c *Cookie) CopyTo(src *Cookie) { + c.Reset() + c.key = append(c.key, src.key...) + c.value = append(c.value, src.value...) + c.expire = src.expire + c.maxAge = src.maxAge + c.domain = append(c.domain, src.domain...) + c.path = append(c.path, src.path...) + c.httpOnly = src.httpOnly + c.secure = src.secure + c.sameSite = src.sameSite +} + +// HTTPOnly returns true if the cookie is http only. +func (c *Cookie) HTTPOnly() bool { + return c.httpOnly +} + +// SetHTTPOnly sets cookie's httpOnly flag to the given value. +func (c *Cookie) SetHTTPOnly(httpOnly bool) { + c.httpOnly = httpOnly +} + +// Secure returns true if the cookie is secure. +func (c *Cookie) Secure() bool { + return c.secure +} + +// SetSecure sets cookie's secure flag to the given value. +func (c *Cookie) SetSecure(secure bool) { + c.secure = secure +} + +// SameSite returns the SameSite mode. +func (c *Cookie) SameSite() CookieSameSite { + return c.sameSite +} + +// SetSameSite sets the cookie's SameSite flag to the given value. +// Set value CookieSameSiteNoneMode will set Secure to true also to avoid browser rejection. +func (c *Cookie) SetSameSite(mode CookieSameSite) { + c.sameSite = mode + if mode == CookieSameSiteNoneMode { + c.SetSecure(true) + } +} + +// Path returns cookie path. +func (c *Cookie) Path() []byte { + return c.path +} + +// SetPath sets cookie path. +func (c *Cookie) SetPath(path string) { + c.buf = append(c.buf[:0], path...) + c.path = normalizePath(c.path, b2s(c.buf)) +} + +// SetPathBytes sets cookie path. +func (c *Cookie) SetPathBytes(path []byte) { + c.buf = append(c.buf[:0], path...) + c.path = normalizePath(c.path, b2s(c.buf)) +} + +// Domain returns cookie domain. +// +// The returned value is valid until the Cookie reused or released (ReleaseCookie). +// Do not store references to the returned value. Make copies instead. +func (c *Cookie) Domain() []byte { + return c.domain +} + +// SetDomain sets cookie domain. +func (c *Cookie) SetDomain(domain string) { + c.domain = append(c.domain[:0], domain...) +} + +// SetDomainBytes sets cookie domain. +func (c *Cookie) SetDomainBytes(domain []byte) { + c.domain = append(c.domain[:0], domain...) +} + +// MaxAge returns the seconds until the cookie is meant to expire or 0 +// if no max age. +func (c *Cookie) MaxAge() int { + return c.maxAge +} + +// SetMaxAge sets cookie expiration time based on seconds. This takes precedence +// over any absolute expiry set on the cookie. +// +// Set max age to 0 to unset. +func (c *Cookie) SetMaxAge(seconds int) { + c.maxAge = seconds +} + +// Expire returns cookie expiration time. +// +// CookieExpireUnlimited is returned if cookie doesn't expire. +func (c *Cookie) Expire() time.Time { + expire := c.expire + if expire.IsZero() { + expire = cookieExpireUnlimited + } + return expire +} + +// SetExpire sets cookie expiration time. +// +// Set expiration time to CookieExpireDelete for expiring (deleting) +// the cookie on the client. +// +// By default cookie lifetime is limited by browser session. +func (c *Cookie) SetExpire(expire time.Time) { + c.expire = expire +} + +// Value returns cookie value. +// +// The returned value is valid until the Cookie reused or released (ReleaseCookie). +// Do not store references to the returned value. Make copies instead. +func (c *Cookie) Value() []byte { + return c.value +} + +// SetValue sets cookie value. +func (c *Cookie) SetValue(value string) { + c.value = append(c.value[:0], value...) +} + +// SetValueBytes sets cookie value. +func (c *Cookie) SetValueBytes(value []byte) { + c.value = append(c.value[:0], value...) +} + +// Key returns cookie name. +// +// The returned value is valid until the Cookie reused or released (ReleaseCookie). +// Do not store references to the returned value. Make copies instead. +func (c *Cookie) Key() []byte { + return c.key +} + +// SetKey sets cookie name. +func (c *Cookie) SetKey(key string) { + c.key = append(c.key[:0], key...) +} + +// SetKeyBytes sets cookie name. +func (c *Cookie) SetKeyBytes(key []byte) { + c.key = append(c.key[:0], key...) +} + +// Reset clears the cookie. +func (c *Cookie) Reset() { + c.key = c.key[:0] + c.value = c.value[:0] + c.expire = zeroTime + c.maxAge = 0 + c.domain = c.domain[:0] + c.path = c.path[:0] + c.httpOnly = false + c.secure = false + c.sameSite = CookieSameSiteDisabled +} + +// AppendBytes appends cookie representation to dst and returns +// the extended dst. +func (c *Cookie) AppendBytes(dst []byte) []byte { + if len(c.key) > 0 { + dst = append(dst, c.key...) + dst = append(dst, '=') + } + dst = append(dst, c.value...) + + if c.maxAge > 0 { + dst = append(dst, ';', ' ') + dst = append(dst, strCookieMaxAge...) + dst = append(dst, '=') + dst = appendUint(dst, c.maxAge) + } else if !c.expire.IsZero() { + dst = append(dst, ';', ' ') + dst = append(dst, strCookieExpires...) + dst = append(dst, '=') + dst = AppendHTTPDate(dst, c.expire) + } + if len(c.domain) > 0 { + dst = appendCookiePart(dst, strCookieDomain, b2s(c.domain)) + } + if len(c.path) > 0 { + dst = appendCookiePart(dst, strCookiePath, b2s(c.path)) + } + if c.httpOnly { + dst = append(dst, ';', ' ') + dst = append(dst, strCookieHTTPOnly...) + } + if c.secure { + dst = append(dst, ';', ' ') + dst = append(dst, strCookieSecure...) + } + switch c.sameSite { + case CookieSameSiteDefaultMode: + dst = append(dst, ';', ' ') + dst = append(dst, strCookieSameSite...) + case CookieSameSiteLaxMode: + dst = append(dst, ';', ' ') + dst = append(dst, strCookieSameSite...) + dst = append(dst, '=') + dst = append(dst, strCookieSameSiteLax...) + case CookieSameSiteStrictMode: + dst = append(dst, ';', ' ') + dst = append(dst, strCookieSameSite...) + dst = append(dst, '=') + dst = append(dst, strCookieSameSiteStrict...) + case CookieSameSiteNoneMode: + dst = append(dst, ';', ' ') + dst = append(dst, strCookieSameSite...) + dst = append(dst, '=') + dst = append(dst, strCookieSameSiteNone...) + } + return dst +} + +// Cookie returns cookie representation. +// +// The returned value is valid until the Cookie reused or released (ReleaseCookie). +// Do not store references to the returned value. Make copies instead. +func (c *Cookie) Cookie() []byte { + c.buf = c.AppendBytes(c.buf[:0]) + return c.buf +} + +// String returns cookie representation. +func (c *Cookie) String() string { + return string(c.Cookie()) +} + +// WriteTo writes cookie representation to w. +// +// WriteTo implements io.WriterTo interface. +func (c *Cookie) WriteTo(w io.Writer) (int64, error) { + n, err := w.Write(c.Cookie()) + return int64(n), err +} + +var errNoCookies = errors.New("no cookies found") + +// Parse parses Set-Cookie header. +func (c *Cookie) Parse(src string) error { + c.buf = append(c.buf[:0], src...) + return c.ParseBytes(c.buf) +} + +// ParseBytes parses Set-Cookie header. +func (c *Cookie) ParseBytes(src []byte) error { + c.Reset() + ntot := 0 + for { + k, v, n := parseCookie(src) + if n == 0 { + break + } else if ntot == 0 { + c.key = append(c.key, k...) + c.value = append(c.value, v...) + } + key := b2s(k) + value := b2s(v) + ntot += n + src = src[n:] + if len(key) != 0 { + // Case insensitive switch on first char + switch key[0] | 0x20 { + case 'm': + if caseInsensitiveCompare(strCookieMaxAge, key) { + maxAge, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return err + } + c.maxAge = int(maxAge) + } + + case 'e': // "expires" + if caseInsensitiveCompare(strCookieExpires, key) { + + // Try the same two formats as net/http + // See: https://github.com/golang/go/blob/00379be17e63a5b75b3237819392d2dc3b313a27/src/net/http/cookie.go#L133-L135 + exptime, err := time.ParseInLocation(time.RFC1123, value, time.UTC) + if err != nil { + exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", value) + if err != nil { + return err + } + } + c.expire = exptime + } + + case 'd': // "domain" + if caseInsensitiveCompare(strCookieDomain, key) { + c.domain = append(c.domain, value...) + } + + case 'p': // "path" + if caseInsensitiveCompare(strCookiePath, key) { + c.path = append(c.path, value...) + } + + case 's': // "samesite" + if caseInsensitiveCompare(strCookieSameSite, key) { + if len(value) > 0 { + // Case insensitive switch on first char + switch value[0] | 0x20 { + case 'l': // "lax" + if caseInsensitiveCompare(strCookieSameSiteLax, value) { + c.sameSite = CookieSameSiteLaxMode + } + case 's': // "strict" + if caseInsensitiveCompare(strCookieSameSiteStrict, value) { + c.sameSite = CookieSameSiteStrictMode + } + case 'n': // "none" + if caseInsensitiveCompare(strCookieSameSiteNone, value) { + c.sameSite = CookieSameSiteNoneMode + } + } + } + } + } + } else if len(value) != 0 { + // Case insensitive switch on first char + switch value[0] | 0x20 { + case 'h': // "httponly" + if caseInsensitiveCompare(strCookieHTTPOnly, value) { + c.httpOnly = true + } + + case 's': // "secure" + if caseInsensitiveCompare(strCookieSecure, value) { + c.secure = true + } else if caseInsensitiveCompare(strCookieSameSite, value) { + c.sameSite = CookieSameSiteDefaultMode + } + } + } // else empty or no match + } + if len(c.key) == 0 && len(c.value) == 0 { + return errNoCookies + } + return nil +} + +func appendCookiePart(dst []byte, key, value string) []byte { + dst = append(dst, ';', ' ') + dst = append(dst, key...) + dst = append(dst, '=') + return append(dst, value...) +} + +func (hb *headerBuf) appendRequestCookieBytes(dst []byte) []byte { + n := len(hb.cookies) + for i := 0; i < n; i++ { + kv := hb.cookies[i] + if !kv.isValid() { + continue + } else if kv.key.len > 0 { + dst = append(dst, hb.musttoken(kv.key)...) + dst = append(dst, '=') + } + dst = append(dst, hb.musttoken(kv.value)...) + if i+1 < n { + dst = append(dst, ';', ' ') + } + } + return dst +} +func (hb *headerBuf) appendResponseCookieBytes(dst []byte) []byte { + n := len(hb.cookies) + for i := 0; i < n; i++ { + kv := hb.cookies[i] + if !kv.isValid() { + continue + } + dst = append(dst, hb.musttoken(kv.value)...) + if i+1 < n { + dst = append(dst, ';', ' ') + } + } + return dst +} + +type cookieScanner struct { + b []byte +} + +// parseCookie parses a cookie inside cookie buffer and adds it to cookie buffer.. +// +// Cookie: \r\n +func parseCookie(cookie []byte) (key, value []byte, cookieEnd int) { + if len(cookie) == 0 { + return nil, nil, 0 + } + eqIdx := bytes.IndexByte(cookie, '=') + semiIdx := bytes.IndexByte(cookie, ';') + if eqIdx > 0 && eqIdx < semiIdx { + // cookies has form key=value; + key = trimCookie(cookie[:eqIdx], false) + } else { + // cookie has no key. + eqIdx = -1 // ensure is -1. + } + if semiIdx > 0 { + // found ';' + value = trimCookie(cookie[eqIdx+1:semiIdx], true) + } else { + value = trimCookie(cookie[eqIdx+1:], true) + } + return key, value, max(len(cookie), semiIdx+1) +} + +func trimCookie(src []byte, trimQuotes bool) []byte { + for len(src) > 0 && src[0] == ' ' { + src = src[1:] // skip leading whitespace. + } + for len(src) > 0 && src[len(src)-1] == ' ' { + src = src[:len(src)-1] // skip trailing whitespace + } + if trimQuotes { + if len(src) > 1 && src[0] == '"' && src[len(src)-1] == '"' { + src = src[1 : len(src)-1] // Trim leading+trailing quotes. + } + } + return src +} + +// caseInsensitiveCompare does a case insensitive equality comparison of +// two []byte. Assumes only letters need to be matched. +func caseInsensitiveCompare(a, b string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i]|0x20 != b[i]|0x20 { + return false + } + } + return true +} + +func normalizePath(dst []byte, src string) []byte { + dst = dst[:0] + dst = addLeadingSlash(dst, src) + dst = decodeArgAppendNoPlus(dst, src) + + // remove duplicate slashes + b := dst + bSize := len(b) + for { + n := strings.Index(b2s(b), strSlashSlash) + if n < 0 { + break + } + b = b[n:] + copy(b, b[1:]) + b = b[:len(b)-1] + bSize-- + } + dst = dst[:bSize] + + // remove /./ parts + b = dst + for { + n := strings.Index(b2s(b), strSlashDotSlash) + if n < 0 { + break + } + nn := n + len(strSlashDotSlash) - 1 + copy(b[n:], b[nn:]) + b = b[:len(b)-nn+n] + } + + // remove /foo/../ parts + for { + n := strings.Index(b2s(b), strSlashDotDotSlash) + if n < 0 { + break + } + nn := strings.LastIndexByte(b2s(b[:n]), slashChar) + if nn < 0 { + nn = 0 + } + n += len(strSlashDotDotSlash) - 1 + copy(b[nn:], b[n:]) + b = b[:len(b)-n+nn] + } + + // remove trailing /foo/.. + n := strings.LastIndex(b2s(b), strSlashDotDot) + if n >= 0 && n+len(strSlashDotDot) == len(b) { + nn := strings.LastIndexByte(b2s(b[:n]), slashChar) + if nn < 0 { + return append(dst[:0], slashChar) + } + b = b[:nn+1] + } + + if filepath.Separator == '\\' { + // remove \.\ parts + for { + n := strings.Index(b2s(b), strBackSlashDotBackSlash) + if n < 0 { + break + } + nn := n + len(strSlashDotSlash) - 1 + copy(b[n:], b[nn:]) + b = b[:len(b)-nn+n] + } + + // remove /foo/..\ parts + for { + n := strings.Index(b2s(b), strSlashDotDotBackSlash) + if n < 0 { + break + } + nn := strings.LastIndexByte(b2s(b[:n]), slashChar) + if nn < 0 { + nn = 0 + } + nn++ + n += len(strSlashDotDotBackSlash) + copy(b[nn:], b[n:]) + b = b[:len(b)-n+nn] + } + + // remove /foo\..\ parts + for { + n := strings.Index(b2s(b), strBackSlashDotDotBackSlash) + if n < 0 { + break + } + nn := strings.LastIndexByte(b2s(b[:n]), slashChar) + if nn < 0 { + nn = 0 + } + n += len(strBackSlashDotDotBackSlash) - 1 + copy(b[nn:], b[n:]) + b = b[:len(b)-n+nn] + } + + // remove trailing \foo\.. + n := strings.LastIndex(b2s(b), strBackSlashDotDot) + if n >= 0 && n+len(strSlashDotDot) == len(b) { + nn := strings.LastIndexByte(b2s(b[:n]), slashChar) + if nn < 0 { + return append(dst[:0], slashChar) + } + b = b[:nn+1] + } + } + + return b +} + +func addLeadingSlash(dst []byte, src string) []byte { + // add leading slash for unix paths + if len(src) == 0 || src[0] != slashChar { + dst = append(dst, slashChar) + } + + return dst +} + +// decodeArgAppendNoPlus is almost identical to decodeArgAppend, but it doesn't +// substitute '+' with ' '. +// +// The function is copy-pasted from decodeArgAppend due to the performance +// reasons only. +func decodeArgAppendNoPlus(dst []byte, src string) []byte { + idx := strings.IndexByte(src, '%') + if idx < 0 { + // fast path: src doesn't contain encoded chars + return append(dst, src...) + } + dst = append(dst, src[:idx]...) + + // slow path + for i := idx; i < len(src); i++ { + c := src[i] + if c == '%' { + if i+2 >= len(src) { + return append(dst, src[i:]...) + } + x2 := hex2intTable[src[i+2]] + x1 := hex2intTable[src[i+1]] + if x1 == 16 || x2 == 16 { + dst = append(dst, '%') + } else { + dst = append(dst, x1<<4|x2) + i += 2 + } + } else { + dst = append(dst, c) + } + } + return dst +} + +// AppendHTTPDate appends HTTP-compliant (RFC1123) representation of date +// to dst and returns the extended dst. +func AppendHTTPDate(dst []byte, date time.Time) []byte { + dst = date.In(time.UTC).AppendFormat(dst, time.RFC1123) + copy(dst[len(dst)-3:], strGMT) + return dst +} diff --git a/httpx/header_parse.go b/httpx/header_parse.go index 915541c..5045244 100644 --- a/httpx/header_parse.go +++ b/httpx/header_parse.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "slices" "strings" "unsafe" ) @@ -17,49 +18,37 @@ var ( errNonNumericChars = errors.New("non-numeric chars found") ) -type headerScanner struct { - b []byte - key []byte - value []byte - err error - - // hLen stores header subslice len - hLen int - - disableNormalizing bool - - // by checking whether the next line contains a colon or not to tell - // it's a header entry or a multi line value of current header entry. - // the side effect of this operation is that we know the index of the - // next colon and new line, so this can be used during next iteration, - // instead of find them again. - nextColon int - nextNewLine int - - initialized bool +func (hb *headerBuf) readFromBytes(b []byte) { + hb.buf = append(hb.buf, b...) } -type headerValueScanner struct { - b string - value string +func (hb *headerBuf) free() int { return cap(hb.buf) - len(hb.buf) } + +func (hb *headerBuf) readFrom(r io.Reader) error { + buf := hb.buf + free := hb.free() + if free == 0 { + return errSmallBuffer + } + n, err := r.Read(buf[len(buf):cap(buf)]) + hb.buf = buf[:len(buf)+n] + return err } -func (h *header) parse(buf []byte) (int, error) { - m, err := h.parseFirstLine(buf) +func (h *header) parse() (err error) { + hb := &h.hbuf + hb.off = 0 // start parsing from 0. + h.method, h.requestURI, h.proto, h.flags, err = hb.parseFirstLine(h.flags) if err != nil { - return 0, err + return err } - h.rawHeaders, _, err = readRawHeaders(h.rawHeaders[:0], b2s(buf[m:])) + var ss scannerState + err = h.parseHeaders(&ss) if err != nil { - return 0, err + return err } - var n int - n, err = h.parseHeaders(buf[m:]) - if err != nil { - return 0, err - } - return m + n, nil + return nil } func (hb *headerBuf) offBuf() []byte { @@ -87,13 +76,14 @@ func (hb *headerBuf) scanUntilByte(c byte) []byte { return buf } -func (hb *headerBuf) parseFirstLine() (method, uri, proto headerSlice, flags flags, err error) { +func (hb *headerBuf) parseFirstLine(initFlags flags) (method, uri, proto headerSlice, flags flags, err error) { var b []byte for len(b) == 0 { b = hb.scanLine() } + flags = initFlags if len(b) < 5 { - return method, uri, proto, 0, errors.New("too short first HTTP line") + return method, uri, proto, flags, errors.New("too short first HTTP line") } methodEnd := max(0, bytes.IndexByte(b, ' ')) @@ -115,6 +105,143 @@ func (hb *headerBuf) parseFirstLine() (method, uri, proto headerSlice, flags fla return method, uri, proto, flags, nil } +type scannerState struct { + err error + + // hLen stores header subslice len + hLen int + + disableNormalizing bool + + // by checking whether the next line contains a colon or not to tell + // it's a header entry or a multi line value of current header entry. + // the side effect of this operation is that we know the index of the + // next colon and new line, so this can be used during next iteration, + // instead of find them again. + nextColon int + nextNewLine int + + initialized bool +} + +func (h *header) parseHeaders(ss *scannerState) (err error) { + hb := &h.hbuf + h.contentLength = -2 + + for kv := hb.nextKV(ss); kv.isValid(); kv = hb.nextKV(ss) { + if h.flags.hasAny(disableSpecialHeader) { + h.hbuf.headers = append(h.hbuf.headers, kv) + continue + } + } + if ss.err != nil && err == nil { + err = ss.err + } + if err != nil { + h.flags |= connectionClose + return err + } + + // if h.contentLength < 0 { + // h.contentLengthBytes = hb.noKV().value + // } + if h.flags.hasAny(noHTTP11) && !h.flags.hasAny(connectionClose) { + // close connection for non-http/1.1 request unless 'Connection: keep-alive' is set. + if !h.hasHeaderValue(strConnection, strKeepAlive) { + h.flags |= connectionClose + } + } + return nil +} + +func (h *header) hasHeaderValue(key, value string) bool { + kv := h.peekHeader(key) + return kv.isValid() && b2s(h.hbuf.musttoken(kv.value)) == value +} + +func (h *header) peekHeaderBytes(key string) []byte { + kv := h.peekHeader(key) + if kv.isValid() { + return h.hbuf.musttoken(kv.value) + } + return nil +} + +// peekHeader returns header key-value for the given key. +// +// The returned value is valid until the request is released, +// either though ReleaseRequest or your request handler returning. +// Do not store references to returned value. Make copies instead. +func (h *header) peekHeader(key string) argsKV { + hb := &h.hbuf + for i := 0; i < len(h.hbuf.headers); i++ { + if b2s(hb.musttoken(h.hbuf.headers[i].key)) == key { + return h.hbuf.headers[i] + } + } + return hb.noKV() +} + +func (h *header) peekPtrHeader(key string) *argsKV { + hb := &h.hbuf + for i := 0; i < len(h.hbuf.headers); i++ { + if b2s(hb.musttoken(h.hbuf.headers[i].key)) == key { + return &h.hbuf.headers[i] + } + } + return nil +} + +func (hb *headerBuf) mustAppendSlice(value string) headerSlice { + L := len(hb.buf) + copy(hb.buf[L:L+len(value)], value) + hb.buf = hb.buf[:L+len(value)] + return hb.slice(hb.buf[L : L+len(value)]) +} + +func (h *header) reuseOrAppend(tok headerSlice, value string) headerSlice { + if tok.len > tokint(len(value)) { + copy(h.hbuf.musttoken(tok), value) + tok.len = tokint(len(value)) + return tok + } + return h.appendSlice(value) +} + +func (h *header) appendSlice(value string) headerSlice { + free := h.hbuf.free() + if len(value) > free { + if h.flags.hasAny(flagNoBufferGrow) { + h.flags |= flagOOMReached + return headerSlice{} + } + h.hbuf.buf = slices.Grow(h.hbuf.buf, len(value)) + } + return h.hbuf.mustAppendSlice(value) +} + +func (h *header) appendHeader(key, value string) { + hb := &h.hbuf + free := hb.free() + buf := h.hbuf.buf + + if len(key)+len(value) > free { + if h.flags.hasAny(flagNoBufferGrow) { + panic(errSmallBuffer) + } + slices.Grow(buf, len(key)+len(value)) + } + k := hb.mustAppendSlice(key) + v := hb.mustAppendSlice(value) + if !h.flags.hasAny(disableNormalizing) { + // TODO + } + hb.headers = append(hb.headers, argsKV{ + key: k, + value: v, + }) +} + func readRawHeaders(dst []byte, buf string) ([]byte, int, error) { n := strings.IndexByte(buf, nChar) if n < 0 { @@ -142,217 +269,125 @@ func readRawHeaders(dst []byte, buf string) ([]byte, int, error) { } } } +func (hb *headerBuf) noKV() argsKV { return argsKV{} } -func (h *header) parseHeaders(buf []byte) (int, error) { - h.contentLength = -2 - h.scanner = headerScanner{} - s := &h.scanner - s.b = buf - s.disableNormalizing = h.disableNormalizing - var err error - for s.next() { - key := b2s(s.key) - value := b2s(s.value) - if len(key) > 0 { - // Spaces between the header key and colon are not allowed. - // See RFC 7230, Section 3.2.4. - - if strings.IndexByte(key, ' ') != -1 || strings.IndexByte(key, '\t') != -1 { - err = fmt.Errorf("invalid header key %q", s.key) - continue - } - - if h.disableSpecialHeader { - h.h = appendArg(h.h, key, value, argsHasValue) - continue - } - - switch s.key[0] | 0x20 { - case 'h': - if caseInsensitiveCompare(key, strHost) { - h.host = append(h.host[:0], value...) - continue - } - case 'u': - if caseInsensitiveCompare(key, strUserAgent) { - h.userAgent = append(h.userAgent[:0], value...) - continue - } - case 'c': - if caseInsensitiveCompare(key, strContentType) { - h.contentType = append(h.contentType[:0], value...) - continue - } - if caseInsensitiveCompare(key, strContentLength) { - if h.contentLength != -1 { - var nerr error - if h.contentLength, nerr = parseContentLength(b2s(s.value)); nerr != nil { - if err == nil { - err = nerr - } - h.contentLength = -2 - } else { - h.contentLengthBytes = append(h.contentLengthBytes[:0], value...) - } - } - continue - } - if caseInsensitiveCompare(key, strConnection) { - if b2s(s.value) == strClose { - h.connectionClose = true - } else { - h.connectionClose = false - h.h = appendArg(h.h, key, value, argsHasValue) - } - continue - } - case 't': - if caseInsensitiveCompare(key, strTransferEncoding) { - if value != strIdentity { - h.contentLength = -1 - h.h = setArg(h.h, strTransferEncoding, strChunked, argsHasValue) - } - continue - } - if caseInsensitiveCompare(key, strTrailer) { - if nerr := h.SetTrailer(value); nerr != nil { - if err == nil { - err = nerr - } - } - continue - } - } - } - h.h = appendArg(h.h, key, value, argsHasValue) +func (hb *headerBuf) nextKV(ss *scannerState) argsKV { + if !ss.initialized { + ss.nextColon = -1 + ss.nextNewLine = -1 + ss.initialized = true } - if s.err != nil && err == nil { - err = s.err + buf := hb.buf[hb.off:] + bLen := len(buf) + if bLen >= 2 && buf[0] == rChar && buf[1] == nChar { + hb.off += 2 + return hb.noKV() // \r\n\r\n Ends header. } - if err != nil { - h.connectionClose = true - return 0, err - } - - if h.contentLength < 0 { - h.contentLengthBytes = h.contentLengthBytes[:0] - } - if h.noHTTP11 && !h.connectionClose { - // close connection for non-http/1.1 request unless 'Connection: keep-alive' is set. - v := peekArgStr(h.h, strConnection) - h.connectionClose = !hasHeaderValue(b2s(v), strKeepAlive) - } - return s.hLen, nil -} - -func (s *headerScanner) next() bool { - if !s.initialized { - s.nextColon = -1 - s.nextNewLine = -1 - s.initialized = true - } - bLen := len(s.b) - if bLen >= 2 && s.b[0] == rChar && s.b[1] == nChar { - s.b = s.b[2:] - s.hLen += 2 - return false - } - if bLen >= 1 && s.b[0] == nChar { - s.b = s.b[1:] - s.hLen++ - return false + if bLen >= 1 && buf[0] == nChar { + hb.off++ + return hb.noKV() // \n\n: Ends header. } var n int - if s.nextColon >= 0 { - n = s.nextColon - s.nextColon = -1 + if ss.nextColon >= 0 { + n = ss.nextColon + ss.nextColon = -1 } else { - n = bytes.IndexByte(s.b, ':') + n = bytes.IndexByte(buf, ':') // There can't be a \n inside the header name, check for this. - x := bytes.IndexByte(s.b, nChar) + x := bytes.IndexByte(buf, nChar) if x < 0 { // A header name should always at some point be followed by a \n // even if it's the one that terminates the header block. - s.err = errNeedMore - return false + ss.err = errNeedMore + return hb.noKV() } if x < n { // There was a \n before the : - s.err = errInvalidName - return false + ss.err = errInvalidName + return hb.noKV() } } if n < 0 { - s.err = errNeedMore - return false + ss.err = errNeedMore + return hb.noKV() } - s.key = s.b[:n] - normalizeHeaderKey(s.key, s.disableNormalizing) + + if bytes.IndexByte(buf[:n], ' ') >= 0 || bytes.IndexByte(buf[:n], '\t') >= 0 { + // Spaces between the header key and colon are not allowed. + // See RFC 7230, Section 3.2.4. + ss.err = errInvalidName + return hb.noKV() + } + + var resultKV argsKV + resultKV.key = hb.slice(buf[:n]) + normalizeHeaderKey(buf[:n], ss.disableNormalizing) n++ - for len(s.b) > n && s.b[n] == ' ' { + for len(buf) > n && buf[n] == ' ' { n++ // the newline index is a relative index, and lines below trimmed `s.b` by `n`, // so the relative newline index also shifted forward. it's safe to decrease // to a minus value, it means it's invalid, and will find the newline again. - s.nextNewLine-- + ss.nextNewLine-- } - s.hLen += n - s.b = s.b[n:] - if s.nextNewLine >= 0 { - n = s.nextNewLine - s.nextNewLine = -1 + ss.hLen += n + buf = buf[n:] + if ss.nextNewLine >= 0 { + n = ss.nextNewLine + ss.nextNewLine = -1 } else { - n = bytes.IndexByte(s.b, nChar) + n = bytes.IndexByte(buf, nChar) } if n < 0 { - s.err = errNeedMore - return false + ss.err = errNeedMore + return hb.noKV() } isMultiLineValue := false for { - if n+1 >= len(s.b) { + if n+1 >= len(buf) { break } - if s.b[n+1] != ' ' && s.b[n+1] != '\t' { + if buf[n+1] != ' ' && buf[n+1] != '\t' { break } - d := bytes.IndexByte(s.b[n+1:], nChar) + d := bytes.IndexByte(buf[n+1:], nChar) if d <= 0 { break - } else if d == 1 && s.b[n+1] == rChar { + } else if d == 1 && buf[n+1] == rChar { break } e := n + d + 1 - if c := bytes.IndexByte(s.b[n+1:e], ':'); c >= 0 { - s.nextColon = c - s.nextNewLine = d - c - 1 + if c := bytes.IndexByte(buf[n+1:e], ':'); c >= 0 { + ss.nextColon = c + ss.nextNewLine = d - c - 1 break } isMultiLineValue = true n = e } - if n >= len(s.b) { - s.err = errNeedMore - return false + if n >= len(buf) { + ss.err = errNeedMore + return hb.noKV() } - oldB := s.b - s.value = s.b[:n] - s.hLen += n + 1 - s.b = s.b[n+1:] + oldB := buf + value := buf[:n] + ss.hLen += n + 1 + buf = buf[n+1:] - if n > 0 && s.value[n-1] == rChar { + if n > 0 && value[n-1] == rChar { n-- } - for n > 0 && s.value[n-1] == ' ' { + for n > 0 && value[n-1] == ' ' { n-- } - s.value = s.value[:n] + value = value[:n] if isMultiLineValue { - s.value, s.b, s.hLen = normalizeHeaderValue(s.value, oldB, s.hLen) + value, buf, ss.hLen = normalizeHeaderValue(value, oldB, ss.hLen) } - return true + resultKV.value = hb.slice(value) + return resultKV } func normalizeHeaderKey(b []byte, disableNormalizing bool) { @@ -437,33 +472,6 @@ func parseContentLength(b string) (int, error) { return v, nil } -func hasHeaderValue(s, value string) bool { - var vs headerValueScanner - vs.b = s - for vs.next() { - if caseInsensitiveCompare(vs.value, value) { - return true - } - } - return false -} - -func (s *headerValueScanner) next() bool { - b := s.b - if len(b) == 0 { - return false - } - n := strings.IndexByte(b, ',') - if n < 0 { - s.value = stripSpace(b) - s.b = b[len(b):] - return true - } - s.value = stripSpace(b[:n]) - s.b = b[n+1:] - return true -} - func nextLine(b []byte) ([]byte, []byte, error) { nNext := bytes.IndexByte(b, nChar) if nNext < 0 { @@ -552,6 +560,7 @@ func (h *header) readLoop(r *bufio.Reader, waitForMore bool) error { func (h *header) tryRead(r *bufio.Reader, n int) error { h.resetSkipNormalize() b, err := r.Peek(n) + if len(b) == 0 { if err == io.EOF { return err @@ -577,35 +586,28 @@ func (h *header) tryRead(r *bufio.Reader, n int) error { return fmt.Errorf("error when reading request headers: %w", err) } b = mustPeekBuffered(r) - headersLen, errParse := h.parse(b) + errParse := h.parse() if errParse != nil { return headerError("request", err, errParse, b, false) } - mustDiscard(r, headersLen) + // mustDiscard(r, headersLen) return nil } +func (h *headerBuf) reset() { + *h = headerBuf{ + buf: h.buf[:0], + headers: h.headers[:0], + cookies: h.cookies[:0], + } +} + func (h *header) resetSkipNormalize() { - h.noHTTP11 = false - h.connectionClose = false - - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - - h.method = h.method[:0] - h.proto = h.proto[:0] - h.requestURI = h.requestURI[:0] - h.host = h.host[:0] - h.contentType = h.contentType[:0] - h.userAgent = h.userAgent[:0] - h.trailer = h.trailer[:0] - h.mulHeader = h.mulHeader[:0] - - h.h = h.h[:0] - h.cookies = h.cookies[:0] - h.cookiesCollected = false - - h.rawHeaders = h.rawHeaders[:0] + h.hbuf.reset() + *h = header{ + hbuf: h.hbuf, + logger: h.logger, + } } func headerError(typ string, err, errParse error, b []byte, secureErrorLogMessage bool) error { @@ -638,6 +640,7 @@ func isOnlyCRLF(b []byte) bool { } return true } + func headerErrorMsg(typ string, err error, b []byte, secureErrorLogMessage bool) error { return fmt.Errorf("error when reading %s headers: %w. Buffer size=%d", typ, err, len(b)) } @@ -671,89 +674,19 @@ func mustDiscard(r *bufio.Reader, n int) { } } -// Peek returns header value for the given key. -// -// The returned value is valid until the request is released, -// either though ReleaseRequest or your request handler returning. -// Do not store references to returned value. Make copies instead. -func (h *header) Peek(key string) []byte { - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - return h.peek(b2s(k)) -} - // Host returns Host header value. func (h *header) Host() []byte { - if h.disableSpecialHeader { - return peekArg(h.h, HeaderHost) - } - return h.host -} - -func getHeaderKeyBytes(kv *argsKV, key string, disableNormalizing bool) []byte { - kv.key = append(kv.key[:0], key...) - normalizeHeaderKey(kv.key, disableNormalizing) - return kv.key -} - -func peekArg(h []argsKV, k string) []byte { - for i, n := 0, len(h); i < n; i++ { - kv := &h[i] - if b2s(kv.key) == k { - return kv.value - } - } - return nil -} - -func (h *header) peek(key string) []byte { - switch key { - case HeaderHost: - return h.Host() - case HeaderContentType: - return h.ContentType() - case HeaderUserAgent: - return h.UserAgent() - case HeaderConnection: - if h.ConnectionClose() { - return []byte(strClose) - } - return peekArg(h.h, key) - case HeaderContentLength: - return h.contentLengthBytes - case HeaderCookie: - if h.cookiesCollected { - return appendRequestCookieBytes(nil, h.cookies) - } - return peekArg(h.h, key) - case HeaderTrailer: - return appendArgsKey(nil, h.trailer, strCommaSpace) - default: - return peekArg(h.h, key) - } + return h.peekHeaderBytes(HeaderHost) } // ConnectionClose returns true if 'Connection: close' header is set. func (h *header) ConnectionClose() bool { - return h.connectionClose + return h.flags.hasAny(connectionClose) } // UserAgent returns User-Agent header value. func (h *header) UserAgent() []byte { - if h.disableSpecialHeader { - return peekArg(h.h, HeaderUserAgent) - } - return h.userAgent -} - -func appendArgsKey(dst []byte, args []argsKV, sep string) []byte { - for i, n := 0, len(args); i < n; i++ { - kv := &args[i] - dst = append(dst, kv.key...) - if i+1 < n { - dst = append(dst, sep...) - } - } - return dst + return h.peekHeaderBytes(HeaderUserAgent) } // b2s converts byte slice to a string without memory allocation. diff --git a/httpx/tokenizer.go b/httpx/tokenizer.go index 667f53c..df9af6e 100644 --- a/httpx/tokenizer.go +++ b/httpx/tokenizer.go @@ -5,17 +5,19 @@ import ( "log/slog" "net/http" "strconv" - "strings" "unsafe" "github.com/soypat/lneto/internal" ) type headerBuf struct { + // buf[:len] holds entire HTTP header data, which may be normalized by [flags]. buf[off:len] holds data not yet processed during parsing. buf []byte - off int // offset into buf for parsing. + // offset into buf for parsing. + off int // args contains key-value store. - args []argsKV + headers []argsKV + cookies []argsKV } type tokint = uint16 @@ -25,38 +27,22 @@ type headerSlice struct { len tokint } -type argSlice struct { - start tokint - len tokint -} - type argsKV struct { key headerSlice value headerSlice // value start >0 means value is present. } +func (kv argsKV) isValid() bool { + return kv.key.start > 0 +} + +func (kv *argsKV) invalidate() { + *kv = argsKV{} +} + func (tb headerBuf) musttoken(slice headerSlice) []byte { return tb.buf[slice.start : slice.start+slice.len] } -func (tb headerBuf) mustargs(slice argSlice) []argsKV { - return tb.args[slice.start : slice.start+slice.len] -} - -func (tb headerBuf) visitArgs(args argSlice, f func(k, v []byte)) { - a := tb.mustargs(args) - for _, arg := range a { - k := tb.musttoken(arg.key) - v := tb.musttoken(arg.value) - f(k, v) - } -} - -func (tb headerBuf) visitArgsKey(args argSlice, f func(k []byte)) { - a := tb.mustargs(args) - for _, arg := range a { - f(tb.musttoken(arg.key)) - } -} func (tb headerBuf) slice(b []byte) headerSlice { base := uintptr(unsafe.Pointer(&tb.buf[0])) @@ -81,69 +67,37 @@ const ( connectionClose noHTTP11 cookiesCollected + flagNoBufferGrow + flagOOMReached ) -func (f flags) hasAll(checkThese flags) bool { - return f&checkThese == checkThese +func (f flags) hasAny(checkThese flags) bool { + return f&checkThese != 0 } type header struct { - buf headerBuf + hbuf headerBuf + logger *slog.Logger contentLength int - h argSlice - cookies argSlice - trailer argSlice - host headerSlice - contentLengthBytes headerSlice - contentType headerSlice - userAgent headerSlice - method headerSlice - proto headerSlice - requestURI headerSlice - rawHeaders headerSlice - mulHeader headerSlice + method headerSlice + requestURI headerSlice + proto headerSlice - flags flags - logger *slog.Logger + flags flags } func (h *header) Set(key, value string) { - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.SetCanonical(b2s(h.bufKV.key), value) + h.SetCanonical(key, value) //TODO: implement non-canonical. } func (h *header) Add(key, value string) { - if h.setSpecialHeader(key, value) { - return - } - k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.h = appendArg(h.h, b2s(k), value, argsHasValue) -} - -// ContentEncoding returns Content-Encoding header value. -func (h *header) ContentEncoding() []byte { - return peekArg(h.h, strContentEncoding) -} - -// SetContentEncoding sets Content-Encoding header value. -func (h *header) SetContentEncoding(contentEncoding string) { - h.Set(strContentEncoding, contentEncoding) -} - -// SetContentType sets Content-Type header value. -func (h *header) SetContentType(contentType string) { - h.contentType = append(h.contentType[:0], contentType...) + h.appendHeader(key, value) } // ContentType returns Content-Type header value. func (h *header) ContentType() []byte { - contentType := h.contentType - if !h.noDefaultContentType && len(h.contentType) == 0 { - contentType = append(contentType, defaultContentType...) - } - return contentType + return h.peekHeaderBytes(HeaderContentType) } // SetCanonical sets the given 'key: value' header assuming that @@ -152,109 +106,36 @@ func (h *header) ContentType() []byte { // If the header is set as a Trailer (forbidden trailers will not be set, see SetTrailer for more details), // it will be sent after the chunked request body. func (h *header) SetCanonical(key, value string) { - if h.setSpecialHeader(key, value) { - return + kv := h.peekPtrHeader(key) + if kv != nil { + kv.invalidate() } - h.setNonSpecial(key, value) -} - -// setSpecialHeader handles special headers and return true when a header is processed. -func (h *header) setSpecialHeader(key, value string) bool { - if len(key) == 0 || h.disableSpecialHeader { - return false - } - h.trace("setSpecialHeader", slog.String("key", key), slog.String("value", value)) - switch key[0] | 0x20 { - case 'c': - switch { - case caseInsensitiveCompare(strContentType, key): - h.SetContentType(value) - return true - case caseInsensitiveCompare(strContentLength, key): - if contentLength, err := parseContentLength(value); err == nil { - h.contentLength = contentLength - h.contentLengthBytes = append(h.contentLengthBytes[:0], value...) - } - return true - case caseInsensitiveCompare(strConnection, key): - if strClose == value { - h.SetConnectionClose() - } else { - h.ResetConnectionClose() - h.setNonSpecial(key, value) - } - return true - case caseInsensitiveCompare(strCookie, key): - h.collectCookies() - h.cookies = parseRequestCookies(h.cookies, value) - return true - } - case 't': // OK - if caseInsensitiveCompare(strTransferEncoding, key) { - // Transfer-Encoding is managed automatically. - return true - } else if caseInsensitiveCompare(strTrailer, key) { - _ = h.SetTrailer(value) - return true - } - case 'h': - if caseInsensitiveCompare(strHost, key) { - h.SetHost(value) - return true - } - case 'u': - if caseInsensitiveCompare(strUserAgent, key) { - h.SetUserAgent(value) - return true - } - } - - return false + h.appendHeader(key, value) } // SetHost sets Host header value. func (h *header) SetHost(host string) { - h.host = append(h.host[:0], host...) + h.Set(HeaderHost, host) } // SetUserAgent sets User-Agent header value. func (h *header) SetUserAgent(userAgent string) { - h.userAgent = append(h.userAgent[:0], userAgent...) + h.Set(HeaderUserAgent, userAgent) } // SetConnectionClose sets 'Connection: close' header. func (h *header) SetConnectionClose() { - h.connectionClose = true + h.flags |= connectionClose } // ResetConnectionClose clears 'Connection: close' header if it exists. func (h *header) ResetConnectionClose() { - if h.connectionClose { - h.connectionClose = false - h.h = delAllArgs(h.h, strConnection) + if h.flags.hasAny(connectionClose) { + h.flags &^= connectionClose + // h.h = delAllArgs(h.h, strConnection) // TODO } } -func (h *header) SetContentRange(startPos, endPos, contentLength int) { - b := h.bufKV.value[:0] - b = append(b, strBytes...) - b = append(b, ' ') - b = appendUint(b, startPos) - b = append(b, '-') - b = appendUint(b, endPos) - b = append(b, '/') - b = appendUint(b, contentLength) - h.bufKV.value = b - - h.setNonSpecial(strContentRange, b2s(h.bufKV.value)) -} - -// setNonSpecial directly put into map i.e. not a basic header. -func (h *header) setNonSpecial(key string, value string) { - h.trace("httpx:setNonSpecial", slog.String("key", key), slog.String("value", value)) - h.h = setArg(h.h, key, value, argsHasValue) -} - func appendUint(b []byte, v int) []byte { if v < 0 { panic("negative uint") @@ -271,144 +152,8 @@ func (h *header) ContentLength() int { return h.contentLength } -// SetContentLength sets Content-Length header value. -// -// Content-Length may be negative: -// -1 means Transfer-Encoding: chunked. -// -2 means Transfer-Encoding: identity. -func (h *header) SetContentLength(contentLength int) { - h.contentLength = contentLength - if contentLength >= 0 { - h.contentLengthBytes = appendUint(h.contentLengthBytes[:0], contentLength) - h.h = delAllArgs(h.h, strTransferEncoding) - } else { - h.contentLengthBytes = h.contentLengthBytes[:0] - h.h = setArg(h.h, strTransferEncoding, strChunked, argsHasValue) - } -} - var ErrBadTrailer = errors.New("contain forbidden trailer") -// SetTrailer sets Trailer header value for chunked request -// to indicate which headers will be sent after the body. -// -// Use Set to set the trailer header later. -// -// Trailers are only supported with chunked transfer. -// Trailers allow the sender to include additional headers at the end of chunked messages. -// -// The following trailers are forbidden: -// 1. necessary for message framing (e.g., Transfer-Encoding and Content-Length), -// 2. routing (e.g., Host), -// 3. request modifiers (e.g., controls and conditionals in Section 5 of [RFC7231]), -// 4. authentication (e.g., see [RFC7235] and [RFC6265]), -// 5. response control data (e.g., see Section 7.1 of [RFC7231]), -// 6. determining how to process the payload (e.g., Content-Encoding, Content-Type, Content-Range, and Trailer) -// -// Return ErrBadTrailer if contain any forbidden trailers. -func (h *header) SetTrailer(trailer string) error { - h.trailer = h.trailer[:0] - return h.AddTrailer(trailer) -} - -// AddTrailerBytes add Trailer header value for chunked response -// to indicate which headers will be sent after the body. -// -// Use Set to set the trailer header later. -// -// Trailers are only supported with chunked transfer. -// Trailers allow the sender to include additional headers at the end of chunked messages. -// -// The following trailers are forbidden: -// 1. necessary for message framing (e.g., Transfer-Encoding and Content-Length), -// 2. routing (e.g., Host), -// 3. request modifiers (e.g., controls and conditionals in Section 5 of [RFC7231]), -// 4. authentication (e.g., see [RFC7235] and [RFC6265]), -// 5. response control data (e.g., see Section 7.1 of [RFC7231]), -// 6. determining how to process the payload (e.g., Content-Encoding, Content-Type, Content-Range, and Trailer) -// -// Return ErrBadTrailer if contain any forbidden trailers. -func (h *header) AddTrailer(trailer string) error { - h.trace("httpx:AddTrailer", slog.String("trailer", trailer)) - var err error - for i := -1; i+1 < len(trailer); { - trailer = trailer[i+1:] - i = strings.IndexByte(trailer, ',') - if i < 0 { - i = len(trailer) - } - key := stripSpace(trailer[:i]) - // Forbidden by RFC 7230, section 4.1.2 - if isBadTrailer(key) { - err = ErrBadTrailer - continue - } - h.bufKV.key = append(h.bufKV.key[:0], key...) - normalizeHeaderKey(h.bufKV.key, h.disableNormalizing) - h.trailer = appendArg(h.trailer, b2s(h.bufKV.key), "", argsNoValue) - } - - return err -} - -func isBadTrailer(key string) bool { - if len(key) == 0 { - return true - } - - switch key[0] | 0x20 { - case 'a': - return caseInsensitiveCompare(key, strAuthorization) - case 'c': - if len(key) > len(HeaderContentType) && caseInsensitiveCompare(key[:8], strContentType[:8]) { - // skip compare prefix 'Content-' - return caseInsensitiveCompare(key[8:], strContentEncoding[8:]) || - caseInsensitiveCompare(key[8:], strContentLength[8:]) || - caseInsensitiveCompare(key[8:], strContentType[8:]) || - caseInsensitiveCompare(key[8:], strContentRange[8:]) - } - return caseInsensitiveCompare(key, strConnection) - case 'e': - return caseInsensitiveCompare(key, strExpect) - case 'h': - return caseInsensitiveCompare(key, strHost) - case 'k': - return caseInsensitiveCompare(key, strKeepAlive) - case 'm': - return caseInsensitiveCompare(key, strMaxForwards) - case 'p': - if len(key) > len(HeaderProxyConnection) && caseInsensitiveCompare(key[:6], strProxyConnection[:6]) { - // skip compare prefix 'Proxy-' - return caseInsensitiveCompare(key[6:], strProxyConnection[6:]) || - caseInsensitiveCompare(key[6:], strProxyAuthenticate[6:]) || - caseInsensitiveCompare(key[6:], strProxyAuthorization[6:]) - } - case 'r': - return caseInsensitiveCompare(key, strRange) - case 't': - return caseInsensitiveCompare(key, strTE) || - caseInsensitiveCompare(key, strTrailer) || - caseInsensitiveCompare(key, strTransferEncoding) - case 'w': - return caseInsensitiveCompare(key, strWWWAuthenticate) - } - return false -} - -// RawHeaders returns raw header key/value bytes. -// -// Depending on server configuration, header keys may be normalized to -// capital-case in place. -// -// This copy is set aside during parsing, so empty slice is returned for all -// cases where parsing did not happen. Similarly, request line is not stored -// during parsing and can not be returned. -// -// The slice is not safe to use after the handler returns. -func (h *header) RawHeaders() []byte { - return h.rawHeaders -} - // DisableNormalizing disables header names' normalization. // // By default all the header names are normalized by uppercasing @@ -422,90 +167,72 @@ func (h *header) RawHeaders() []byte { // // Disable header names' normalization only if know what are you doing. func (h *header) DisableNormalizing() { - h.disableNormalizing = true -} - -// DisableSpecialHeader disables special header processing. -// fasthttp will not set any special headers for you, such as Host, Content-Type, User-Agent, etc. -// You must set everything yourself. -// If RequestHeader.Read() is called, special headers will be ignored. -// This can be used to control case and order of special headers. -// This is generally not recommended. -func (h *header) DisableSpecialHeader() { - h.disableSpecialHeader = true + h.flags |= disableNormalizing } // Method returns HTTP request method. func (h *header) Method() []byte { - if len(h.method) == 0 { - h.method = append(h.method, http.MethodGet...) - } - return h.method + return h.hbuf.musttoken(h.method) } func (h *header) SetMethod(method string) { - h.method = append(h.method[:0], method...) + h.method = h.reuseOrAppend(h.method, method) } // SetRequestURI sets RequestURI for the first HTTP request line. func (h *header) SetRequestURI(requestURI string) { - h.requestURI = append(h.requestURI[:0], requestURI...) + h.requestURI = h.reuseOrAppend(h.requestURI, requestURI) } // RequestURI returns RequestURI from the first HTTP request line. func (h *header) RequestURI() []byte { - requestURI := h.requestURI - if len(requestURI) == 0 { - requestURI = append(requestURI, '/') + if h.requestURI.start == 0 { + return nil + } else if h.requestURI.len == 0 { + h.requestURI = h.appendSlice("/") } - return requestURI + return h.hbuf.musttoken(h.requestURI) } // Protocol returns HTTP protocol. func (h *header) Protocol() []byte { - if len(h.proto) == 0 { - h.proto = append(h.proto, strHTTP11...) + if h.proto.len == 0 { + h.proto = h.appendSlice(strHTTP11) } - return h.proto + return h.hbuf.musttoken(h.proto) } func (h *header) SetProtocol(protocol string) { - h.proto = append(h.proto[:0], protocol...) + h.proto = h.reuseOrAppend(h.proto, protocol) } // AppendReqRespCommon appends request/response common header representation to dst and returns the extended buffer. func (h *header) AppendReqRespCommon(dst []byte) []byte { - for i, n := 0, len(h.h); i < n; i++ { - kv := &h.h[i] - // Exclude trailer from header - exclude := false - for _, t := range h.trailer { - if b2s(kv.key) == b2s(t.key) { - exclude = true - break - } - } - if !exclude { - dst = appendHeaderLine(dst, b2s(kv.key), b2s(kv.value)) + for i, n := 0, len(h.hbuf.headers); i < n; i++ { + kv := &h.hbuf.headers[i] + if kv.isValid() { + key := h.hbuf.musttoken(kv.key) + value := h.hbuf.musttoken(kv.value) + dst = appendHeaderLine(dst, b2s(key), b2s(value)) } } - if len(h.trailer) > 0 { - aux := appendArgsKey(nil, h.trailer, strCommaSpace) - dst = appendHeaderLine(dst, strTrailer, b2s(aux)) - } + // if len(h.trailer) > 0 { + // aux := appendArgsKey(nil, h.trailer, strCommaSpace) + // dst = appendHeaderLine(dst, strTrailer, b2s(aux)) + // } // there is no need in h.collectCookies() here, since if cookies aren't collected yet, // they all are located in h.h. - n := len(h.cookies) - if n > 0 && !h.disableSpecialHeader { + n := len(h.hbuf.cookies) + if n > 0 && !h.flags.hasAny(disableSpecialHeader) { dst = append(dst, strCookie...) dst = append(dst, strColonSpace...) - dst = appendRequestCookieBytes(dst, h.cookies) + h.hbuf.appendRequestCookieBytes(dst) dst = append(dst, strCRLF...) } - if h.ConnectionClose() && !h.disableSpecialHeader { + if h.ConnectionClose() && !h.flags.hasAny(disableSpecialHeader) { dst = appendHeaderLine(dst, strConnection, strClose) } @@ -524,31 +251,37 @@ func (h *header) ignoreBody() bool { } func (h *header) collectCookies() { - if h.cookiesCollected { + if h.flags.hasAny(cookiesCollected) { return } - - for i, n := 0, len(h.h); i < n; i++ { - kv := &h.h[i] - if caseInsensitiveCompare(b2s(kv.key), strCookie) { - h.cookies = parseRequestCookies(h.cookies, b2s(kv.value)) - tmp := *kv - copy(h.h[i:], h.h[i+1:]) - n-- - i-- - h.h[n] = tmp - h.h = h.h[:n] + n := len(h.hbuf.headers) + for i := 0; i < n; i++ { + kv := h.hbuf.headers[i] + if kv.isValid() && caseInsensitiveCompare(b2s(h.hbuf.musttoken(kv.key)), HeaderCookie) { + cookie := h.hbuf.musttoken(kv.value) + for len(cookie) > 0 { + key, value, n := parseCookie(cookie) + h.hbuf.cookies = append(h.hbuf.cookies, argsKV{ + key: h.hbuf.slice(key), + value: h.hbuf.slice(value), + }) + cookie = cookie[n:] + } } } - h.cookiesCollected = true + h.flags |= cookiesCollected +} + +func (h *header) parseReqCookie(value []byte) { + } func (h *header) MethodIs(method string) bool { - return b2s(h.method) == method + return b2s(h.Method()) == method } // IsGet returns true if request method is GET. -func (h *header) IsGet() bool { return len(h.method) == 0 || h.MethodIs(http.MethodGet) } +func (h *header) IsGet() bool { return h.method.len == 0 || h.MethodIs(http.MethodGet) } // IsHead returns true if request method is HEAD. func (h *header) IsHead() bool { return h.MethodIs(http.MethodHead) } @@ -575,7 +308,7 @@ func (h *header) IsTrace() bool { return h.MethodIs(http.MethodTrace) } func (h *header) IsPatch() bool { return h.MethodIs(http.MethodPatch) } // IsHTTP11 returns true if the request is HTTP/1.1. -func (h *header) IsHTTP11() bool { return !h.noHTTP11 } +func (h *header) IsHTTP11() bool { return !h.flags.hasAny(noHTTP11) } // Embed this type into a struct, which mustn't be copied, // so `go vet` gives a warning if this struct is copied. @@ -596,27 +329,3 @@ func (h *header) debug(msg string, attrs ...slog.Attr) { func (h *header) info(msg string, attrs ...slog.Attr) { internal.LogAttrs(h.logger, slog.LevelInfo, msg, attrs...) } - -func normalizeHeaderKey(b []byte, disableNormalizing bool) { - if disableNormalizing { - return - } - - n := len(b) - if n == 0 { - return - } - - b[0] = toUpperTable[b[0]] - for i := 1; i < n; i++ { - p := &b[i] - if *p == '-' { - i++ - if i < n { - b[i] = toUpperTable[b[i]] - } - continue - } - *p = toLowerTable[*p] - } -} From b63ce5a964fbcbb9b145c7e9134cf141967e639e Mon Sep 17 00:00:00 2001 From: soypat Date: Sun, 25 May 2025 02:01:53 -0300 Subject: [PATCH 3/8] tests passing babyyyyy, header is 112 bytes of pure ecstasy --- httpx/header_parse.go | 130 +++++++++++++++--------------------------- httpx/header_test.go | 33 +++++++++++ httpx/tokenizer.go | 10 +++- 3 files changed, 87 insertions(+), 86 deletions(-) create mode 100644 httpx/header_test.go diff --git a/httpx/header_parse.go b/httpx/header_parse.go index 5045244..16c6875 100644 --- a/httpx/header_parse.go +++ b/httpx/header_parse.go @@ -88,6 +88,9 @@ func (hb *headerBuf) parseFirstLine(initFlags flags) (method, uri, proto headerS methodEnd := max(0, bytes.IndexByte(b, ' ')) reqURIEnd := bytes.IndexByte(b[methodEnd+1:], ' ') + if reqURIEnd >= 0 { + reqURIEnd += methodEnd + 1 + } switch { case reqURIEnd < 0: flags |= noHTTP11 @@ -106,11 +109,7 @@ func (hb *headerBuf) parseFirstLine(initFlags flags) (method, uri, proto headerS } type scannerState struct { - err error - - // hLen stores header subslice len - hLen int - + err error disableNormalizing bool // by checking whether the next line contains a colon or not to tell @@ -128,7 +127,7 @@ func (h *header) parseHeaders(ss *scannerState) (err error) { hb := &h.hbuf h.contentLength = -2 - for kv := hb.nextKV(ss); kv.isValid(); kv = hb.nextKV(ss) { + for kv := hb.nextKV2(ss); kv.isValid(); kv = hb.nextKV2(ss) { if h.flags.hasAny(disableSpecialHeader) { h.hbuf.headers = append(h.hbuf.headers, kv) continue @@ -271,49 +270,46 @@ func readRawHeaders(dst []byte, buf string) ([]byte, int, error) { } func (hb *headerBuf) noKV() argsKV { return argsKV{} } -func (hb *headerBuf) nextKV(ss *scannerState) argsKV { +func (hb *headerBuf) nextKV2(ss *scannerState) argsKV { if !ss.initialized { ss.nextColon = -1 ss.nextNewLine = -1 - ss.initialized = true } buf := hb.buf[hb.off:] - bLen := len(buf) - if bLen >= 2 && buf[0] == rChar && buf[1] == nChar { + blen := len(buf) + if blen >= 2 && buf[0] == '\r' && buf[1] == '\n' { hb.off += 2 return hb.noKV() // \r\n\r\n Ends header. - } - if bLen >= 1 && buf[0] == nChar { - hb.off++ - return hb.noKV() // \n\n: Ends header. + } else if blen >= 1 && buf[0] == '\n' { + hb.off += 1 + return hb.noKV() // \n\n Ends header. } - var n int + // n is parsing offset. Will start by storing colon index. + n := 0 if ss.nextColon >= 0 { + // Retake from last colon found. n = ss.nextColon ss.nextColon = -1 } else { n = bytes.IndexByte(buf, ':') - - // There can't be a \n inside the header name, check for this. - x := bytes.IndexByte(buf, nChar) + x := bytes.IndexByte(buf, '\n') if x < 0 { // A header name should always at some point be followed by a \n // even if it's the one that terminates the header block. ss.err = errNeedMore return hb.noKV() - } - if x < n { - // There was a \n before the : + } else if x < n { + // There was a \n before the colon! This is invalid. ss.err = errInvalidName return hb.noKV() + } else if n < 0 { + // No colon found, probably missing data. + ss.err = errNeedMore + return hb.noKV() } } - if n < 0 { - ss.err = errNeedMore - return hb.noKV() - } - + // n stores colon position by now. if bytes.IndexByte(buf[:n], ' ') >= 0 || bytes.IndexByte(buf[:n], '\t') >= 0 { // Spaces between the header key and colon are not allowed. // See RFC 7230, Section 3.2.4. @@ -321,72 +317,38 @@ func (hb *headerBuf) nextKV(ss *scannerState) argsKV { return hb.noKV() } + // Ready to store key.. var resultKV argsKV resultKV.key = hb.slice(buf[:n]) normalizeHeaderKey(buf[:n], ss.disableNormalizing) - n++ + n++ // consume colon. for len(buf) > n && buf[n] == ' ' { - n++ - // the newline index is a relative index, and lines below trimmed `s.b` by `n`, - // so the relative newline index also shifted forward. it's safe to decrease - // to a minus value, it means it's invalid, and will find the newline again. - ss.nextNewLine-- + n++ // Trim leading spaces. } - ss.hLen += n - buf = buf[n:] - if ss.nextNewLine >= 0 { - n = ss.nextNewLine - ss.nextNewLine = -1 - } else { - n = bytes.IndexByte(buf, nChar) - } - if n < 0 { - ss.err = errNeedMore - return hb.noKV() - } - isMultiLineValue := false - for { - if n+1 >= len(buf) { - break - } - if buf[n+1] != ' ' && buf[n+1] != '\t' { - break - } - d := bytes.IndexByte(buf[n+1:], nChar) - if d <= 0 { - break - } else if d == 1 && buf[n+1] == rChar { - break - } - e := n + d + 1 - if c := bytes.IndexByte(buf[n+1:e], ':'); c >= 0 { - ss.nextColon = c - ss.nextNewLine = d - c - 1 - break - } - isMultiLineValue = true - n = e - } - if n >= len(buf) { - ss.err = errNeedMore - return hb.noKV() - } - oldB := buf - value := buf[:n] - ss.hLen += n + 1 - buf = buf[n+1:] + // n now points to start of value. + valueStart := n - if n > 0 && value[n-1] == rChar { - n-- + // Find end of value. Values may be multiline, in which case we must treat newlines followed by whitespace as part of the value. + for { + nl := bytes.IndexByte(buf[n:], '\n') + if nl < 0 || nl+n+1 == len(buf) { + // No newline or newline is last character and can't know if is multiline. + ss.err = errNeedMore + return hb.noKV() + } + n += nl + 1 // Index of the newly found newline. + nextChar := buf[n] + if nextChar != ' ' && nextChar != '\t' { + break // End of value found. + } } - for n > 0 && value[n-1] == ' ' { - n-- + + valueEnd := n - 1 // Trim newline. + if valueEnd > valueStart && buf[valueEnd-1] == '\r' { + valueEnd-- // Trim \r character if present before value. } - value = value[:n] - if isMultiLineValue { - value, buf, ss.hLen = normalizeHeaderValue(value, oldB, ss.hLen) - } - resultKV.value = hb.slice(value) + resultKV.value = hb.slice(buf[valueStart:valueEnd]) + hb.off += n return resultKV } diff --git a/httpx/header_test.go b/httpx/header_test.go new file mode 100644 index 0000000..e16c310 --- /dev/null +++ b/httpx/header_test.go @@ -0,0 +1,33 @@ +package httpx + +import ( + "bytes" + "net/http" + "strings" + "testing" +) + +func TestHeaderParseRequest(t *testing.T) { + const ( + wantMethod = "GET" + wantURI = "/" + wantMessage = "hello world!" + ) + req, err := http.NewRequest(wantMethod, wantURI, strings.NewReader(wantMessage)) + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + req.Write(&buf) + var hdr header + err = hdr.ParseBytes(buf.Bytes()) + if err != nil { + t.Fatal(err) + } + if !hdr.MethodIs(wantMethod) { + t.Errorf("want method %s, got %q", wantMethod, hdr.Method()) + } + if !bytes.Equal(hdr.RequestURI(), []byte(wantURI)) { + t.Errorf("want request URI %q, got %q", wantURI, hdr.RequestURI()) + } +} diff --git a/httpx/tokenizer.go b/httpx/tokenizer.go index df9af6e..0164a10 100644 --- a/httpx/tokenizer.go +++ b/httpx/tokenizer.go @@ -45,8 +45,8 @@ func (tb headerBuf) musttoken(slice headerSlice) []byte { } func (tb headerBuf) slice(b []byte) headerSlice { - base := uintptr(unsafe.Pointer(&tb.buf[0])) - off := uintptr(unsafe.Pointer(&b[0])) + base := uintptr(unsafe.Pointer(unsafe.SliceData(tb.buf))) + off := uintptr(unsafe.Pointer(unsafe.SliceData(b))) if off < base || off > base+uintptr(len(tb.buf)) { panic("httpx: argument buffer does not alias header buffer") } @@ -87,6 +87,12 @@ type header struct { flags flags } +func (h *header) ParseBytes(b []byte) error { + h.resetSkipNormalize() + h.hbuf.readFromBytes(b) + return h.parse() +} + func (h *header) Set(key, value string) { h.SetCanonical(key, value) //TODO: implement non-canonical. } From ca7bf2f36676e85d2520bd11ae90513c97acc2db Mon Sep 17 00:00:00 2001 From: soypat Date: Sun, 25 May 2025 16:58:52 -0300 Subject: [PATCH 4/8] httpraw looking good --- http/httpraw/cookie.go | 165 +++++++++ http/httpraw/header.go | 358 ++++++++++++++++++ http/httpraw/header_test.go | 146 ++++++++ http/httpraw/parse.go | 411 +++++++++++++++++++++ httpx/cookie.go | 700 ------------------------------------ httpx/definitions.go | 270 -------------- httpx/header_parse.go | 663 ---------------------------------- httpx/header_test.go | 33 -- httpx/tokenizer.go | 337 ----------------- 9 files changed, 1080 insertions(+), 2003 deletions(-) create mode 100644 http/httpraw/cookie.go create mode 100644 http/httpraw/header.go create mode 100644 http/httpraw/header_test.go create mode 100644 http/httpraw/parse.go delete mode 100644 httpx/cookie.go delete mode 100644 httpx/definitions.go delete mode 100644 httpx/header_parse.go delete mode 100644 httpx/header_test.go delete mode 100644 httpx/tokenizer.go diff --git a/http/httpraw/cookie.go b/http/httpraw/cookie.go new file mode 100644 index 0000000..091d3b0 --- /dev/null +++ b/http/httpraw/cookie.go @@ -0,0 +1,165 @@ +package httpraw + +import ( + "bytes" + "errors" +) + +// Cookie implements cookie key-value parsing. Methods function similarly to eponymous [Header] methods. +type Cookie struct { + buf []byte + kvs []argsKV // first key-value pair is the data Key/Value pair. +} + +// Reset functions very similarly to [Header.Reset]. Can be used for in-place cookie parsing. +func (c *Cookie) Reset(buf []byte) { + if buf == nil { + buf = c.buf[:0] + } + *c = Cookie{ + buf: buf, + kvs: c.kvs[:0], + } +} + +func (c *Cookie) Key() []byte { + if len(c.kvs) == 0 || c.kvs[0].key.len == 0 { + return nil + } + return tok2bytes(c.buf, c.kvs[0].key) +} + +func (c *Cookie) Value() []byte { + if len(c.kvs) == 0 || c.kvs[0].value.len == 0 { + return nil + } + return tok2bytes(c.buf, c.kvs[0].value) +} + +func (c *Cookie) ParseBytes(cookie []byte) error { + c.Reset(nil) + c.buf = append(c.buf[:0], cookie...) + return c.Parse() +} + +func (c *Cookie) CopyTo(dst *Cookie) { + dst.buf = append(dst.buf[:0], c.buf...) + dst.kvs = append(dst.kvs[:0], c.kvs...) +} + +func (c *Cookie) Parse() error { + if len(c.kvs) > 0 { + return errors.New("cookies already parsed, reset before parsing again") + } + off := 0 + for { + k, v, n := parseCookie(c.buf[off:]) + if n == 0 { + break + } + c.kvs = append(c.kvs, argsKV{ + key: bytes2tok(c.buf, k), + value: bytes2tok(c.buf, v), + }) + off += n + } + return nil +} + +func (c *Cookie) ForEach(cb func(key, value []byte) error) error { + nc := len(c.kvs) + for i := 0; i < nc; i++ { + kv := c.kvs[i] + key := tok2bytes(c.buf, kv.key) + value := tok2bytes(c.buf, kv.value) + err := cb(key, value) + if err != nil { + return err + } + } + return nil +} + +func (c *Cookie) Get(key string) []byte { + nc := len(c.kvs) + for i := 0; i < nc; i++ { + kv := c.kvs[i] + if b2s(tok2bytes(c.buf, kv.key)) == key { + return tok2bytes(c.buf, kv.value) + } + } + return nil +} + +func (c *Cookie) HasValueOrKey(keyOrSingleValue string) bool { + nc := len(c.kvs) + for i := 0; i < nc; i++ { + kv := c.kvs[i] + if kv.key.len == 0 && b2s(tok2bytes(c.buf, kv.value)) == keyOrSingleValue || + b2s(tok2bytes(c.buf, kv.key)) == keyOrSingleValue { + return true + } + } + return false +} + +// parseCookie parses a cookie inside cookie buffer and adds it to cookie buffer.. +// +// Cookie: \r\n +func parseCookie(cookie []byte) (key, value []byte, cookieEnd int) { + if len(cookie) == 0 { + return nil, nil, 0 + } + valueEnd := bytes.IndexByte(cookie, ';') + if valueEnd < 0 { // Ouch this `if` looks like it kills CPU pipepline. + valueEnd = len(cookie) + cookieEnd = len(cookie) + } else { + cookieEnd = valueEnd + 1 + } + eqIdx := bytes.IndexByte(cookie[:valueEnd], '=') + key = cookie[:0] + if eqIdx > 0 { + key = trimCookie(cookie[:eqIdx], false) + } + value = trimCookie(cookie[eqIdx+1:valueEnd], true) + return key, value, cookieEnd +} + +func trimCookie(src []byte, trimQuotes bool) []byte { + for len(src) > 0 && src[0] == ' ' { + src = src[1:] // skip leading whitespace. + } + for len(src) > 0 && src[len(src)-1] == ' ' { + src = src[:len(src)-1] // skip trailing whitespace + } + if trimQuotes { + if len(src) > 1 && src[0] == '"' && src[len(src)-1] == '"' { + src = src[1 : len(src)-1] // Trim leading+trailing quotes. + } + } + return src +} + +func (c *Cookie) String() string { + buf := c.AppendKeyValues(nil) + return b2s(buf) +} + +func (c *Cookie) AppendKeyValues(dst []byte) []byte { + nc := len(c.kvs) + for i := 0; i < nc; i++ { + kv := c.kvs[i] + key := tok2bytes(c.buf, kv.key) + value := tok2bytes(c.buf, kv.value) + if len(key) != 0 { + dst = append(dst, key...) + dst = append(dst, '=') + } + dst = append(dst, value...) + if i+1 < nc { + dst = append(dst, ';', ' ') + } + } + return dst +} diff --git a/http/httpraw/header.go b/http/httpraw/header.go new file mode 100644 index 0000000..a174c19 --- /dev/null +++ b/http/httpraw/header.go @@ -0,0 +1,358 @@ +package httpraw + +import ( + "errors" + "io" + "net/http" + "slices" +) + +const ( + strHTTP11 = "HTTP/1.1" + strCRLF = "\r\n" + headerCookie = "Cookie" + headerConnection = "Connection" + strClose = "close" +) + +type flags uint16 + +const ( + flagNoBufferGrow flags = 1 << iota + flagDoneParsingHeader + flagOOMReached + flagConnClose + flagNoHTTP11 + flagMangledBuffer // set when header fields appended to buffer via Add,Set calls +) + +func (f flags) hasAny(checkThese flags) bool { + return f&checkThese != 0 +} + +// Header implements "raw" HTTP validation and header key-value parsing, validation and marshalling. +// +// It does NOT implement: +// - Normalization. +// - Cookies. +// - Special header optimizations. +// - Safe API. Users can easily mangle HTTP body with calls. +type Header struct { + hbuf headerBuf + + // Request fields. + method headerSlice + requestURI headerSlice + proto headerSlice + + // Response fields. + statusCode headerSlice + statusText headerSlice + + flags flags + _ noCopy +} + +// EnableBufferGrow disables buffer growth during parsing if b is false. Is enabled by default. +func (h *Header) EnableBufferGrow(b bool) { + if !b { + h.flags |= flagNoBufferGrow + } else { + h.flags &^= flagNoBufferGrow + } +} + +// ParseBytes copies the bytes into buffer and parses the HTTP header. It fails if HTTP header data is incomplete. +func (h *Header) ParseBytes(asResponse bool, b []byte) error { + h.Reset(nil) + h.hbuf.readFromBytes(b) + return h.parse(asResponse) +} + +// Parse parses accumulated data in-place with no copying. One can set HTTP header data buffer with [Header.Reset]. +// It fails if HTTP data is incomplete. +func (h *Header) Parse(asResponse bool) error { + h.Reset(h.hbuf.buf) + return h.parse(asResponse) +} + +// TryParse begins parsing or resumes parsing from a failed previous attempt from any of the Parse* methods. +// It fails if HTTP data is incomplete. It panics if called after header parsing completed succesfully. +// As long as ok returns true future calls to TryParse may succeed. +// +// ok, err := h.TryParse() +// for ; ok; ok, err = h.TryParse() { +// _, err = h.ReadFrom(r, 256) +// if err != nil && err != io.EOF { +// return err +// } +// } +// if err != nil { +// return err +// } +func (h *Header) TryParse(asResponse bool) (ok bool, err error) { + if h.flags.hasAny(flagDoneParsingHeader) { + return false, errors.New("TryParse called after header parsed") + } else if h.flags.hasAny(flagMangledBuffer) { + return false, errMangledBuffer + } + if asResponse && h.statusCode.len == 0 || !asResponse && h.requestURI.start == 0 { + err = h.parseFirstLine(asResponse) + if err != nil { + return err == errNeedMore, err + } + } + err = h.parseNextHeaders() + return err == nil || err == errNeedMore, err +} + +// ReadFromLimited reads at most maxBytesToRead from reader and appends them to underlying buffer. +// Used to accumulate HTTP header for later parsing with [Header.TryParse]. +func (h *Header) ReadFromLimited(r io.Reader, maxBytesToRead int) (int, error) { + if maxBytesToRead <= 0 { + return 0, errSmallBuffer + } else if h.flags.hasAny(flagMangledBuffer) { + return 0, errMangledBuffer + } + free := h.Free() + if free < maxBytesToRead { + if h.flags.hasAny(flagNoBufferGrow) { + return 0, errSmallBuffer + } + h.hbuf.buf = slices.Grow(h.hbuf.buf, maxBytesToRead) + } + blen := len(h.hbuf.buf) + b := h.hbuf.buf[blen:min(blen+maxBytesToRead, cap(h.hbuf.buf))] + n, err := r.Read(b) + h.hbuf.buf = h.hbuf.buf[:blen+n] + return n, err +} + +// ReadFromBytes appends argument buffer to underlying buffer. +// Used to accumulate HTTP header for later parsing with [Header.TryParse]. +func (h *Header) ReadFromBytes(b []byte) (int, error) { + if len(b) == 0 { + return 0, errSmallBuffer + } + free := h.Free() + if free < len(b) { + if h.flags.hasAny(flagNoBufferGrow) { + return 0, errSmallBuffer + } + h.hbuf.buf = slices.Grow(h.hbuf.buf, len(b)) + } + h.hbuf.readFromBytes(b) + return len(b), nil +} + +// Free returns amount of bytes free in underlying buffer. +func (h *Header) Free() int { + return h.hbuf.free() +} + +// ForEach iterates over header key-value field tuples. +func (h *Header) ForEach(cb func(key, value []byte) error) error { + return h.hbuf.forEach(cb) +} + +func (hb *headerBuf) forEach(cb func(key, value []byte) error) error { + nh := len(hb.headers) + for i := 0; i < nh; i++ { + kv := hb.headers[i] + if !kv.isValid() { + continue + } + key := hb.musttoken(kv.key) + value := hb.musttoken(kv.value) + err := cb(key, value) + if err != nil { + return err + } + } + return nil +} + +// Reset discards all parsed data and sets the buffer data to buf. This method +// can be used to avoid copying and growing buffers. Call [Header.Parse] after setting buffer +// data with Reset to parse data in-place. +// If buf is nil then the current buffer is reused. There are 3 ways to use Reset: +// +// h.Reset(prealloc[:0]); h.ParseBytes(httpHeader) // Tell header to use a pre-allocated buffer capacity. +// h.Reset(httpHeader); h.Parse() // Parse bytes in place with no copying. +// h.Reset(nil) // Reuse buffer previously set in a call to Reset. +func (h *Header) Reset(buf []byte) { + if h.flags.hasAny(flagNoBufferGrow) && len(buf) < 32 { + panic("small buffer and flagNoBufferGrow set") + } + const persistentFlags = flagNoBufferGrow + h.hbuf.reset(buf) + *h = Header{ + hbuf: h.hbuf, + flags: h.flags & persistentFlags, + } +} + +// Body returns the surplus data following headers. It is only valid as long as Parse* or Reset methods are not called. +func (h *Header) Body() ([]byte, error) { + if h.flags.hasAny(flagMangledBuffer) { + return nil, errMangledBuffer + } else if h.flags.hasAny(flagDoneParsingHeader) { + return h.hbuf.buf[h.hbuf.off:], nil + } + return nil, errUnparsed +} + +// Set sets a key-value pair in the HTTP header. It mangles the buffer. +func (h *Header) Set(key, value string) { + kv := h.peekPtrHeader(key) + if kv != nil { + kv.invalidate() + } + h.appendHeader(key, value) +} + +func (h *Header) Get(key string) []byte { + kv := h.peekHeader(key) + if kv.isValid() { + return h.hbuf.musttoken(kv.value) + } + return nil +} + +func (h *Header) Add(key, value string) { + h.appendHeader(key, value) +} + +// Method returns HTTP request method. +func (h *Header) Method() []byte { + return h.getNonEmptyValue(h.method) +} + +// SetRequestURI sets RequestURI for the first HTTP request line. +func (h *Header) SetRequestURI(requestURI string) { + h.requestURI = h.reuseOrAppend(h.requestURI, requestURI) +} + +// RequestURI returns RequestURI from the first HTTP request line. +func (h *Header) RequestURI() []byte { + return h.getNonEmptyValue(h.requestURI) +} + +func (h *Header) SetMethod(method string) { + h.method = h.reuseOrAppend(h.method, method) +} + +// Protocol returns HTTP protocol. +func (h *Header) Protocol() []byte { + return h.getNonEmptyValue(h.proto) +} + +func (h *Header) SetProtocol(protocol string) { + h.proto = h.reuseOrAppend(h.proto, protocol) +} + +func (h *Header) Status() (code, statusText []byte) { + if h.statusCode.len == 0 { + return nil, nil + } + return h.hbuf.musttoken(h.statusCode), h.hbuf.musttoken(h.statusText) +} + +func (h *Header) SetStatus(code, statusText string) { + h.statusCode = h.reuseOrAppend(h.statusCode, code) + h.statusText = h.reuseOrAppend(h.statusText, statusText) +} + +func (h *Header) getNonEmptyValue(s headerSlice) []byte { + if s.len == 0 { + return nil // If empty then value is invalid, return nil. + } + return h.hbuf.musttoken(s) +} + +// AppendRequest appends the request representation to the buffer and returns the result. +func (h *Header) AppendRequest(dst []byte) ([]byte, error) { + if h.flags.hasAny(flagOOMReached) { + return dst, errOOM + } else if h.requestURI.len == 0 || h.proto.len == 0 || h.method.len == 0 { + return dst, errors.New("need method/protocol/request URI to create request header") + } + method := h.Method() + if len(method) == 0 { + dst = append(dst, http.MethodGet...) + } else { + dst = append(dst, method...) + } + uri := h.RequestURI() + proto := h.Protocol() + + dst = append(dst, ' ') + dst = append(dst, uri...) + dst = append(dst, ' ') + dst = append(dst, proto...) + dst = append(dst, strCRLF...) + + dst = h.AppendHeaders(dst) + + return append(dst, strCRLF...), nil +} + +// AppendResponse appends the response representation to the buffer and returns the result. +func (h *Header) AppendResponse(dst []byte) ([]byte, error) { + if h.flags.hasAny(flagOOMReached) { + return dst, errOOM + } else if h.statusCode.len == 0 || h.statusText.len == 0 { + return dst, errors.New("invalid status code or text") + } + code, text := h.Status() + dst = append(dst, code...) + dst = append(dst, ' ') + dst = append(dst, text...) + dst = append(dst, strCRLF...) + + dst = h.AppendHeaders(dst) + + return append(dst, strCRLF...), nil +} + +// AppendHeaders appends headers to buffer. Use AppendRequest and AppendResponse over this. +// Does not append extra \r\n to end. Appends nothing if contains no headers. +func (h *Header) AppendHeaders(dst []byte) []byte { + for i, n := 0, len(h.hbuf.headers); i < n; i++ { + kv := &h.hbuf.headers[i] + if kv.isValid() { + key := h.hbuf.musttoken(kv.key) + value := h.hbuf.musttoken(kv.value) + dst = appendHeaderLine(dst, b2s(key), b2s(value)) + } + } + return dst +} + +func (h *Header) String() string { + buf, err := h.AppendRequest(nil) + if err != nil { + buf, err = h.AppendResponse(nil) + if err != nil { + return err.Error() + } + } + return b2s(buf) +} + +func appendHeaderLine(dst []byte, key, value string) []byte { + dst = append(dst, key...) + dst = append(dst, ':', ' ') + dst = append(dst, value...) + return append(dst, strCRLF...) +} + +// Embed this type into a struct, which mustn't be copied, +// so `go vet` gives a warning if this struct is copied. +// +// See https://github.com/golang/go/issues/8005#issuecomment-190753527 for details. +// and also: https://stackoverflow.com/questions/52494458/nocopy-minimal-example +type noCopy struct{} + +func (*noCopy) Lock() {} +func (*noCopy) Unlock() {} diff --git a/http/httpraw/header_test.go b/http/httpraw/header_test.go new file mode 100644 index 0000000..c00ba22 --- /dev/null +++ b/http/httpraw/header_test.go @@ -0,0 +1,146 @@ +package httpraw + +import ( + "bytes" + "fmt" + "net/http" + "strconv" + "strings" + "testing" + "time" +) + +func TestHeaderParseRequest(t *testing.T) { + const ( + wantMethod = "GET" + wantURI = "/data/set" + wantMessage = "hello world!" + asRequest = false + asResponse = true + ) + req, err := http.NewRequest(wantMethod, wantURI, strings.NewReader(wantMessage)) + if err != nil { + t.Fatal(err) + } + var wantCookie http.Cookie + wantCookie.SameSite = http.SameSiteLaxMode + wantCookie.MaxAge = 360000 + wantCookie.Name = "key" + wantCookie.Value = "value" + wantCookie.Expires = time.Now().Add(time.Hour) + wantCookie.Domain = "DOM" + wantCookie.HttpOnly = true + wantCookie.Secure = true + wantCookie.Path = "/abc" + req.Header.Set("Cookie", wantCookie.String()) + t.Log("valid cookie:", wantCookie.Valid() == nil, wantCookie.String()) + + var buf bytes.Buffer + req.Write(&buf) + var hdr Header + msg := buf.Bytes() + + start := time.Now() + err = hdr.ParseBytes(asRequest, msg) + elapsed := time.Since(start) + if err != nil { + t.Fatal(err) + } + fmt.Printf("%s\nparsed in %s\n\n", msg, elapsed.String()) + if string(hdr.Method()) != wantMethod { + t.Errorf("want method %s, got %q", wantMethod, hdr.Method()) + } + if !bytes.Equal(hdr.RequestURI(), []byte(wantURI)) { + t.Errorf("want request URI %q, got %q", wantURI, hdr.RequestURI()) + } + contentLength, _ := strconv.Atoi(string(hdr.Get("Content-Length"))) + if contentLength != len(wantMessage) { + t.Errorf("want Content-Length %d, got %d", len(wantMessage), contentLength) + } + var c Cookie + cookie := hdr.Get("Cookie") + c.Reset(cookie) + err = c.Parse() + if err != nil { + t.Error(err) + } + key := string(c.Key()) + if key != wantCookie.Name { + t.Errorf("want cookie key %q, got %q", wantCookie.Name, key) + } + value := string(c.Value()) + if value != wantCookie.Value { + t.Errorf("want cookie key %q, got %q", wantCookie.Value, value) + } + domain := string(c.Get("Domain")) + if domain != wantCookie.Domain { + t.Errorf("want domain %q, got %q", wantCookie.Domain, domain) + } + httpOnly := c.HasValueOrKey("HttpOnly") + if httpOnly != wantCookie.HttpOnly { + t.Errorf("want cookie HttpOnly %v, got %v", wantCookie.HttpOnly, httpOnly) + } + secure := c.HasValueOrKey("Secure") + if secure != wantCookie.Secure { + t.Errorf("want cookie HttpOnly %v, got %v", wantCookie.Secure, secure) + } + samesite := string(c.Get("SameSite")) + if samesite != strSameSite(wantCookie.SameSite) { + t.Errorf("want cookie SameSite %v, got %v", strSameSite(wantCookie.SameSite), samesite) + } + body, err := hdr.Body() + if err != nil { + t.Error(err) + } + if string(body) != wantMessage { + t.Errorf("want body message %q, got %q", wantMessage, body) + } + cookieStr := string(c.AppendKeyValues(nil)) + if wantCookie.String() != cookieStr { + t.Errorf("want full cookie representation\n%qgot:\n%q", wantCookie.String(), cookieStr) + } + data, _ := hdr.AppendRequest(nil) + fmt.Printf("%s", data) +} + +func BenchmarkParseBytes(b *testing.B) { + b.StopTimer() + const ( + wantMethod = "GET" + wantURI = "/data/set" + wantMessage = "hello world!" + asRequest = false + ) + req, _ := http.NewRequest(wantMethod, wantURI, strings.NewReader(wantMessage)) + var buf bytes.Buffer + req.Write(&buf) + data := buf.Bytes() + b.StartTimer() + + for i := 0; i < b.N; i++ { + var hdr Header + err := hdr.ParseBytes(asRequest, data) + if err != nil { + b.Fatal(err) + } + _, err = hdr.Body() + if err != nil { + b.Fatal(err) + } + } +} + +func strSameSite(mode http.SameSite) string { + switch mode { + case http.SameSiteLaxMode: + return "Lax" + case http.SameSiteDefaultMode: + return "" + case http.SameSiteStrictMode: + return "Strict" + case http.SameSiteNoneMode: + return "None" + default: + panic("invalid same site") + } +} diff --git a/http/httpraw/parse.go b/http/httpraw/parse.go new file mode 100644 index 0000000..3116512 --- /dev/null +++ b/http/httpraw/parse.go @@ -0,0 +1,411 @@ +package httpraw + +import ( + "bytes" + "errors" + "slices" + "unsafe" +) + +var ( + errNeedMore = errors.New("need more data: cannot find trailing lf") + errUnparsed = errors.New("need to finish parsing") + errInvalidName = errors.New("invalid header name") + errSmallBuffer = errors.New("small read buffer. Increase ReadBufferSize") + errOOM = errors.New("httpraw: buffer out of memory") + // Header.Set and Header.Add mangles the buffer. + // Call them after retrieving the Body. Do not call them before parsing the header (why would you even do that?). + errMangledBuffer = errors.New("httpraw: mangled buffer") + errNoCookies = errors.New("no cookie found") +) + +type headerBuf struct { + // buf[:len] holds entire HTTP header data, which may be normalized by [flags]. buf[off:len] holds data not yet processed during parsing. + buf []byte + // offset into buf for parsing. + off int + // args contains key-value store. + headers []argsKV +} + +type tokint = uint16 + +type headerSlice struct { + start tokint + len tokint +} + +type argsKV struct { + key headerSlice + value headerSlice // value start >0 means value is present. +} + +type scannerState struct { + err error + + // by checking whether the next line contains a colon or not to tell + // it's a header entry or a multi line value of current header entry. + // the side effect of this operation is that we know the index of the + // next colon and new line, so this can be used during next iteration, + // instead of find them again. + nextColon int + nextNewLine int + + initialized bool +} + +func (h *Header) parse(asResponse bool) (err error) { + err = h.parseFirstLine(asResponse) + if err != nil { + return err + } + return h.parseNextHeaders() +} + +func (h *Header) parseFirstLine(asResponse bool) (err error) { + if asResponse { + h.statusCode, h.statusText, h.flags, err = h.hbuf.parseFirstLineResponse(h.flags) + } else { + h.method, h.requestURI, h.proto, h.flags, err = h.hbuf.parseFirstLineRequest(h.flags) + } + return err +} + +func (h *Header) parseNextHeaders() error { + var ss scannerState + h.hbuf.parseNextHeaders(&ss) + if ss.err != nil { + h.flags |= flagConnClose + return ss.err + } + h.flags |= flagDoneParsingHeader + return nil +} + +func (hb *headerBuf) readFromBytes(b []byte) { + hb.buf = append(hb.buf, b...) +} + +func (hb *headerBuf) free() int { return cap(hb.buf) - len(hb.buf) } + +func (hb *headerBuf) parseNextHeaders(ss *scannerState) { + for kv := hb.next(ss); kv.isValid(); kv = hb.next(ss) { + hb.headers = append(hb.headers, kv) + } +} + +func (hb *headerBuf) offBuf() []byte { + return hb.buf[hb.off:] +} + +func (hb *headerBuf) scanLine() []byte { + buf := hb.scanUntilByte('\n') + if len(buf) > 0 && buf[len(buf)-1] == '\r' { + buf = buf[:len(buf)-1] // exclude carriage return. + } + if hb.off < len(hb.buf) { + hb.off++ // consume newline. + } + return buf +} + +func (hb *headerBuf) scanUntilByte(c byte) []byte { + buf := hb.offBuf() + idx := bytes.IndexByte(buf, c) + if idx >= 0 { + buf = buf[:idx] + } + hb.off += len(buf) + return buf +} + +func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto headerSlice, flags flags, err error) { + hb.off = 0 // Parsing first line resets offset. + var b []byte + for len(b) == 0 { + b = hb.scanLine() + } + flags = initFlags + if len(b) < 5 { + return method, uri, proto, flags, errNeedMore + } + + methodEnd := max(0, bytes.IndexByte(b, ' ')) + reqURIEnd := bytes.IndexByte(b[methodEnd+1:], ' ') + if reqURIEnd > 0 { + reqURIEnd += methodEnd + 1 + uri = hb.slice(b[methodEnd+1 : reqURIEnd]) + if b2s(b[methodEnd+1:reqURIEnd]) != strHTTP11 { + flags |= flagNoHTTP11 + } + } else if reqURIEnd == 0 { + return method, uri, proto, flags, errors.New("empty URI") + } else { + // No version provided. + reqURIEnd = methodEnd + 1 + flags |= flagNoHTTP11 + uri = hb.slice(b[methodEnd+1 : reqURIEnd]) + } + proto = hb.slice(b[reqURIEnd:]) + method = hb.slice(b[:methodEnd]) + return method, uri, proto, flags, nil +} + +func (hb *headerBuf) parseFirstLineResponse(initFlags flags) (statusCode, statusText headerSlice, flags flags, err error) { + hb.off = 0 // Parsing first line resets offset. + var b []byte + for len(b) == 0 { + b = hb.scanLine() + } + flags = initFlags + if len(b) < 5 { + return statusCode, statusText, flags, errNeedMore + } + + statusCodeEnd := max(0, bytes.IndexByte(b, ' ')) + if statusCodeEnd < 0 { + return statusCode, statusText, flags, errors.New("missing status code") + } + code := b[:statusCodeEnd] + text := b[statusCodeEnd:] + if len(code) > 3 { + return statusCode, statusText, flags, errors.New("long status code") + } + for i := range code { + if code[i] > '9' || code[i] < '0' { + return statusCode, statusText, flags, errors.New("invalid status code") + } + } + statusCode = hb.slice(code) + statusText = hb.slice(text) + return statusCode, statusText, flags, nil +} + +func (kv argsKV) isValid() bool { + return kv.key.start > 0 +} + +func (kv *argsKV) invalidate() { + *kv = argsKV{} +} + +func (tb headerBuf) musttoken(slice headerSlice) []byte { + return tok2bytes(tb.buf, slice) + +} + +func (tb headerBuf) slice(b []byte) headerSlice { + return bytes2tok(tb.buf, b) +} + +func (kv argsKV) HasValue() bool { return kv.value.start > 0 } + +func (h *Header) hasHeaderValue(key, value string) bool { + kv := h.peekHeader(key) + return kv.isValid() && b2s(h.hbuf.musttoken(kv.value)) == value +} + +// peekHeader returns header key-value for the given key. +// +// The returned value is valid until the request is released, +// either though ReleaseRequest or your request handler returning. +// Do not store references to returned value. Make copies instead. +func (h *Header) peekHeader(key string) argsKV { + hb := &h.hbuf + for i := 0; i < len(h.hbuf.headers); i++ { + if b2s(hb.musttoken(h.hbuf.headers[i].key)) == key { + return h.hbuf.headers[i] + } + } + return hb.noKV() +} + +func (h *Header) peekPtrHeader(key string) *argsKV { + hb := &h.hbuf + for i := 0; i < len(h.hbuf.headers); i++ { + if b2s(hb.musttoken(h.hbuf.headers[i].key)) == key { + return &h.hbuf.headers[i] + } + } + return nil +} + +func (hb *headerBuf) mustAppendSlice(value string) headerSlice { + L := len(hb.buf) + copy(hb.buf[L:L+len(value)], value) + hb.buf = hb.buf[:L+len(value)] + return hb.slice(hb.buf[L : L+len(value)]) +} + +func (h *Header) reuseOrAppend(tok headerSlice, value string) headerSlice { + if tok.len > tokint(len(value)) { + copy(h.hbuf.musttoken(tok), value) + tok.len = tokint(len(value)) + return tok + } + return h.appendSlice(value) +} + +func (h *Header) appendSlice(value string) headerSlice { + free := h.hbuf.free() + if len(value) > free { + if h.flags.hasAny(flagNoBufferGrow) { + h.flags |= flagOOMReached + return headerSlice{} + } + h.hbuf.buf = slices.Grow(h.hbuf.buf, len(value)) + } + h.flags |= flagMangledBuffer + return h.hbuf.mustAppendSlice(value) +} + +func (h *Header) appendHeader(key, value string) { + hb := &h.hbuf + free := hb.free() + buf := h.hbuf.buf + + if len(key)+len(value) > free { + if h.flags.hasAny(flagNoBufferGrow) { + panic(errSmallBuffer) + } + hb.buf = slices.Grow(buf, len(key)+len(value)) + } + h.flags |= flagMangledBuffer + k := hb.mustAppendSlice(key) + v := hb.mustAppendSlice(value) + hb.headers = append(hb.headers, argsKV{ + key: k, + value: v, + }) +} + +func (hb *headerBuf) noKV() argsKV { return argsKV{} } + +func (hb *headerBuf) next(ss *scannerState) argsKV { + if !ss.initialized { + ss.nextColon = -1 + ss.nextNewLine = -1 + } + buf := hb.buf[hb.off:] + blen := len(buf) + if blen >= 2 && buf[0] == '\r' && buf[1] == '\n' { + hb.off += 2 + return hb.noKV() // \r\n\r\n Ends header. + } else if blen >= 1 && buf[0] == '\n' { + hb.off += 1 + return hb.noKV() // \n\n Ends header. + } + + // n is parsing offset. Will start by storing colon index. + n := 0 + if ss.nextColon >= 0 { + // Retake from last colon found. + n = ss.nextColon + ss.nextColon = -1 + } else { + n = bytes.IndexByte(buf, ':') + x := bytes.IndexByte(buf, '\n') + if x < 0 { + // A header name should always at some point be followed by a \n + // even if it's the one that terminates the header block. + ss.err = errNeedMore + return hb.noKV() + } else if x < n { + // There was a \n before the colon! This is invalid. + ss.err = errInvalidName + return hb.noKV() + } else if n < 0 { + // No colon found, probably missing data. + ss.err = errNeedMore + return hb.noKV() + } + } + // n stores colon position by now. + if bytes.IndexByte(buf[:n], ' ') >= 0 || bytes.IndexByte(buf[:n], '\t') >= 0 { + // Spaces between the header key and colon are not allowed. + // See RFC 7230, Section 3.2.4. + ss.err = errInvalidName + return hb.noKV() + } + + // Ready to store key.. + var resultKV argsKV + resultKV.key = hb.slice(buf[:n]) + n++ // consume colon. + for len(buf) > n && buf[n] == ' ' { + n++ // Trim leading spaces. + } + // n now points to start of value. + valueStart := n + + // Find end of value. Values may be multiline, in which case we must treat newlines followed by whitespace as part of the value. + for { + nl := bytes.IndexByte(buf[n:], '\n') + if nl < 0 || nl+n+1 == len(buf) { + // No newline or newline is last character and can't know if is multiline. + ss.err = errNeedMore + return hb.noKV() + } + n += nl + 1 // Index of the newly found newline. + nextChar := buf[n] + if nextChar != ' ' && nextChar != '\t' { + break // End of value found. + } + } + + valueEnd := n - 1 // Trim newline. + if valueEnd > valueStart && buf[valueEnd-1] == '\r' { + valueEnd-- // Trim \r character if present before value. + } + resultKV.value = hb.slice(buf[valueStart:valueEnd]) + hb.off += n + return resultKV +} + +// reset sets the buffer data and discards all parsed data. +func (h *headerBuf) reset(buf []byte) { + if buf == nil { + buf = h.buf[:0] // Reuse buffer but discard raw data on nil input. + } + *h = headerBuf{ + buf: buf, + headers: h.headers[:0], + } +} + +// ConnectionClose returns true if 'Connection: close' header is set or if a invalid header was found. +func (h *Header) ConnectionClose() bool { + closed := h.flags.hasAny(flagConnClose) || + (h.flags.hasAny(flagNoHTTP11) && !h.hasHeaderValue("Connection", "keep-alive")) + if closed { + h.flags |= flagConnClose + } + return closed +} + +// b2s converts byte slice to a string without memory allocation. +// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . +func b2s(b []byte) string { + return unsafe.String(unsafe.SliceData(b), len(b)) +} + +// s2b converts string to a byte slice without memory allocation. +func s2b(s string) []byte { + return unsafe.Slice(unsafe.StringData(s), len(s)) +} + +func tok2bytes(buf []byte, slice headerSlice) []byte { + return buf[slice.start : slice.start+slice.len] +} + +func bytes2tok(buf, value []byte) headerSlice { + base := uintptr(unsafe.Pointer(unsafe.SliceData(buf))) + off := uintptr(unsafe.Pointer(unsafe.SliceData(value))) + if off < base || off > base+uintptr(len(buf)) { + panic("httpx: argument buffer does not alias header buffer") + } + return headerSlice{ + start: tokint(off - base), + len: tokint(len(value)), + } +} diff --git a/httpx/cookie.go b/httpx/cookie.go deleted file mode 100644 index 611af48..0000000 --- a/httpx/cookie.go +++ /dev/null @@ -1,700 +0,0 @@ -package httpx - -import ( - "bytes" - "errors" - "io" - "path/filepath" - "strconv" - "strings" - "sync" - "time" -) - -var zeroTime time.Time - -var ( - // cookieExpireDelete may be set on Cookie.Expire for expiring the given cookie. - cookieExpireDelete = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - - // cookieExpireUnlimited indicates that the cookie doesn't expire. - cookieExpireUnlimited = zeroTime -) - -// CookieSameSite is an enum for the mode in which the SameSite flag should be set for the given cookie. -// See https://tools.ietf.org/html/draft-ietf-httpbis-cookie-same-site-00 for details. -type CookieSameSite int - -const ( - // CookieSameSiteDisabled removes the SameSite flag. - CookieSameSiteDisabled CookieSameSite = iota - // CookieSameSiteDefaultMode sets the SameSite flag. - CookieSameSiteDefaultMode - // CookieSameSiteLaxMode sets the SameSite flag with the "Lax" parameter. - CookieSameSiteLaxMode - // CookieSameSiteStrictMode sets the SameSite flag with the "Strict" parameter. - CookieSameSiteStrictMode - // CookieSameSiteNoneMode sets the SameSite flag with the "None" parameter. - // See https://tools.ietf.org/html/draft-west-cookie-incrementalism-00 - CookieSameSiteNoneMode -) - -// acquireCookie returns an empty Cookie object from the pool. -// -// The returned object may be returned back to the pool with ReleaseCookie. -// This allows reducing GC load. -func acquireCookie() *Cookie { - return cookiePool.Get().(*Cookie) -} - -// releaseCookie returns the Cookie object acquired with AcquireCookie back -// to the pool. -// -// Do not access released Cookie object, otherwise data races may occur. -func releaseCookie(c *Cookie) { - c.Reset() - cookiePool.Put(c) -} - -var cookiePool = &sync.Pool{ - New: func() any { - return &Cookie{} - }, -} - -// Cookie represents HTTP response cookie. -// -// Do not copy Cookie objects. Create new object and use CopyTo instead. -// -// Cookie instance MUST NOT be used from concurrently running goroutines. -type Cookie struct { - noCopy noCopy - - key []byte - value []byte - expire time.Time - maxAge int - domain []byte - path []byte - - httpOnly bool - secure bool - sameSite CookieSameSite - - bufKV argsKV - buf []byte -} - -// CopyTo copies src cookie to c. -func (c *Cookie) CopyTo(src *Cookie) { - c.Reset() - c.key = append(c.key, src.key...) - c.value = append(c.value, src.value...) - c.expire = src.expire - c.maxAge = src.maxAge - c.domain = append(c.domain, src.domain...) - c.path = append(c.path, src.path...) - c.httpOnly = src.httpOnly - c.secure = src.secure - c.sameSite = src.sameSite -} - -// HTTPOnly returns true if the cookie is http only. -func (c *Cookie) HTTPOnly() bool { - return c.httpOnly -} - -// SetHTTPOnly sets cookie's httpOnly flag to the given value. -func (c *Cookie) SetHTTPOnly(httpOnly bool) { - c.httpOnly = httpOnly -} - -// Secure returns true if the cookie is secure. -func (c *Cookie) Secure() bool { - return c.secure -} - -// SetSecure sets cookie's secure flag to the given value. -func (c *Cookie) SetSecure(secure bool) { - c.secure = secure -} - -// SameSite returns the SameSite mode. -func (c *Cookie) SameSite() CookieSameSite { - return c.sameSite -} - -// SetSameSite sets the cookie's SameSite flag to the given value. -// Set value CookieSameSiteNoneMode will set Secure to true also to avoid browser rejection. -func (c *Cookie) SetSameSite(mode CookieSameSite) { - c.sameSite = mode - if mode == CookieSameSiteNoneMode { - c.SetSecure(true) - } -} - -// Path returns cookie path. -func (c *Cookie) Path() []byte { - return c.path -} - -// SetPath sets cookie path. -func (c *Cookie) SetPath(path string) { - c.buf = append(c.buf[:0], path...) - c.path = normalizePath(c.path, b2s(c.buf)) -} - -// SetPathBytes sets cookie path. -func (c *Cookie) SetPathBytes(path []byte) { - c.buf = append(c.buf[:0], path...) - c.path = normalizePath(c.path, b2s(c.buf)) -} - -// Domain returns cookie domain. -// -// The returned value is valid until the Cookie reused or released (ReleaseCookie). -// Do not store references to the returned value. Make copies instead. -func (c *Cookie) Domain() []byte { - return c.domain -} - -// SetDomain sets cookie domain. -func (c *Cookie) SetDomain(domain string) { - c.domain = append(c.domain[:0], domain...) -} - -// SetDomainBytes sets cookie domain. -func (c *Cookie) SetDomainBytes(domain []byte) { - c.domain = append(c.domain[:0], domain...) -} - -// MaxAge returns the seconds until the cookie is meant to expire or 0 -// if no max age. -func (c *Cookie) MaxAge() int { - return c.maxAge -} - -// SetMaxAge sets cookie expiration time based on seconds. This takes precedence -// over any absolute expiry set on the cookie. -// -// Set max age to 0 to unset. -func (c *Cookie) SetMaxAge(seconds int) { - c.maxAge = seconds -} - -// Expire returns cookie expiration time. -// -// CookieExpireUnlimited is returned if cookie doesn't expire. -func (c *Cookie) Expire() time.Time { - expire := c.expire - if expire.IsZero() { - expire = cookieExpireUnlimited - } - return expire -} - -// SetExpire sets cookie expiration time. -// -// Set expiration time to CookieExpireDelete for expiring (deleting) -// the cookie on the client. -// -// By default cookie lifetime is limited by browser session. -func (c *Cookie) SetExpire(expire time.Time) { - c.expire = expire -} - -// Value returns cookie value. -// -// The returned value is valid until the Cookie reused or released (ReleaseCookie). -// Do not store references to the returned value. Make copies instead. -func (c *Cookie) Value() []byte { - return c.value -} - -// SetValue sets cookie value. -func (c *Cookie) SetValue(value string) { - c.value = append(c.value[:0], value...) -} - -// SetValueBytes sets cookie value. -func (c *Cookie) SetValueBytes(value []byte) { - c.value = append(c.value[:0], value...) -} - -// Key returns cookie name. -// -// The returned value is valid until the Cookie reused or released (ReleaseCookie). -// Do not store references to the returned value. Make copies instead. -func (c *Cookie) Key() []byte { - return c.key -} - -// SetKey sets cookie name. -func (c *Cookie) SetKey(key string) { - c.key = append(c.key[:0], key...) -} - -// SetKeyBytes sets cookie name. -func (c *Cookie) SetKeyBytes(key []byte) { - c.key = append(c.key[:0], key...) -} - -// Reset clears the cookie. -func (c *Cookie) Reset() { - c.key = c.key[:0] - c.value = c.value[:0] - c.expire = zeroTime - c.maxAge = 0 - c.domain = c.domain[:0] - c.path = c.path[:0] - c.httpOnly = false - c.secure = false - c.sameSite = CookieSameSiteDisabled -} - -// AppendBytes appends cookie representation to dst and returns -// the extended dst. -func (c *Cookie) AppendBytes(dst []byte) []byte { - if len(c.key) > 0 { - dst = append(dst, c.key...) - dst = append(dst, '=') - } - dst = append(dst, c.value...) - - if c.maxAge > 0 { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieMaxAge...) - dst = append(dst, '=') - dst = appendUint(dst, c.maxAge) - } else if !c.expire.IsZero() { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieExpires...) - dst = append(dst, '=') - dst = AppendHTTPDate(dst, c.expire) - } - if len(c.domain) > 0 { - dst = appendCookiePart(dst, strCookieDomain, b2s(c.domain)) - } - if len(c.path) > 0 { - dst = appendCookiePart(dst, strCookiePath, b2s(c.path)) - } - if c.httpOnly { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieHTTPOnly...) - } - if c.secure { - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSecure...) - } - switch c.sameSite { - case CookieSameSiteDefaultMode: - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSameSite...) - case CookieSameSiteLaxMode: - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSameSite...) - dst = append(dst, '=') - dst = append(dst, strCookieSameSiteLax...) - case CookieSameSiteStrictMode: - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSameSite...) - dst = append(dst, '=') - dst = append(dst, strCookieSameSiteStrict...) - case CookieSameSiteNoneMode: - dst = append(dst, ';', ' ') - dst = append(dst, strCookieSameSite...) - dst = append(dst, '=') - dst = append(dst, strCookieSameSiteNone...) - } - return dst -} - -// Cookie returns cookie representation. -// -// The returned value is valid until the Cookie reused or released (ReleaseCookie). -// Do not store references to the returned value. Make copies instead. -func (c *Cookie) Cookie() []byte { - c.buf = c.AppendBytes(c.buf[:0]) - return c.buf -} - -// String returns cookie representation. -func (c *Cookie) String() string { - return string(c.Cookie()) -} - -// WriteTo writes cookie representation to w. -// -// WriteTo implements io.WriterTo interface. -func (c *Cookie) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(c.Cookie()) - return int64(n), err -} - -var errNoCookies = errors.New("no cookies found") - -// Parse parses Set-Cookie header. -func (c *Cookie) Parse(src string) error { - c.buf = append(c.buf[:0], src...) - return c.ParseBytes(c.buf) -} - -// ParseBytes parses Set-Cookie header. -func (c *Cookie) ParseBytes(src []byte) error { - c.Reset() - ntot := 0 - for { - k, v, n := parseCookie(src) - if n == 0 { - break - } else if ntot == 0 { - c.key = append(c.key, k...) - c.value = append(c.value, v...) - } - key := b2s(k) - value := b2s(v) - ntot += n - src = src[n:] - if len(key) != 0 { - // Case insensitive switch on first char - switch key[0] | 0x20 { - case 'm': - if caseInsensitiveCompare(strCookieMaxAge, key) { - maxAge, err := strconv.ParseUint(value, 10, 32) - if err != nil { - return err - } - c.maxAge = int(maxAge) - } - - case 'e': // "expires" - if caseInsensitiveCompare(strCookieExpires, key) { - - // Try the same two formats as net/http - // See: https://github.com/golang/go/blob/00379be17e63a5b75b3237819392d2dc3b313a27/src/net/http/cookie.go#L133-L135 - exptime, err := time.ParseInLocation(time.RFC1123, value, time.UTC) - if err != nil { - exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", value) - if err != nil { - return err - } - } - c.expire = exptime - } - - case 'd': // "domain" - if caseInsensitiveCompare(strCookieDomain, key) { - c.domain = append(c.domain, value...) - } - - case 'p': // "path" - if caseInsensitiveCompare(strCookiePath, key) { - c.path = append(c.path, value...) - } - - case 's': // "samesite" - if caseInsensitiveCompare(strCookieSameSite, key) { - if len(value) > 0 { - // Case insensitive switch on first char - switch value[0] | 0x20 { - case 'l': // "lax" - if caseInsensitiveCompare(strCookieSameSiteLax, value) { - c.sameSite = CookieSameSiteLaxMode - } - case 's': // "strict" - if caseInsensitiveCompare(strCookieSameSiteStrict, value) { - c.sameSite = CookieSameSiteStrictMode - } - case 'n': // "none" - if caseInsensitiveCompare(strCookieSameSiteNone, value) { - c.sameSite = CookieSameSiteNoneMode - } - } - } - } - } - } else if len(value) != 0 { - // Case insensitive switch on first char - switch value[0] | 0x20 { - case 'h': // "httponly" - if caseInsensitiveCompare(strCookieHTTPOnly, value) { - c.httpOnly = true - } - - case 's': // "secure" - if caseInsensitiveCompare(strCookieSecure, value) { - c.secure = true - } else if caseInsensitiveCompare(strCookieSameSite, value) { - c.sameSite = CookieSameSiteDefaultMode - } - } - } // else empty or no match - } - if len(c.key) == 0 && len(c.value) == 0 { - return errNoCookies - } - return nil -} - -func appendCookiePart(dst []byte, key, value string) []byte { - dst = append(dst, ';', ' ') - dst = append(dst, key...) - dst = append(dst, '=') - return append(dst, value...) -} - -func (hb *headerBuf) appendRequestCookieBytes(dst []byte) []byte { - n := len(hb.cookies) - for i := 0; i < n; i++ { - kv := hb.cookies[i] - if !kv.isValid() { - continue - } else if kv.key.len > 0 { - dst = append(dst, hb.musttoken(kv.key)...) - dst = append(dst, '=') - } - dst = append(dst, hb.musttoken(kv.value)...) - if i+1 < n { - dst = append(dst, ';', ' ') - } - } - return dst -} -func (hb *headerBuf) appendResponseCookieBytes(dst []byte) []byte { - n := len(hb.cookies) - for i := 0; i < n; i++ { - kv := hb.cookies[i] - if !kv.isValid() { - continue - } - dst = append(dst, hb.musttoken(kv.value)...) - if i+1 < n { - dst = append(dst, ';', ' ') - } - } - return dst -} - -type cookieScanner struct { - b []byte -} - -// parseCookie parses a cookie inside cookie buffer and adds it to cookie buffer.. -// -// Cookie: \r\n -func parseCookie(cookie []byte) (key, value []byte, cookieEnd int) { - if len(cookie) == 0 { - return nil, nil, 0 - } - eqIdx := bytes.IndexByte(cookie, '=') - semiIdx := bytes.IndexByte(cookie, ';') - if eqIdx > 0 && eqIdx < semiIdx { - // cookies has form key=value; - key = trimCookie(cookie[:eqIdx], false) - } else { - // cookie has no key. - eqIdx = -1 // ensure is -1. - } - if semiIdx > 0 { - // found ';' - value = trimCookie(cookie[eqIdx+1:semiIdx], true) - } else { - value = trimCookie(cookie[eqIdx+1:], true) - } - return key, value, max(len(cookie), semiIdx+1) -} - -func trimCookie(src []byte, trimQuotes bool) []byte { - for len(src) > 0 && src[0] == ' ' { - src = src[1:] // skip leading whitespace. - } - for len(src) > 0 && src[len(src)-1] == ' ' { - src = src[:len(src)-1] // skip trailing whitespace - } - if trimQuotes { - if len(src) > 1 && src[0] == '"' && src[len(src)-1] == '"' { - src = src[1 : len(src)-1] // Trim leading+trailing quotes. - } - } - return src -} - -// caseInsensitiveCompare does a case insensitive equality comparison of -// two []byte. Assumes only letters need to be matched. -func caseInsensitiveCompare(a, b string) bool { - if len(a) != len(b) { - return false - } - for i := 0; i < len(a); i++ { - if a[i]|0x20 != b[i]|0x20 { - return false - } - } - return true -} - -func normalizePath(dst []byte, src string) []byte { - dst = dst[:0] - dst = addLeadingSlash(dst, src) - dst = decodeArgAppendNoPlus(dst, src) - - // remove duplicate slashes - b := dst - bSize := len(b) - for { - n := strings.Index(b2s(b), strSlashSlash) - if n < 0 { - break - } - b = b[n:] - copy(b, b[1:]) - b = b[:len(b)-1] - bSize-- - } - dst = dst[:bSize] - - // remove /./ parts - b = dst - for { - n := strings.Index(b2s(b), strSlashDotSlash) - if n < 0 { - break - } - nn := n + len(strSlashDotSlash) - 1 - copy(b[n:], b[nn:]) - b = b[:len(b)-nn+n] - } - - // remove /foo/../ parts - for { - n := strings.Index(b2s(b), strSlashDotDotSlash) - if n < 0 { - break - } - nn := strings.LastIndexByte(b2s(b[:n]), slashChar) - if nn < 0 { - nn = 0 - } - n += len(strSlashDotDotSlash) - 1 - copy(b[nn:], b[n:]) - b = b[:len(b)-n+nn] - } - - // remove trailing /foo/.. - n := strings.LastIndex(b2s(b), strSlashDotDot) - if n >= 0 && n+len(strSlashDotDot) == len(b) { - nn := strings.LastIndexByte(b2s(b[:n]), slashChar) - if nn < 0 { - return append(dst[:0], slashChar) - } - b = b[:nn+1] - } - - if filepath.Separator == '\\' { - // remove \.\ parts - for { - n := strings.Index(b2s(b), strBackSlashDotBackSlash) - if n < 0 { - break - } - nn := n + len(strSlashDotSlash) - 1 - copy(b[n:], b[nn:]) - b = b[:len(b)-nn+n] - } - - // remove /foo/..\ parts - for { - n := strings.Index(b2s(b), strSlashDotDotBackSlash) - if n < 0 { - break - } - nn := strings.LastIndexByte(b2s(b[:n]), slashChar) - if nn < 0 { - nn = 0 - } - nn++ - n += len(strSlashDotDotBackSlash) - copy(b[nn:], b[n:]) - b = b[:len(b)-n+nn] - } - - // remove /foo\..\ parts - for { - n := strings.Index(b2s(b), strBackSlashDotDotBackSlash) - if n < 0 { - break - } - nn := strings.LastIndexByte(b2s(b[:n]), slashChar) - if nn < 0 { - nn = 0 - } - n += len(strBackSlashDotDotBackSlash) - 1 - copy(b[nn:], b[n:]) - b = b[:len(b)-n+nn] - } - - // remove trailing \foo\.. - n := strings.LastIndex(b2s(b), strBackSlashDotDot) - if n >= 0 && n+len(strSlashDotDot) == len(b) { - nn := strings.LastIndexByte(b2s(b[:n]), slashChar) - if nn < 0 { - return append(dst[:0], slashChar) - } - b = b[:nn+1] - } - } - - return b -} - -func addLeadingSlash(dst []byte, src string) []byte { - // add leading slash for unix paths - if len(src) == 0 || src[0] != slashChar { - dst = append(dst, slashChar) - } - - return dst -} - -// decodeArgAppendNoPlus is almost identical to decodeArgAppend, but it doesn't -// substitute '+' with ' '. -// -// The function is copy-pasted from decodeArgAppend due to the performance -// reasons only. -func decodeArgAppendNoPlus(dst []byte, src string) []byte { - idx := strings.IndexByte(src, '%') - if idx < 0 { - // fast path: src doesn't contain encoded chars - return append(dst, src...) - } - dst = append(dst, src[:idx]...) - - // slow path - for i := idx; i < len(src); i++ { - c := src[i] - if c == '%' { - if i+2 >= len(src) { - return append(dst, src[i:]...) - } - x2 := hex2intTable[src[i+2]] - x1 := hex2intTable[src[i+1]] - if x1 == 16 || x2 == 16 { - dst = append(dst, '%') - } else { - dst = append(dst, x1<<4|x2) - i += 2 - } - } else { - dst = append(dst, c) - } - } - return dst -} - -// AppendHTTPDate appends HTTP-compliant (RFC1123) representation of date -// to dst and returns the extended dst. -func AppendHTTPDate(dst []byte, date time.Time) []byte { - dst = date.In(time.UTC).AppendFormat(dst, time.RFC1123) - copy(dst[len(dst)-3:], strGMT) - return dst -} diff --git a/httpx/definitions.go b/httpx/definitions.go deleted file mode 100644 index ce73fca..0000000 --- a/httpx/definitions.go +++ /dev/null @@ -1,270 +0,0 @@ -package httpx - -const ( - slashChar = '/' - rChar = '\r' - nChar = '\n' - defaultServerName = "fasthttp" - defaultUserAgent = "fasthttp" - defaultContentType = "text/plain; charset=utf-8" -) - -const ( - strSlashSlash = "//" - strSlashDotDot = "/.." - strSlashDotSlash = "/./" - strSlashDotDotSlash = "/../" - strBackSlashDotDot = `\..` - strBackSlashDotBackSlash = `\.\` - strSlashDotDotBackSlash = `/..\` - strBackSlashDotDotBackSlash = `\..\` - strCRLF = "\r\n" - strHTTP = "http" - strHTTPS = "https" - strHTTP10 = "HTTP/1.0" - strHTTP11 = "HTTP/1.1" - strColon = ":" - strColonSlashSlash = "://" - strColonSpace = ": " - strCommaSpace = ", " - strGMT = "GMT" - - strResponseContinue = "HTTP/1.1 100 Continue\r\n\r\n" - - strExpect = HeaderExpect - strConnection = HeaderConnection - strContentLength = HeaderContentLength - strContentType = HeaderContentType - strDate = HeaderDate - strHost = HeaderHost - strReferer = HeaderReferer - strServer = HeaderServer - strTransferEncoding = HeaderTransferEncoding - strContentEncoding = HeaderContentEncoding - strAcceptEncoding = HeaderAcceptEncoding - strUserAgent = HeaderUserAgent - strCookie = HeaderCookie - strSetCookie = HeaderSetCookie - strLocation = HeaderLocation - strIfModifiedSince = HeaderIfModifiedSince - strLastModified = HeaderLastModified - strAcceptRanges = HeaderAcceptRanges - strRange = HeaderRange - strContentRange = HeaderContentRange - strAuthorization = HeaderAuthorization - strTE = HeaderTE - strTrailer = HeaderTrailer - strMaxForwards = HeaderMaxForwards - strProxyConnection = HeaderProxyConnection - strProxyAuthenticate = HeaderProxyAuthenticate - strProxyAuthorization = HeaderProxyAuthorization - strWWWAuthenticate = HeaderWWWAuthenticate - strVary = HeaderVary - - strCookieExpires = "expires" - strCookieDomain = "domain" - strCookiePath = "path" - strCookieHTTPOnly = "HttpOnly" - strCookieSecure = "secure" - strCookieMaxAge = "max-age" - strCookieSameSite = "SameSite" - strCookieSameSiteLax = "Lax" - strCookieSameSiteStrict = "Strict" - strCookieSameSiteNone = "None" - - strClose = "close" - strGzip = "gzip" - strBr = "br" - strDeflate = "deflate" - strKeepAlive = "keep-alive" - strUpgrade = "Upgrade" - strChunked = "chunked" - strIdentity = "identity" - str100Continue = "100-continue" - strPostArgsContentType = "application/x-www-form-urlencoded" - strDefaultContentType = "application/octet-stream" - strMultipartFormData = "multipart/form-data" - strBoundary = "boundary" - strBytes = "bytes" - strBasicSpace = "Basic " - - strApplicationSlash = "application/" - strImageSVG = "image/svg" - strImageIcon = "image/x-icon" - strFontSlash = "font/" - strMultipartSlash = "multipart/" - strTextSlash = "text/" -) - -// Headers. -const ( - // Authentication. - HeaderAuthorization = "Authorization" - HeaderProxyAuthenticate = "Proxy-Authenticate" - HeaderProxyAuthorization = "Proxy-Authorization" - HeaderWWWAuthenticate = "WWW-Authenticate" - - // Caching. - HeaderAge = "Age" - HeaderCacheControl = "Cache-Control" - HeaderClearSiteData = "Clear-Site-Data" - HeaderExpires = "Expires" - HeaderPragma = "Pragma" - HeaderWarning = "Warning" - - // Client hints. - HeaderAcceptCH = "Accept-CH" - HeaderAcceptCHLifetime = "Accept-CH-Lifetime" - HeaderContentDPR = "Content-DPR" - HeaderDPR = "DPR" - HeaderEarlyData = "Early-Data" - HeaderSaveData = "Save-Data" - HeaderViewportWidth = "Viewport-Width" - HeaderWidth = "Width" - - // Conditionals. - HeaderETag = "ETag" - HeaderIfMatch = "If-Match" - HeaderIfModifiedSince = "If-Modified-Since" - HeaderIfNoneMatch = "If-None-Match" - HeaderIfUnmodifiedSince = "If-Unmodified-Since" - HeaderLastModified = "Last-Modified" - HeaderVary = "Vary" - - // Connection management. - HeaderConnection = "Connection" - HeaderKeepAlive = "Keep-Alive" - HeaderProxyConnection = "Proxy-Connection" - - // Content negotiation. - HeaderAccept = "Accept" - HeaderAcceptCharset = "Accept-Charset" - HeaderAcceptEncoding = "Accept-Encoding" - HeaderAcceptLanguage = "Accept-Language" - - // Controls. - HeaderCookie = "Cookie" - HeaderExpect = "Expect" - HeaderMaxForwards = "Max-Forwards" - HeaderSetCookie = "Set-Cookie" - - // CORS. - HeaderAccessControlAllowCredentials = "Access-Control-Allow-Credentials" - HeaderAccessControlAllowHeaders = "Access-Control-Allow-Headers" - HeaderAccessControlAllowMethods = "Access-Control-Allow-Methods" - HeaderAccessControlAllowOrigin = "Access-Control-Allow-Origin" - HeaderAccessControlExposeHeaders = "Access-Control-Expose-Headers" - HeaderAccessControlMaxAge = "Access-Control-Max-Age" - HeaderAccessControlRequestHeaders = "Access-Control-Request-Headers" - HeaderAccessControlRequestMethod = "Access-Control-Request-Method" - HeaderOrigin = "Origin" - HeaderTimingAllowOrigin = "Timing-Allow-Origin" - HeaderXPermittedCrossDomainPolicies = "X-Permitted-Cross-Domain-Policies" - - // Do Not Track. - HeaderDNT = "DNT" - HeaderTk = "Tk" - - // Downloads. - HeaderContentDisposition = "Content-Disposition" - - // Message body information. - HeaderContentEncoding = "Content-Encoding" - HeaderContentLanguage = "Content-Language" - HeaderContentLength = "Content-Length" - HeaderContentLocation = "Content-Location" - HeaderContentType = "Content-Type" - - // Proxies. - HeaderForwarded = "Forwarded" - HeaderVia = "Via" - HeaderXForwardedFor = "X-Forwarded-For" - HeaderXForwardedHost = "X-Forwarded-Host" - HeaderXForwardedProto = "X-Forwarded-Proto" - - // Redirects. - HeaderLocation = "Location" - - // Request context. - HeaderFrom = "From" - HeaderHost = "Host" - HeaderReferer = "Referer" - HeaderReferrerPolicy = "Referrer-Policy" - HeaderUserAgent = "User-Agent" - - // Response context. - HeaderAllow = "Allow" - HeaderServer = "Server" - - // Range requests. - HeaderAcceptRanges = "Accept-Ranges" - HeaderContentRange = "Content-Range" - HeaderIfRange = "If-Range" - HeaderRange = "Range" - - // Security. - HeaderContentSecurityPolicy = "Content-Security-Policy" - HeaderContentSecurityPolicyReportOnly = "Content-Security-Policy-Report-Only" - HeaderCrossOriginResourcePolicy = "Cross-Origin-Resource-Policy" - HeaderExpectCT = "Expect-CT" - HeaderFeaturePolicy = "Feature-Policy" - HeaderPublicKeyPins = "Public-Key-Pins" - HeaderPublicKeyPinsReportOnly = "Public-Key-Pins-Report-Only" - HeaderStrictTransportSecurity = "Strict-Transport-Security" - HeaderUpgradeInsecureRequests = "Upgrade-Insecure-Requests" - HeaderXContentTypeOptions = "X-Content-Type-Options" - HeaderXDownloadOptions = "X-Download-Options" - HeaderXFrameOptions = "X-Frame-Options" - HeaderXPoweredBy = "X-Powered-By" - HeaderXXSSProtection = "X-XSS-Protection" - - // Server-sent event. - HeaderLastEventID = "Last-Event-ID" - HeaderNEL = "NEL" - HeaderPingFrom = "Ping-From" - HeaderPingTo = "Ping-To" - HeaderReportTo = "Report-To" - - // Transfer coding. - HeaderTE = "TE" - HeaderTrailer = "Trailer" - HeaderTransferEncoding = "Transfer-Encoding" - - // WebSockets. - HeaderSecWebSocketAccept = "Sec-WebSocket-Accept" - HeaderSecWebSocketExtensions = "Sec-WebSocket-Extensions" /* #nosec G101 */ - HeaderSecWebSocketKey = "Sec-WebSocket-Key" - HeaderSecWebSocketProtocol = "Sec-WebSocket-Protocol" - HeaderSecWebSocketVersion = "Sec-WebSocket-Version" - - // Other. - HeaderAcceptPatch = "Accept-Patch" - HeaderAcceptPushPolicy = "Accept-Push-Policy" - HeaderAcceptSignature = "Accept-Signature" - HeaderAltSvc = "Alt-Svc" - HeaderDate = "Date" - HeaderIndex = "Index" - HeaderLargeAllocation = "Large-Allocation" - HeaderLink = "Link" - HeaderPushPolicy = "Push-Policy" - HeaderRetryAfter = "Retry-After" - HeaderServerTiming = "Server-Timing" - HeaderSignature = "Signature" - HeaderSignedHeaders = "Signed-Headers" - HeaderSourceMap = "SourceMap" - HeaderUpgrade = "Upgrade" - HeaderXDNSPrefetchControl = "X-DNS-Prefetch-Control" - HeaderXPingback = "X-Pingback" - HeaderXRequestedWith = "X-Requested-With" - HeaderXRobotsTag = "X-Robots-Tag" - HeaderXUACompatible = "X-UA-Compatible" -) - -// Probably replace these with short functions to take up less program memory -const ( - hex2intTable = "\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x00\x01\x02\x03\x04\x05\x06\a\b\t\x10\x10\x10\x10\x10\x10\x10\n\v\f\r\x0e\x0f\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\n\v\f\r\x0e\x0f\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10" - toLowerTable = "\x00\x01\x02\x03\x04\x05\x06\a\b\t\n\v\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@abcdefghijklmnopqrstuvwxyz[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" - toUpperTable = "\x00\x01\x02\x03\x04\x05\x06\a\b\t\n\v\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~\u007f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" - quotedArgShouldEscapeTable = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" - quotedPathShouldEscapeTable = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x01\x00\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" -) diff --git a/httpx/header_parse.go b/httpx/header_parse.go deleted file mode 100644 index 16c6875..0000000 --- a/httpx/header_parse.go +++ /dev/null @@ -1,663 +0,0 @@ -package httpx - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "slices" - "strings" - "unsafe" -) - -var ( - errNeedMore = errors.New("need more data: cannot find trailing lf") - errInvalidName = errors.New("invalid header name") - errSmallBuffer = errors.New("small read buffer. Increase ReadBufferSize") - errNonNumericChars = errors.New("non-numeric chars found") -) - -func (hb *headerBuf) readFromBytes(b []byte) { - hb.buf = append(hb.buf, b...) -} - -func (hb *headerBuf) free() int { return cap(hb.buf) - len(hb.buf) } - -func (hb *headerBuf) readFrom(r io.Reader) error { - buf := hb.buf - free := hb.free() - if free == 0 { - return errSmallBuffer - } - n, err := r.Read(buf[len(buf):cap(buf)]) - hb.buf = buf[:len(buf)+n] - return err -} - -func (h *header) parse() (err error) { - hb := &h.hbuf - hb.off = 0 // start parsing from 0. - h.method, h.requestURI, h.proto, h.flags, err = hb.parseFirstLine(h.flags) - if err != nil { - return err - } - - var ss scannerState - err = h.parseHeaders(&ss) - if err != nil { - return err - } - return nil -} - -func (hb *headerBuf) offBuf() []byte { - return hb.buf[hb.off:] -} - -func (hb *headerBuf) scanLine() []byte { - buf := hb.scanUntilByte('\n') - if len(buf) > 0 && buf[len(buf)-1] == '\r' { - buf = buf[:len(buf)-1] // exclude carriage return. - } - if hb.off < len(hb.buf) { - hb.off++ // consume newline. - } - return buf -} - -func (hb *headerBuf) scanUntilByte(c byte) []byte { - buf := hb.offBuf() - idx := bytes.IndexByte(buf, c) - if idx >= 0 { - buf = buf[:idx] - } - hb.off += len(buf) - return buf -} - -func (hb *headerBuf) parseFirstLine(initFlags flags) (method, uri, proto headerSlice, flags flags, err error) { - var b []byte - for len(b) == 0 { - b = hb.scanLine() - } - flags = initFlags - if len(b) < 5 { - return method, uri, proto, flags, errors.New("too short first HTTP line") - } - - methodEnd := max(0, bytes.IndexByte(b, ' ')) - reqURIEnd := bytes.IndexByte(b[methodEnd+1:], ' ') - if reqURIEnd >= 0 { - reqURIEnd += methodEnd + 1 - } - switch { - case reqURIEnd < 0: - flags |= noHTTP11 - reqURIEnd = methodEnd + 1 - case reqURIEnd == 0: - return method, uri, proto, flags, errors.New("empty URI") - case b2s(b[reqURIEnd+1:]) != strHTTP11: - flags |= noHTTP11 - fallthrough - default: - proto = hb.slice(b[reqURIEnd+1:]) - } - uri = hb.slice(b[methodEnd+1 : reqURIEnd]) - method = hb.slice(b[:methodEnd]) - return method, uri, proto, flags, nil -} - -type scannerState struct { - err error - disableNormalizing bool - - // by checking whether the next line contains a colon or not to tell - // it's a header entry or a multi line value of current header entry. - // the side effect of this operation is that we know the index of the - // next colon and new line, so this can be used during next iteration, - // instead of find them again. - nextColon int - nextNewLine int - - initialized bool -} - -func (h *header) parseHeaders(ss *scannerState) (err error) { - hb := &h.hbuf - h.contentLength = -2 - - for kv := hb.nextKV2(ss); kv.isValid(); kv = hb.nextKV2(ss) { - if h.flags.hasAny(disableSpecialHeader) { - h.hbuf.headers = append(h.hbuf.headers, kv) - continue - } - } - if ss.err != nil && err == nil { - err = ss.err - } - if err != nil { - h.flags |= connectionClose - return err - } - - // if h.contentLength < 0 { - // h.contentLengthBytes = hb.noKV().value - // } - if h.flags.hasAny(noHTTP11) && !h.flags.hasAny(connectionClose) { - // close connection for non-http/1.1 request unless 'Connection: keep-alive' is set. - if !h.hasHeaderValue(strConnection, strKeepAlive) { - h.flags |= connectionClose - } - } - return nil -} - -func (h *header) hasHeaderValue(key, value string) bool { - kv := h.peekHeader(key) - return kv.isValid() && b2s(h.hbuf.musttoken(kv.value)) == value -} - -func (h *header) peekHeaderBytes(key string) []byte { - kv := h.peekHeader(key) - if kv.isValid() { - return h.hbuf.musttoken(kv.value) - } - return nil -} - -// peekHeader returns header key-value for the given key. -// -// The returned value is valid until the request is released, -// either though ReleaseRequest or your request handler returning. -// Do not store references to returned value. Make copies instead. -func (h *header) peekHeader(key string) argsKV { - hb := &h.hbuf - for i := 0; i < len(h.hbuf.headers); i++ { - if b2s(hb.musttoken(h.hbuf.headers[i].key)) == key { - return h.hbuf.headers[i] - } - } - return hb.noKV() -} - -func (h *header) peekPtrHeader(key string) *argsKV { - hb := &h.hbuf - for i := 0; i < len(h.hbuf.headers); i++ { - if b2s(hb.musttoken(h.hbuf.headers[i].key)) == key { - return &h.hbuf.headers[i] - } - } - return nil -} - -func (hb *headerBuf) mustAppendSlice(value string) headerSlice { - L := len(hb.buf) - copy(hb.buf[L:L+len(value)], value) - hb.buf = hb.buf[:L+len(value)] - return hb.slice(hb.buf[L : L+len(value)]) -} - -func (h *header) reuseOrAppend(tok headerSlice, value string) headerSlice { - if tok.len > tokint(len(value)) { - copy(h.hbuf.musttoken(tok), value) - tok.len = tokint(len(value)) - return tok - } - return h.appendSlice(value) -} - -func (h *header) appendSlice(value string) headerSlice { - free := h.hbuf.free() - if len(value) > free { - if h.flags.hasAny(flagNoBufferGrow) { - h.flags |= flagOOMReached - return headerSlice{} - } - h.hbuf.buf = slices.Grow(h.hbuf.buf, len(value)) - } - return h.hbuf.mustAppendSlice(value) -} - -func (h *header) appendHeader(key, value string) { - hb := &h.hbuf - free := hb.free() - buf := h.hbuf.buf - - if len(key)+len(value) > free { - if h.flags.hasAny(flagNoBufferGrow) { - panic(errSmallBuffer) - } - slices.Grow(buf, len(key)+len(value)) - } - k := hb.mustAppendSlice(key) - v := hb.mustAppendSlice(value) - if !h.flags.hasAny(disableNormalizing) { - // TODO - } - hb.headers = append(hb.headers, argsKV{ - key: k, - value: v, - }) -} - -func readRawHeaders(dst []byte, buf string) ([]byte, int, error) { - n := strings.IndexByte(buf, nChar) - if n < 0 { - return dst[:0], 0, errNeedMore - } - if (n == 1 && buf[0] == rChar) || n == 0 { - // empty headers - return dst, n + 1, nil - } - - n++ - b := buf - m := n - for { - b = b[m:] - m = strings.IndexByte(b, nChar) - if m < 0 { - return dst, 0, errNeedMore - } - m++ - n += m - if (m == 2 && b[0] == rChar) || m == 1 { - dst = append(dst, buf[:n]...) - return dst, n, nil - } - } -} -func (hb *headerBuf) noKV() argsKV { return argsKV{} } - -func (hb *headerBuf) nextKV2(ss *scannerState) argsKV { - if !ss.initialized { - ss.nextColon = -1 - ss.nextNewLine = -1 - } - buf := hb.buf[hb.off:] - blen := len(buf) - if blen >= 2 && buf[0] == '\r' && buf[1] == '\n' { - hb.off += 2 - return hb.noKV() // \r\n\r\n Ends header. - } else if blen >= 1 && buf[0] == '\n' { - hb.off += 1 - return hb.noKV() // \n\n Ends header. - } - - // n is parsing offset. Will start by storing colon index. - n := 0 - if ss.nextColon >= 0 { - // Retake from last colon found. - n = ss.nextColon - ss.nextColon = -1 - } else { - n = bytes.IndexByte(buf, ':') - x := bytes.IndexByte(buf, '\n') - if x < 0 { - // A header name should always at some point be followed by a \n - // even if it's the one that terminates the header block. - ss.err = errNeedMore - return hb.noKV() - } else if x < n { - // There was a \n before the colon! This is invalid. - ss.err = errInvalidName - return hb.noKV() - } else if n < 0 { - // No colon found, probably missing data. - ss.err = errNeedMore - return hb.noKV() - } - } - // n stores colon position by now. - if bytes.IndexByte(buf[:n], ' ') >= 0 || bytes.IndexByte(buf[:n], '\t') >= 0 { - // Spaces between the header key and colon are not allowed. - // See RFC 7230, Section 3.2.4. - ss.err = errInvalidName - return hb.noKV() - } - - // Ready to store key.. - var resultKV argsKV - resultKV.key = hb.slice(buf[:n]) - normalizeHeaderKey(buf[:n], ss.disableNormalizing) - n++ // consume colon. - for len(buf) > n && buf[n] == ' ' { - n++ // Trim leading spaces. - } - // n now points to start of value. - valueStart := n - - // Find end of value. Values may be multiline, in which case we must treat newlines followed by whitespace as part of the value. - for { - nl := bytes.IndexByte(buf[n:], '\n') - if nl < 0 || nl+n+1 == len(buf) { - // No newline or newline is last character and can't know if is multiline. - ss.err = errNeedMore - return hb.noKV() - } - n += nl + 1 // Index of the newly found newline. - nextChar := buf[n] - if nextChar != ' ' && nextChar != '\t' { - break // End of value found. - } - } - - valueEnd := n - 1 // Trim newline. - if valueEnd > valueStart && buf[valueEnd-1] == '\r' { - valueEnd-- // Trim \r character if present before value. - } - resultKV.value = hb.slice(buf[valueStart:valueEnd]) - hb.off += n - return resultKV -} - -func normalizeHeaderKey(b []byte, disableNormalizing bool) { - if disableNormalizing { - return - } - - n := len(b) - if n == 0 { - return - } - - b[0] = toUpperTable[b[0]] - for i := 1; i < n; i++ { - p := &b[i] - if *p == '-' { - i++ - if i < n { - b[i] = toUpperTable[b[i]] - } - continue - } - *p = toLowerTable[*p] - } -} - -func normalizeHeaderValue(ov, ob []byte, headerLength int) (nv, nb []byte, nhl int) { - nv = ov - length := len(ov) - if length <= 0 { - return - } - write := 0 - shrunk := 0 - lineStart := false - for read := 0; read < length; read++ { - c := ov[read] - switch { - case c == rChar || c == nChar: - shrunk++ - if c == nChar { - lineStart = true - } - continue - case lineStart && c == '\t': - c = ' ' - default: - lineStart = false - } - nv[write] = c - write++ - } - - nv = nv[:write] - copy(ob[write:], ob[write+shrunk:]) - - // Check if we need to skip \r\n or just \n - skip := 0 - if ob[write] == rChar { - if ob[write+1] == nChar { - skip += 2 - } else { - skip++ - } - } else if ob[write] == nChar { - skip++ - } - - nb = ob[write+skip : len(ob)-shrunk] - nhl = headerLength - shrunk - return -} - -func parseContentLength(b string) (int, error) { - v, n, err := parseUintBuf(b) - if err != nil { - return -1, fmt.Errorf("cannot parse Content-Length: %w", err) - } - if n != len(b) { - return -1, fmt.Errorf("cannot parse Content-Length: %w", errNonNumericChars) - } - return v, nil -} - -func nextLine(b []byte) ([]byte, []byte, error) { - nNext := bytes.IndexByte(b, nChar) - if nNext < 0 { - return nil, nil, errNeedMore - } - n := nNext - if n > 0 && b[n-1] == rChar { - n-- - } - return b[:n], b[nNext+1:], nil -} - -func stripSpace(b string) string { - for len(b) > 0 && b[0] == ' ' { - b = b[1:] - } - for len(b) > 0 && b[len(b)-1] == ' ' { - b = b[:len(b)-1] - } - return b -} - -var ( - errEmptyInt = errors.New("empty integer") - errUnexpectedFirstChar = errors.New("unexpected first char found. Expecting 0-9") - errUnexpectedTrailingChar = errors.New("unexpected trailing char found. Expecting 0-9") - errTooLongInt = errors.New("too long int") -) - -func parseUintBuf(b string) (int, int, error) { - n := len(b) - if n == 0 { - return -1, 0, errEmptyInt - } - v := 0 - for i := 0; i < n; i++ { - c := b[i] - k := c - '0' - if k > 9 { - if i == 0 { - return -1, i, errUnexpectedFirstChar - } - return v, i, nil - } - vNew := 10*v + int(k) - // Test for overflow. - if vNew < v { - return -1, i, errTooLongInt - } - v = vNew - } - return v, n, nil -} - -/* - -Request Parsing - -*/ - -// Read reads request header from r. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (h *header) Read(r *bufio.Reader) error { - return h.readLoop(r, true) -} - -// readLoop reads request header from r optionally loops until it has enough data. -// -// io.EOF is returned if r is closed before reading the first header byte. -func (h *header) readLoop(r *bufio.Reader, waitForMore bool) error { - n := 1 - for { - err := h.tryRead(r, n) - if err == nil { - return nil - } - if !waitForMore || err != errNeedMore { - h.resetSkipNormalize() - return err - } - n = r.Buffered() + 1 - } -} - -func (h *header) tryRead(r *bufio.Reader, n int) error { - h.resetSkipNormalize() - b, err := r.Peek(n) - - if len(b) == 0 { - if err == io.EOF { - return err - } - - if err == nil { - panic("bufio.Reader.Peek() returned nil, nil") - } - - // This is for go 1.6 bug. See https://github.com/golang/go/issues/14121 . - if err == bufio.ErrBufferFull { - return &ErrSmallBuffer{ - error: fmt.Errorf("error when reading request headers: %w (n=%d, r.Buffered()=%d)", errSmallBuffer, n, r.Buffered()), - } - } - - // n == 1 on the first read for the request. - if n == 1 { - // We didn't read a single byte. - return ErrNothingRead{err} - } - - return fmt.Errorf("error when reading request headers: %w", err) - } - b = mustPeekBuffered(r) - errParse := h.parse() - if errParse != nil { - return headerError("request", err, errParse, b, false) - } - // mustDiscard(r, headersLen) - return nil -} - -func (h *headerBuf) reset() { - *h = headerBuf{ - buf: h.buf[:0], - headers: h.headers[:0], - cookies: h.cookies[:0], - } -} - -func (h *header) resetSkipNormalize() { - h.hbuf.reset() - *h = header{ - hbuf: h.hbuf, - logger: h.logger, - } -} - -func headerError(typ string, err, errParse error, b []byte, secureErrorLogMessage bool) error { - if errParse != errNeedMore { - return headerErrorMsg(typ, errParse, b, secureErrorLogMessage) - } - if err == nil { - return errNeedMore - } - - // Buggy servers may leave trailing CRLFs after http body. - // Treat this case as EOF. - if isOnlyCRLF(b) { - return io.EOF - } - - if err != bufio.ErrBufferFull { - return headerErrorMsg(typ, err, b, secureErrorLogMessage) - } - return &ErrSmallBuffer{ - error: headerErrorMsg(typ, errSmallBuffer, b, secureErrorLogMessage), - } -} - -func isOnlyCRLF(b []byte) bool { - for _, ch := range b { - if ch != rChar && ch != nChar { - return false - } - } - return true -} - -func headerErrorMsg(typ string, err error, b []byte, secureErrorLogMessage bool) error { - return fmt.Errorf("error when reading %s headers: %w. Buffer size=%d", typ, err, len(b)) -} - -// ErrNothingRead is returned when a keep-alive connection is closed, -// either because the remote closed it or because of a read timeout. -type ErrNothingRead struct { - error -} - -// ErrSmallBuffer is returned when the provided buffer size is too small -// for reading request and/or response headers. -// -// ReadBufferSize value from Server or clients should reduce the number -// of such errors. -type ErrSmallBuffer struct { - error -} - -func mustPeekBuffered(r *bufio.Reader) []byte { - buf, err := r.Peek(r.Buffered()) - if len(buf) == 0 || err != nil { - panic(fmt.Sprintf("bufio.Reader.Peek() returned unexpected data (%q, %v)", buf, err)) - } - return buf -} - -func mustDiscard(r *bufio.Reader, n int) { - if _, err := r.Discard(n); err != nil { - panic(fmt.Sprintf("bufio.Reader.Discard(%d) failed: %v", n, err)) - } -} - -// Host returns Host header value. -func (h *header) Host() []byte { - return h.peekHeaderBytes(HeaderHost) -} - -// ConnectionClose returns true if 'Connection: close' header is set. -func (h *header) ConnectionClose() bool { - return h.flags.hasAny(connectionClose) -} - -// UserAgent returns User-Agent header value. -func (h *header) UserAgent() []byte { - return h.peekHeaderBytes(HeaderUserAgent) -} - -// b2s converts byte slice to a string without memory allocation. -// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . -func b2s(b []byte) string { - return unsafe.String(unsafe.SliceData(b), len(b)) -} - -// s2b converts string to a byte slice without memory allocation. -func s2b(s string) []byte { - return unsafe.Slice(unsafe.StringData(s), len(s)) -} diff --git a/httpx/header_test.go b/httpx/header_test.go deleted file mode 100644 index e16c310..0000000 --- a/httpx/header_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package httpx - -import ( - "bytes" - "net/http" - "strings" - "testing" -) - -func TestHeaderParseRequest(t *testing.T) { - const ( - wantMethod = "GET" - wantURI = "/" - wantMessage = "hello world!" - ) - req, err := http.NewRequest(wantMethod, wantURI, strings.NewReader(wantMessage)) - if err != nil { - t.Fatal(err) - } - var buf bytes.Buffer - req.Write(&buf) - var hdr header - err = hdr.ParseBytes(buf.Bytes()) - if err != nil { - t.Fatal(err) - } - if !hdr.MethodIs(wantMethod) { - t.Errorf("want method %s, got %q", wantMethod, hdr.Method()) - } - if !bytes.Equal(hdr.RequestURI(), []byte(wantURI)) { - t.Errorf("want request URI %q, got %q", wantURI, hdr.RequestURI()) - } -} diff --git a/httpx/tokenizer.go b/httpx/tokenizer.go deleted file mode 100644 index 0164a10..0000000 --- a/httpx/tokenizer.go +++ /dev/null @@ -1,337 +0,0 @@ -package httpx - -import ( - "errors" - "log/slog" - "net/http" - "strconv" - "unsafe" - - "github.com/soypat/lneto/internal" -) - -type headerBuf struct { - // buf[:len] holds entire HTTP header data, which may be normalized by [flags]. buf[off:len] holds data not yet processed during parsing. - buf []byte - // offset into buf for parsing. - off int - // args contains key-value store. - headers []argsKV - cookies []argsKV -} - -type tokint = uint16 - -type headerSlice struct { - start tokint - len tokint -} - -type argsKV struct { - key headerSlice - value headerSlice // value start >0 means value is present. -} - -func (kv argsKV) isValid() bool { - return kv.key.start > 0 -} - -func (kv *argsKV) invalidate() { - *kv = argsKV{} -} - -func (tb headerBuf) musttoken(slice headerSlice) []byte { - return tb.buf[slice.start : slice.start+slice.len] -} - -func (tb headerBuf) slice(b []byte) headerSlice { - base := uintptr(unsafe.Pointer(unsafe.SliceData(tb.buf))) - off := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - if off < base || off > base+uintptr(len(tb.buf)) { - panic("httpx: argument buffer does not alias header buffer") - } - return headerSlice{ - start: tokint(off - base), - len: tokint(len(b)), - } -} - -func (kv argsKV) HasValue() bool { return kv.value.start > 0 } - -type flags uint8 - -const ( - disableNormalizing flags = 1 << iota - disableSpecialHeader - noDefaultContentType - connectionClose - noHTTP11 - cookiesCollected - flagNoBufferGrow - flagOOMReached -) - -func (f flags) hasAny(checkThese flags) bool { - return f&checkThese != 0 -} - -type header struct { - hbuf headerBuf - logger *slog.Logger - contentLength int - - method headerSlice - requestURI headerSlice - proto headerSlice - - flags flags -} - -func (h *header) ParseBytes(b []byte) error { - h.resetSkipNormalize() - h.hbuf.readFromBytes(b) - return h.parse() -} - -func (h *header) Set(key, value string) { - h.SetCanonical(key, value) //TODO: implement non-canonical. -} - -func (h *header) Add(key, value string) { - h.appendHeader(key, value) -} - -// ContentType returns Content-Type header value. -func (h *header) ContentType() []byte { - return h.peekHeaderBytes(HeaderContentType) -} - -// SetCanonical sets the given 'key: value' header assuming that -// key is in canonical form. -// -// If the header is set as a Trailer (forbidden trailers will not be set, see SetTrailer for more details), -// it will be sent after the chunked request body. -func (h *header) SetCanonical(key, value string) { - kv := h.peekPtrHeader(key) - if kv != nil { - kv.invalidate() - } - h.appendHeader(key, value) -} - -// SetHost sets Host header value. -func (h *header) SetHost(host string) { - h.Set(HeaderHost, host) -} - -// SetUserAgent sets User-Agent header value. -func (h *header) SetUserAgent(userAgent string) { - h.Set(HeaderUserAgent, userAgent) -} - -// SetConnectionClose sets 'Connection: close' header. -func (h *header) SetConnectionClose() { - h.flags |= connectionClose -} - -// ResetConnectionClose clears 'Connection: close' header if it exists. -func (h *header) ResetConnectionClose() { - if h.flags.hasAny(connectionClose) { - h.flags &^= connectionClose - // h.h = delAllArgs(h.h, strConnection) // TODO - } -} - -func appendUint(b []byte, v int) []byte { - if v < 0 { - panic("negative uint") - } - return strconv.AppendUint(b, uint64(v), 10) -} - -// ContentLength returns Content-Length header value. -// -// It may be negative: -// -1 means Transfer-Encoding: chunked. -// -2 means Transfer-Encoding: identity. -func (h *header) ContentLength() int { - return h.contentLength -} - -var ErrBadTrailer = errors.New("contain forbidden trailer") - -// DisableNormalizing disables header names' normalization. -// -// By default all the header names are normalized by uppercasing -// the first letter and all the first letters following dashes, -// while lowercasing all the other letters. -// Examples: -// -// - CONNECTION -> Connection -// - conteNT-tYPE -> Content-Type -// - foo-bar-baz -> Foo-Bar-Baz -// -// Disable header names' normalization only if know what are you doing. -func (h *header) DisableNormalizing() { - h.flags |= disableNormalizing -} - -// Method returns HTTP request method. -func (h *header) Method() []byte { - return h.hbuf.musttoken(h.method) -} - -func (h *header) SetMethod(method string) { - h.method = h.reuseOrAppend(h.method, method) -} - -// SetRequestURI sets RequestURI for the first HTTP request line. -func (h *header) SetRequestURI(requestURI string) { - h.requestURI = h.reuseOrAppend(h.requestURI, requestURI) -} - -// RequestURI returns RequestURI from the first HTTP request line. -func (h *header) RequestURI() []byte { - if h.requestURI.start == 0 { - return nil - } else if h.requestURI.len == 0 { - h.requestURI = h.appendSlice("/") - } - return h.hbuf.musttoken(h.requestURI) -} - -// Protocol returns HTTP protocol. -func (h *header) Protocol() []byte { - if h.proto.len == 0 { - h.proto = h.appendSlice(strHTTP11) - } - return h.hbuf.musttoken(h.proto) -} - -func (h *header) SetProtocol(protocol string) { - h.proto = h.reuseOrAppend(h.proto, protocol) -} - -// AppendReqRespCommon appends request/response common header representation to dst and returns the extended buffer. -func (h *header) AppendReqRespCommon(dst []byte) []byte { - for i, n := 0, len(h.hbuf.headers); i < n; i++ { - kv := &h.hbuf.headers[i] - if kv.isValid() { - key := h.hbuf.musttoken(kv.key) - value := h.hbuf.musttoken(kv.value) - dst = appendHeaderLine(dst, b2s(key), b2s(value)) - } - } - - // if len(h.trailer) > 0 { - // aux := appendArgsKey(nil, h.trailer, strCommaSpace) - // dst = appendHeaderLine(dst, strTrailer, b2s(aux)) - // } - - // there is no need in h.collectCookies() here, since if cookies aren't collected yet, - // they all are located in h.h. - n := len(h.hbuf.cookies) - if n > 0 && !h.flags.hasAny(disableSpecialHeader) { - dst = append(dst, strCookie...) - dst = append(dst, strColonSpace...) - h.hbuf.appendRequestCookieBytes(dst) - dst = append(dst, strCRLF...) - } - - if h.ConnectionClose() && !h.flags.hasAny(disableSpecialHeader) { - dst = appendHeaderLine(dst, strConnection, strClose) - } - - return append(dst, strCRLF...) -} - -func appendHeaderLine(dst []byte, key, value string) []byte { - dst = append(dst, key...) - dst = append(dst, strColonSpace...) - dst = append(dst, value...) - return append(dst, strCRLF...) -} - -func (h *header) ignoreBody() bool { - return h.IsGet() || h.IsHead() -} - -func (h *header) collectCookies() { - if h.flags.hasAny(cookiesCollected) { - return - } - n := len(h.hbuf.headers) - for i := 0; i < n; i++ { - kv := h.hbuf.headers[i] - if kv.isValid() && caseInsensitiveCompare(b2s(h.hbuf.musttoken(kv.key)), HeaderCookie) { - cookie := h.hbuf.musttoken(kv.value) - for len(cookie) > 0 { - key, value, n := parseCookie(cookie) - h.hbuf.cookies = append(h.hbuf.cookies, argsKV{ - key: h.hbuf.slice(key), - value: h.hbuf.slice(value), - }) - cookie = cookie[n:] - } - } - } - h.flags |= cookiesCollected -} - -func (h *header) parseReqCookie(value []byte) { - -} - -func (h *header) MethodIs(method string) bool { - return b2s(h.Method()) == method -} - -// IsGet returns true if request method is GET. -func (h *header) IsGet() bool { return h.method.len == 0 || h.MethodIs(http.MethodGet) } - -// IsHead returns true if request method is HEAD. -func (h *header) IsHead() bool { return h.MethodIs(http.MethodHead) } - -// IsPost returns true if request method is POST. -func (h *header) IsPost() bool { return h.MethodIs(http.MethodPost) } - -// IsPut returns true if request method is PUT. -func (h *header) IsPut() bool { return h.MethodIs(http.MethodPut) } - -// IsDelete returns true if request method is DELETE. -func (h *header) IsDelete() bool { return h.MethodIs(http.MethodDelete) } - -// IsConnect returns true if request method is CONNECT. -func (h *header) IsConnect() bool { return h.MethodIs(http.MethodConnect) } - -// IsOptions returns true if request method is OPTIONS. -func (h *header) IsOptions() bool { return h.MethodIs(http.MethodOptions) } - -// IsTrace returns true if request method is TRACE. -func (h *header) IsTrace() bool { return h.MethodIs(http.MethodTrace) } - -// IsPatch returns true if request method is PATCH. -func (h *header) IsPatch() bool { return h.MethodIs(http.MethodPatch) } - -// IsHTTP11 returns true if the request is HTTP/1.1. -func (h *header) IsHTTP11() bool { return !h.flags.hasAny(noHTTP11) } - -// Embed this type into a struct, which mustn't be copied, -// so `go vet` gives a warning if this struct is copied. -// -// See https://github.com/golang/go/issues/8005#issuecomment-190753527 for details. -// and also: https://stackoverflow.com/questions/52494458/nocopy-minimal-example -type noCopy struct{} - -func (*noCopy) Lock() {} -func (*noCopy) Unlock() {} - -func (h *header) trace(msg string, attrs ...slog.Attr) { - internal.LogAttrs(h.logger, internal.LevelTrace, msg, attrs...) -} -func (h *header) debug(msg string, attrs ...slog.Attr) { - internal.LogAttrs(h.logger, slog.LevelDebug, msg, attrs...) -} -func (h *header) info(msg string, attrs ...slog.Attr) { - internal.LogAttrs(h.logger, slog.LevelInfo, msg, attrs...) -} From c3f0c218f753d7d4f0e254fdea0675b175e55f99 Mon Sep 17 00:00:00 2001 From: soypat Date: Sun, 25 May 2025 17:01:31 -0300 Subject: [PATCH 5/8] add httpraw to readme --- README.md | 2 +- http/httpraw/header.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b14a2e0..6265824 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Userspace networking primitives. ## Packages - `lneto`: Low-level Networking Operations, or "El Neto", the networking package. Zero copy network frame marshalling and unmarshalling. - [`lneto/frames.go`](./frames.go): Ethernet, IPv4/IPv6, ARP, TCP, UDP packet marshalling/unmarshalling. - +- [`lneto/http/httpraw`](./http/httpraw/): Heapless HTTP header processing and validation. Does no implement header normalization. - [`lneto/tcp`](./ntp): TCP implementation and low level logic. - [`lneto/dhcpv4`](./dhcpv4): DHCP version 4 protocol implementation and low level logic. - [`lneto/dns`](./dns): DNS protocol implementation and low level logic. diff --git a/http/httpraw/header.go b/http/httpraw/header.go index a174c19..392da47 100644 --- a/http/httpraw/header.go +++ b/http/httpraw/header.go @@ -34,7 +34,7 @@ func (f flags) hasAny(checkThese flags) bool { // // It does NOT implement: // - Normalization. -// - Cookies. +// - Cookies (see [Cookie]). // - Special header optimizations. // - Safe API. Users can easily mangle HTTP body with calls. type Header struct { From 1cc1882648f194f0fff1454989e10b13d9150824 Mon Sep 17 00:00:00 2001 From: soypat Date: Sun, 25 May 2025 17:05:20 -0300 Subject: [PATCH 6/8] bump to go1.22 so vulncheck passes --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d8dd8d7..411d062 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/soypat/lneto -go 1.21 +go 1.22 From ebbc8e3ba48d6262226b2061c0407fd2bd3c0d9a Mon Sep 17 00:00:00 2001 From: soypat Date: Sun, 25 May 2025 17:09:18 -0300 Subject: [PATCH 7/8] ok needed to bump further than previously thought --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 411d062..1e4a7d1 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/soypat/lneto -go 1.22 +go 1.23.8 From 969243d480bbe50c990f16acbf2cf286fbd08cd1 Mon Sep 17 00:00:00 2001 From: soypat Date: Sun, 25 May 2025 17:10:49 -0300 Subject: [PATCH 8/8] return error when no cookies found --- http/httpraw/cookie.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/httpraw/cookie.go b/http/httpraw/cookie.go index 091d3b0..d16b746 100644 --- a/http/httpraw/cookie.go +++ b/http/httpraw/cookie.go @@ -63,6 +63,9 @@ func (c *Cookie) Parse() error { }) off += n } + if len(c.kvs) == 0 { + return errNoCookies + } return nil }