diff --git a/README.md b/README.md index 40e9f04..65db113 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Userspace networking primitives. - Zero scheduling required. No goroutines/channels use in Lneto. Can be run in event loop. - Heapless packet processing - [`httpraw`](https://github.com/soypat/lneto/tree/main/http/httpraw) is likely the most performant HTTP/1.1 processing package in the Go ecosystem. Based on [`fasthttp`](https://github.com/valyala/fasthttp) but simpler and more thoughtful memory use. + - [`httφ`](https://github.com/soypat/lneto/tree/main/http/httphi) - Heapless HTTP router and zero-copy response writing with Go's standard library API. - Lean memory footprint - HTTP header struct is 80 bytes with no runtime usage nor heap usage other than buffer - Entire Ethernet+IPv4+UDP+DHCP+DNS+NTP stack in ~2kB RAM. diff --git a/http/httphi/httphi.go b/http/httphi/httphi.go new file mode 100644 index 0000000..9414a32 --- /dev/null +++ b/http/httphi/httphi.go @@ -0,0 +1,486 @@ +package httphi + +import ( + "io" + "log/slog" + "strconv" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/soypat/lneto" + "github.com/soypat/lneto/http/httpraw" + "github.com/soypat/lneto/internal" +) + +//go:generate stringer -type Method,status -linecomment -output stringers.go + +type conn = io.ReadWriteCloser + +// Router hosts concurrent safe data. +type Router struct { + mu sync.Mutex + gen atomic.Uint32 + numGoro int + reqBuf int + respBuf int + pendingConns chan job + mux Mux + + globbuf []byte + exchs []Exchange + freeList *Exchange + + backoff lneto.BackoffStrategy + log *slog.Logger +} + +type job struct { + exch *Exchange +} + +type Mux interface { + Handler(get Method, uri []byte) HandlerFunc +} + +type RouterConfig struct { + // FixedNumGoroutines must be set to either -1 (freely allocate new goroutines) or to the number of goroutines + // to spawn on [Router.Configure] being called. + FixedNumGoroutines int + // RequestBufferSize determines the buffer allocated + // for processing requests. + RequestBufferSize int + // ResponseMinBufferSize determines buffer allocated for processing responses. + // Response buffer will reuse unused request memory so this is not a strict limit. + ResponseMinBufferSize int + + NormalizeOutgoingKeys bool + MaxAwaitingConns int + Backoff lneto.BackoffStrategy + Mux Mux + Logger *slog.Logger +} + +func (cfg RouterConfig) Validate() error { + workerMode := cfg.workerMode() + if workerMode && cfg.MaxAwaitingConns == 0 || + cfg.Mux == nil || + !workerMode && cfg.FixedNumGoroutines != -1 { + return lneto.ErrInvalidConfig + } else if cfg.Backoff == nil { + return lneto.ErrMissingHALConfig + } + return nil +} + +func (cfg RouterConfig) workerMode() bool { + return cfg.FixedNumGoroutines > 0 +} + +// Teardown stops fixed goroutines. +func (r *Router) TeardownGoroutines() { + r.gen.Add(1) + if r.pendingConns != nil { + close(r.pendingConns) + } +} + +func (r *Router) Configure(cfg RouterConfig) error { + if err := cfg.Validate(); err != nil { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + r.TeardownGoroutines() + gen := r.gen.Load() + numgoro := cfg.FixedNumGoroutines + workerMode := cfg.workerMode() + r.reqBuf = cfg.RequestBufferSize + r.respBuf = cfg.ResponseMinBufferSize + r.mux = cfg.Mux + if !workerMode { + r.backoff = cfg.Backoff + r.numGoro = 0 + r.pendingConns = nil + return nil + } + if workerMode { + jobqueue := make(chan job, cfg.MaxAwaitingConns) + if gen > 1 { + // Previously existing goroutine manager, wait a bit for it to close. + time.Sleep(5 * time.Millisecond) + } + internal.SliceReuse(&r.exchs, numgoro) + r.exchs = r.exchs[:numgoro] + rawBuflen := cfg.RequestBufferSize + cfg.ResponseMinBufferSize + internal.SliceReuse(&r.globbuf, numgoro*rawBuflen) + for i := range numgoro { + // TODO exchange buffer alloc + goff := i * rawBuflen + r.exchs[i].Configure(r.globbuf[goff:goff+rawBuflen], cfg.RequestBufferSize, cfg.NormalizeOutgoingKeys) + go r.goroWorker(gen, jobqueue, cfg.Backoff, cfg.Mux) + } + r.pendingConns = jobqueue + r.numGoro = numgoro + } + return nil +} + +func (r *Router) Handle(conn conn) error { + r.mu.Lock() + defer r.mu.Unlock() + exch := r.getExch(conn) + if exch == nil { + return lneto.ErrExhausted + } + + if r.numGoro == 0 { + go r.goroHandle(exch, r.backoff, r.mux) + return nil + } + select { + case r.pendingConns <- job{exch: exch}: + default: + exch.used.Store(false) // release. + } + return lneto.ErrPacketDrop +} + +func (r *Router) goroWorker(gen uint32, queue chan job, backoff lneto.BackoffStrategy, mux Mux) { + for job := range queue { + exch := job.exch + if gen != r.gen.Load() { + return + } else if exch == nil { + panic("httplo: unreachable nil job") + } + r.goroHandle(exch, backoff, mux) + } +} + +func (r *Router) goroHandle(exch *Exchange, backoff lneto.BackoffStrategy, mux Mux) { + defer r.freeExch(exch) + reqhdr := &exch.reqHdr + reqhdr.Reset(nil) + var consecutiveBackoffs uint + for { + n, err := reqhdr.ReadFromLimited(exch.rw, reqhdr.BufferFree()) + if err != nil { + r.error("goroHandle:ReadFromLimited", slog.String("err", err.Error())) + exch.rw.Close() + return + } else if n == 0 { + backoff(consecutiveBackoffs) + consecutiveBackoffs++ + continue + } + consecutiveBackoffs = 0 + const asRequest = false + needMore, err := reqhdr.TryParse(asRequest) + if !needMore && err == nil { + // Done! + break + } else if err != nil { + r.error("goroHandle:TryParse", slog.String("err", err.Error())) + exch.rw.Close() + return + } + } + r.info("goroHandle:headerParsedSuccess") + // Setup Exchange fields necessary for correct functioning. + parsed := reqhdr.BufferParsed() + exch.respRemains = reqhdr.BufferReceived() - parsed + exch.respHeaderOff = uint16(parsed) + exch.respHeaderLen = 0 + // Mux URI. + uri := reqhdr.RequestURI() + meth := reqhdr.Method() + handler := mux.Handler(MethodFromBytes(meth), uri) + if handler != nil { + handler(exch) + } + // Reuse request space as response header start. + + // TODO write response from exchange here. + exch.rw.Close() + +} + +func (r *Router) freeExch(exch *Exchange) { + const freelistMaxDepth = 5 + r.mu.Lock() + if r.freeList == nil { + r.freeList = exch + } else { + node := r.freeList + depth := 0 + for depth < freelistMaxDepth && node.nextFree != nil { + node = node.nextFree + depth++ + } + node.nextFree = exch + } + exch.Release() + r.mu.Unlock() +} + +func (r *Router) getExch(conn conn) (exch *Exchange) { + r.mu.Lock() + defer r.mu.Unlock() + if r.freeList != nil { + if r.freeList.Acquire(conn) { + exch = r.freeList + } + r.freeList = r.freeList.nextFree + } + if exch != nil { + return exch + } + for i := range r.exchs { + if r.exchs[i].Acquire(conn) { + return &r.exchs[i] + } + } + + if r.numGoro == 0 { + exch := new(Exchange) + exch.Configure(make([]byte, r.respBuf+r.reqBuf), r.reqBuf, false) + + return exch + } + return nil +} + +func (r *Router) allocBuffers(exch *Exchange) { + +} + +func (r *Router) error(msg string, attrs ...slog.Attr) { + internal.LogAttrs(r.log, slog.LevelError, msg, attrs...) +} + +func (r *Router) info(msg string, attrs ...slog.Attr) { + internal.LogAttrs(r.log, slog.LevelInfo, msg, attrs...) +} + +type Exchange struct { + used atomic.Bool + respTopBuf [32]byte + respTopWritten uint8 + + rawbuf []byte + respHeaderOff uint16 + respHeaderLen uint16 + reqHdr httpraw.Header + rw conn + + respRemains int + headerWritten bool + normalizeKeys bool + nextFree *Exchange +} + +func (exch *Exchange) Configure(rawbuf []byte, requestLim int, normalizeKeys bool) { + respSize := len(rawbuf) - requestLim + if respSize < 0 { + panic("request lim larger than buffer") + } + exch.reqHdr.Reset(rawbuf[:0:requestLim]) + exch.normalizeKeys = normalizeKeys +} + +func (exch *Exchange) Acquire(conn conn) bool { + if !exch.used.CompareAndSwap(false, true) { + return false + } + exch.respTopWritten = 0 + exch.respHeaderOff = 0 + exch.respHeaderLen = 0 + exch.respRemains = 0 + exch.rw = conn + exch.headerWritten = false + exch.nextFree = nil + exch.reqHdr.Reset(nil) + return true +} + +func (exch *Exchange) Release() { + exch.rw = nil + exch.used.Store(false) +} + +func (exch *Exchange) SetHeader(key, value string) (enoughMemory bool) { + off := int(exch.respHeaderOff) + int(exch.respHeaderLen) + free := len(exch.rawbuf) - off + if len(key)+len(value)+2 > free { + return false + } + n := copy(exch.rawbuf[off:], key) + if exch.normalizeKeys { + httpraw.NormalizeHeaderKey(exch.rawbuf[off : off+n]) + } + exch.rawbuf[n] = ':' + n++ + n += copy(exch.rawbuf[off+n:], value) + exch.rawbuf[off+n] = '\r' + exch.rawbuf[off+n+1] = '\n' + n += 2 + exch.respHeaderLen += uint16(n) + return true +} + +func (exch *Exchange) StageWriteStatus(code int) { + if code >= 1000 || exch.headerWritten { + return + } else if code == 200 { + // Common case. + exch.respTopWritten = uint8(copy(exch.respTopBuf[:], "HTTP/1.1 200 OK\r\n")) + return + } + n := copy(exch.respTopBuf[:], "HTTP/1.1 ") + n += len(strconv.AppendInt(exch.respTopBuf[n:n], int64(code), 10)) + text := StatusText(code) + exch.respTopBuf[n] = ' ' + n++ + n += copy(exch.respTopBuf[n:], text) + exch.respTopBuf[n] = '\r' + exch.respTopBuf[n+1] = '\n' + exch.respTopWritten = uint8(n) +} + +func (exch *Exchange) WriteHeader(code int) { + if !exch.headerWritten { + exch.StageWriteStatus(code) + exch.FlushHeader() + } +} +func (exch *Exchange) FlushHeader() (int, error) { + if exch.headerWritten { + return 0, nil + } + if exch.respTopWritten == 0 { + exch.StageWriteStatus(200) + } + exch.headerWritten = true + ng, err := exch.rw.Write(exch.respTopBuf[:exch.respTopWritten]) + if err != nil { + return ng, err + } + off := int(exch.respHeaderOff) + headers := exch.rawbuf[off : off+int(exch.respHeaderLen)+2] + headers[len(headers)-1] = '\n' + headers[len(headers)-2] = '\r' + ng2, err := exch.rw.Write(headers) + return ng + ng2, err +} + +func (exch *Exchange) Write(buf []byte) (int, error) { + if !exch.headerWritten { + exch.FlushHeader() + } + if len(buf) == 0 { + return 0, nil + } + return exch.rw.Write(buf) +} + +func (exch *Exchange) RequestHeaderRaw() *httpraw.Header { + return &exch.reqHdr +} + +func (exch *Exchange) RequestParseCookie(dst *httpraw.Cookie, key string) error { + value := exch.RequestHeader(key) + return dst.ParseBytes(value) +} + +func (exch *Exchange) RequestHeader(key string) []byte { + header := exch.RequestHeaderRaw() + return header.Get(key) +} + +func (exch *Exchange) RequestURI() []byte { + return exch.RequestHeaderRaw().RequestURI() +} + +func (exch *Exchange) RequestMethod() []byte { + return exch.RequestHeaderRaw().Method() +} + +func (exch *Exchange) RequestConnectionClose() bool { + return exch.RequestHeaderRaw().ConnectionClose() +} + +func (exch *Exchange) ReadBody(dst []byte) (n int, _ error) { + if exch.respRemains > 0 { + _, err := exch.reqHdr.Body() + if err != nil { + return 0, err // Returns mangled buffer error if request header has been misused. + } + surplus := exch.rawbuf[exch.reqHdr.BufferParsed():exch.reqHdr.BufferReceived()] + toRead := surplus[len(surplus)-exch.respRemains:] + n = copy(dst, toRead) + exch.respRemains -= n + if len(dst) == n { + return n, nil + } + dst = dst[n:] + } + nr, err := exch.rw.Read(dst) + return nr + n, err +} + +type HandlerFunc func(ex *Exchange) + +type Method uint8 + +const ( + MethUndefined Method = iota // undefined + MethGet // GET + // lol. + MethHead // HEAD + MethPost // POST + MethPut // PUT + // RFC 5789 + MethPatch // PATCH + MethDelete // DELETE + MethConnect // CONNECT + MethOptions // OPTIONS + MethTrace // TRACE + MethUnknown // unknown +) + +func MethodFromBytes(meth []byte) (res Method) { + if len(meth) == 0 { + return MethUndefined + } + switch unsafe.String(&meth[0], len(meth)) { + case "GET": + res = MethGet + case "HEAD": + res = MethHead + case "POST": + res = MethPost + case "PUT": + res = MethPut + case "PATCH": + res = MethPatch + case "DELETE": + res = MethDelete + case "CONNECT": + res = MethConnect + case "OPTIONS": + res = MethOptions + case "TRACE": + res = MethTrace + default: + res = MethUnknown + } + return res +} + +// b2s converts byte slice to a string without memory allocation. +// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . +func b2s(b []byte) string { + return unsafe.String(unsafe.SliceData(b), len(b)) +} diff --git a/http/httphi/status.go b/http/httphi/status.go new file mode 100644 index 0000000..237dbab --- /dev/null +++ b/http/httphi/status.go @@ -0,0 +1,271 @@ +package httphi + +// StatusText returns a text for the HTTP status code. It returns the empty +// string if the code is unknown. +func StatusText(code int) string { + switch status(code) { + case StatusContinue: + return "Continue" + case StatusSwitchingProtocols: + return "Switching Protocols" + case StatusProcessing: + return "Processing" + case StatusEarlyHints: + return "Early Hints" + case StatusOK: + return "OK" + case StatusCreated: + return "Created" + case StatusAccepted: + return "Accepted" + case StatusNonAuthoritativeInfo: + return "Non-Authoritative Information" + case StatusNoContent: + return "No Content" + case StatusResetContent: + return "Reset Content" + case StatusPartialContent: + return "Partial Content" + case StatusMultiStatus: + return "Multi-Status" + case StatusAlreadyReported: + return "Already Reported" + case StatusIMUsed: + return "IM Used" + case StatusMultipleChoices: + return "Multiple Choices" + case StatusMovedPermanently: + return "Moved Permanently" + case StatusFound: + return "Found" + case StatusSeeOther: + return "See Other" + case StatusNotModified: + return "Not Modified" + case StatusUseProxy: + return "Use Proxy" + case StatusTemporaryRedirect: + return "Temporary Redirect" + case StatusPermanentRedirect: + return "Permanent Redirect" + case StatusBadRequest: + return "Bad Request" + case StatusUnauthorized: + return "Unauthorized" + case StatusPaymentRequired: + return "Payment Required" + case StatusForbidden: + return "Forbidden" + case StatusNotFound: + return "Not Found" + case StatusMethodNotAllowed: + return "Method Not Allowed" + case StatusNotAcceptable: + return "Not Acceptable" + case StatusProxyAuthRequired: + return "Proxy Authentication Required" + case StatusRequestTimeout: + return "Request Timeout" + case StatusConflict: + return "Conflict" + case StatusGone: + return "Gone" + case StatusLengthRequired: + return "Length Required" + case StatusPreconditionFailed: + return "Precondition Failed" + case StatusRequestEntityTooLarge: + return "Request Entity Too Large" + case StatusRequestURITooLong: + return "Request URI Too Long" + case StatusUnsupportedMediaType: + return "Unsupported Media Type" + case StatusRequestedRangeNotSatisfiable: + return "Requested Range Not Satisfiable" + case StatusExpectationFailed: + return "Expectation Failed" + case StatusTeapot: + return "I'm a teapot" + case StatusMisdirectedRequest: + return "Misdirected Request" + case StatusUnprocessableEntity: + return "Unprocessable Entity" + case StatusLocked: + return "Locked" + case StatusFailedDependency: + return "Failed Dependency" + case StatusTooEarly: + return "Too Early" + case StatusUpgradeRequired: + return "Upgrade Required" + case StatusPreconditionRequired: + return "Precondition Required" + case StatusTooManyRequests: + return "Too Many Requests" + case StatusRequestHeaderFieldsTooLarge: + return "Request Header Fields Too Large" + case StatusUnavailableForLegalReasons: + return "Unavailable For Legal Reasons" + case StatusInternalServerError: + return "Internal Server Error" + case StatusNotImplemented: + return "Not Implemented" + case StatusBadGateway: + return "Bad Gateway" + case StatusServiceUnavailable: + return "Service Unavailable" + case StatusGatewayTimeout: + return "Gateway Timeout" + case StatusHTTPVersionNotSupported: + return "HTTP Version Not Supported" + case StatusVariantAlsoNegotiates: + return "Variant Also Negotiates" + case StatusInsufficientStorage: + return "Insufficient Storage" + case StatusLoopDetected: + return "Loop Detected" + case StatusNotExtended: + return "Not Extended" + case StatusNetworkAuthenticationRequired: + return "Network Authentication Required" + default: + return "" + } +} + +type status int + +// HTTP status codes as registered with IANA. +// See: https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml +const ( + // RFC 9110, 15.2.1 + StatusContinue status = 100 // Continue + // RFC 9110, 15.2.2 + StatusSwitchingProtocols status = 101 // Switching Protocols + // RFC 2518, 10.1 + StatusProcessing status = 102 // Processing + // RFC 8297 + StatusEarlyHints status = 103 // Early Hints + + // RFC 9110, 15.3.1 + StatusOK status = 200 // OK + // RFC 9110, 15.3.2 + StatusCreated status = 201 // Created + // RFC 9110, 15.3.3 + StatusAccepted status = 202 // Accepted + // RFC 9110, 15.3.4 + StatusNonAuthoritativeInfo status = 203 // Non-Authoritative Information + // RFC 9110, 15.3.5 + StatusNoContent status = 204 // No Content + // RFC 9110, 15.3.6 + StatusResetContent status = 205 // Reset Content + // RFC 9110, 15.3.7 + StatusPartialContent status = 206 // Partial Content + // RFC 4918, 11.1 + StatusMultiStatus status = 207 // Multi-Status + // RFC 5842, 7.1 + StatusAlreadyReported status = 208 // Already Reported + // RFC 3229, 10.4.1 + StatusIMUsed status = 226 // IM Used + + // RFC 9110, 15.4.1 + StatusMultipleChoices status = 300 // Multiple Choices + // RFC 9110, 15.4.2 + StatusMovedPermanently status = 301 // Moved Permanently + // RFC 9110, 15.4.3 + StatusFound status = 302 // Found + // RFC 9110, 15.4.4 + StatusSeeOther status = 303 // See Other + // RFC 9110, 15.4.5 + StatusNotModified status = 304 // Not Modified + // RFC 9110, 15.4.6 + StatusUseProxy status = 305 // Use Proxy + // RFC 9110, 15.4.7 (Unused) + _ status = 306 + // RFC 9110, 15.4.8 + StatusTemporaryRedirect status = 307 // Temporary Redirect + // RFC 9110, 15.4.9 + StatusPermanentRedirect status = 308 // Permanent Redirect + + // RFC 9110, 15.5.1 + StatusBadRequest status = 400 // Bad Request + // RFC 9110, 15.5.2 + StatusUnauthorized status = 401 // Unauthorized + // RFC 9110, 15.5.3 + StatusPaymentRequired status = 402 // Payment Required + // RFC 9110, 15.5.4 + StatusForbidden status = 403 // Forbidden + // RFC 9110, 15.5.5 + StatusNotFound status = 404 // Not Found + // RFC 9110, 15.5.6 + StatusMethodNotAllowed status = 405 // Method Not Allowed + // RFC 9110, 15.5.7 + StatusNotAcceptable status = 406 // Not Acceptable + // RFC 9110, 15.5.8 + StatusProxyAuthRequired status = 407 // Proxy Authentication Required + // RFC 9110, 15.5.9 + StatusRequestTimeout status = 408 // Request Timeout + // RFC 9110, 15.5.10 + StatusConflict status = 409 // Conflict + // RFC 9110, 15.5.11 + StatusGone status = 410 // Gone + // RFC 9110, 15.5.12 + StatusLengthRequired status = 411 // Length Required + // RFC 9110, 15.5.13 + StatusPreconditionFailed status = 412 // Precondition Failed + // RFC 9110, 15.5.14 + StatusRequestEntityTooLarge status = 413 // Request Entity Too Large + // RFC 9110, 15.5.15 + StatusRequestURITooLong status = 414 // Request URI Too Long + // RFC 9110, 15.5.16 + StatusUnsupportedMediaType status = 415 // Unsupported Media Type + // RFC 9110, 15.5.17 + StatusRequestedRangeNotSatisfiable status = 416 // Requested Range Not Satisfiable + // RFC 9110, 15.5.18 + StatusExpectationFailed status = 417 // Expectation Failed + // RFC 9110, 15.5.19 (Unused) + StatusTeapot status = 418 // I'm a teapot + // RFC 9110, 15.5.20 + StatusMisdirectedRequest status = 421 // Misdirected Request + // RFC 9110, 15.5.21 + StatusUnprocessableEntity status = 422 // Unprocessable Entity + // RFC 4918, 11.3 + StatusLocked status = 423 // Locked + // RFC 4918, 11.4 + StatusFailedDependency status = 424 // Failed Dependency + // RFC 8470, 5.2. + StatusTooEarly status = 425 // Too Early + // RFC 9110, 15.5.22 + StatusUpgradeRequired status = 426 // Upgrade Required + // RFC 6585, 3 + StatusPreconditionRequired status = 428 // Precondition Required + // RFC 6585, 4 + StatusTooManyRequests status = 429 // Too Many Requests + // RFC 6585, 5 + StatusRequestHeaderFieldsTooLarge status = 431 // Request Header Fields Too Large + // RFC 7725, 3 + StatusUnavailableForLegalReasons status = 451 // Unavailable For Legal Reasons + + // RFC 9110, 15.6.1 + StatusInternalServerError status = 500 // Internal Server Error + // RFC 9110, 15.6.2 + StatusNotImplemented status = 501 // Not Implemented + // RFC 9110, 15.6.3 + StatusBadGateway status = 502 // Bad Gateway + // RFC 9110, 15.6.4 + StatusServiceUnavailable status = 503 // Service Unavailable + // RFC 9110, 15.6.5 + StatusGatewayTimeout status = 504 // Gateway Timeout + // RFC 9110, 15.6.6 + StatusHTTPVersionNotSupported status = 505 // HTTP Version Not Supported + // RFC 2295, 8.1 + StatusVariantAlsoNegotiates status = 506 // Variant Also Negotiates + // RFC 4918, 11.5 + StatusInsufficientStorage status = 507 // Insufficient Storage + // RFC 5842, 7.2 + StatusLoopDetected status = 508 // Loop Detected + // RFC 2774, 7 + StatusNotExtended status = 510 // Not Extended + // RFC 6585, 6 + StatusNetworkAuthenticationRequired status = 511 // Network Authentication Required +) diff --git a/http/httpraw/header.go b/http/httpraw/header.go index d92ba68..1e09760 100644 --- a/http/httpraw/header.go +++ b/http/httpraw/header.go @@ -15,19 +15,21 @@ const ( strClose = "close" ) -type flags uint16 +type Flags uint16 const ( - flagNoBufferGrow flags = 1 << iota + flagNoBufferGrow Flags = 1 << iota flagDoneParsingHeader flagOOMReached flagConnClose flagNoHTTP11 flagMangledBuffer // set when header fields appended to buffer via Add,Set calls flagReaderEOF + // set if [Header.SetStatus] or [Header.SetStatusInt] has been called. + FlagStatusSet ) -func (f flags) hasAny(checkThese flags) bool { +func (f Flags) HasAny(checkThese Flags) bool { return f&checkThese != 0 } @@ -50,10 +52,13 @@ type Header struct { statusCode headerSlice statusText headerSlice - flags flags + flags Flags _ noCopy } +// Flags returns [Flags] to signal status code has been set, Connection:Close or other useful signals provided by flags. +func (h *Header) Flags() Flags { return h.flags } + // EnableBufferGrowth disables buffer growth during parsing if b is false. Is enabled by default. // Disabling buffer growth prevents allocations but methods may throw errors on insufficient memory. func (h *Header) EnableBufferGrowth(b bool) { @@ -97,9 +102,9 @@ func (h *Header) Parse(asResponse bool) error { // return err // } func (h *Header) TryParse(asResponse bool) (needMoreData bool, err error) { - if h.flags.hasAny(flagDoneParsingHeader) { + if h.flags.HasAny(flagDoneParsingHeader) { return false, errAlreadyParsed - } else if h.flags.hasAny(flagMangledBuffer) { + } else if h.flags.HasAny(flagMangledBuffer) { return false, errMangledBuffer } if asResponse && h.statusCode.len == 0 || !asResponse && h.requestURI.start == 0 { @@ -114,7 +119,7 @@ func (h *Header) TryParse(asResponse bool) (needMoreData bool, err error) { // ParsingSuccess returns true if TryParse was successful, that is to say it returned needMoreData==false and err==nil. func (h *Header) ParsingSuccess() bool { - return h.flags.hasAny(flagDoneParsingHeader) + return h.flags.HasAny(flagDoneParsingHeader) } // ReadFromLimited reads at most maxBytesToRead from reader and appends them to underlying buffer. @@ -123,18 +128,23 @@ func (h *Header) ParsingSuccess() bool { func (h *Header) ReadFromLimited(r io.Reader, maxBytesToRead int) (int, error) { if maxBytesToRead <= 0 { return 0, errSmallBuffer - } else if h.flags.hasAny(flagMangledBuffer) { + } else if h.flags.HasAny(flagMangledBuffer) { return 0, errMangledBuffer + } else if h.flags.HasAny(flagReaderEOF) { + return 0, io.EOF // Now we do return EOF. } free := h.BufferFree() if free < maxBytesToRead { - if h.flags.hasAny(flagNoBufferGrow) { + if h.flags.HasAny(flagNoBufferGrow) { return 0, errSmallBuffer } h.hbuf.buf = slices.Grow(h.hbuf.buf, maxBytesToRead) } blen := len(h.hbuf.buf) b := h.hbuf.buf[blen:min(blen+maxBytesToRead, cap(h.hbuf.buf))] + if len(b) == 0 { + return 0, errSmallBuffer + } n, err := r.Read(b) if err != nil && err == io.EOF { h.flags |= flagReaderEOF @@ -154,7 +164,7 @@ func (h *Header) ReadFromBytes(b []byte) (int, error) { } free := h.BufferFree() if free < len(b) { - if h.flags.hasAny(flagNoBufferGrow) { + if h.flags.HasAny(flagNoBufferGrow) { return 0, errSmallBuffer } h.hbuf.buf = slices.Grow(h.hbuf.buf, len(b)) @@ -166,7 +176,7 @@ func (h *Header) ReadFromBytes(b []byte) (int, error) { // BufferReceived returns the amoung of bytes read during calls to Read* methods. // Returns 0 if buffer is invalid/mangled. func (h *Header) BufferReceived() int { - if h.flags.hasAny(flagMangledBuffer | flagOOMReached) { + if h.flags.HasAny(flagMangledBuffer | flagOOMReached) { return 0 } return len(h.hbuf.buf) @@ -176,7 +186,7 @@ func (h *Header) BufferReceived() int { // If the Parse* method completed without error then BufferParsed returns the header's length including the final "\r\n\r\n" text. // BufferParsed returns 0 if the buffer is invalid/mangled or if no header data has been parsed succesfully. func (h *Header) BufferParsed() int { - if h.flags.hasAny(flagMangledBuffer | flagOOMReached) { + if h.flags.HasAny(flagMangledBuffer | flagOOMReached) { return 0 } return h.hbuf.off @@ -234,7 +244,7 @@ func (hb *headerBuf) forEach(cb func(key, value []byte) error) error { // h.Reset(httpHeader); h.Parse() // Parse bytes in place with no copying. // h.Reset(nil) // Reuse buffer previously set in a call to Reset. func (h *Header) Reset(buf []byte) { - if h.flags.hasAny(flagNoBufferGrow) && cap(buf) < 32 { + if h.flags.HasAny(flagNoBufferGrow) && cap(buf) < 32 { panic("small buffer and flagNoBufferGrow set") } const persistentFlags = flagNoBufferGrow @@ -250,9 +260,9 @@ func (h *Header) Reset(buf []byte) { // Body returns the surplus data following headers. It is only valid as long as Parse* or Reset methods are not called. func (h *Header) Body() ([]byte, error) { debuglog("http:body") - if h.flags.hasAny(flagMangledBuffer) { + if h.flags.HasAny(flagMangledBuffer) { return nil, errMangledBuffer - } else if h.flags.hasAny(flagDoneParsingHeader) { + } else if h.flags.HasAny(flagDoneParsingHeader) { return h.hbuf.buf[h.hbuf.off:], nil } return nil, errUnparsed @@ -368,12 +378,20 @@ func (h *Header) Status() (code, statusText []byte) { return h.hbuf.musttoken(h.statusCode), h.hbuf.musttoken(h.statusText) } -// Status sets the response header's status code and status text. i.e: "200" "OK". +// SetStatus sets the response header's status code and status text. i.e: "200" "OK". func (h *Header) SetStatus(code, statusText string) { + h.flags |= FlagStatusSet h.statusCode = h.reuseOrAppend(h.statusCode, code) h.statusText = h.reuseOrAppend(h.statusText, statusText) } +// SetStatusInt is identical to [Header.SetStatus] but performs integer to text conversion for status code. +func (h *Header) SetStatusInt(code int64, statusText string) { + h.flags |= FlagStatusSet + h.statusCode = h.reuseOrAppendInt(h.statusCode, code, 10) + h.statusText = h.reuseOrAppend(h.statusText, statusText) +} + func (h *Header) getNonEmptyValue(s headerSlice) []byte { if s.len == 0 { return nil // If empty then value is invalid, return nil. @@ -384,7 +402,7 @@ func (h *Header) getNonEmptyValue(s headerSlice) []byte { // AppendRequest appends the request header representation to the buffer and returns the result. func (h *Header) AppendRequest(dst []byte) ([]byte, error) { proto := h.Protocol() - if h.flags.hasAny(flagOOMReached) { + if h.flags.HasAny(flagOOMReached) { return dst, errOOM } else if h.requestURI.len == 0 || h.method.len == 0 { return dst, errNeedMethodURI @@ -413,8 +431,18 @@ func (h *Header) AppendRequest(dst []byte) ([]byte, error) { // AppendResponse appends the response header representation to the buffer and returns the result. func (h *Header) AppendResponse(dst []byte) ([]byte, error) { + dst, err := h.AppendResponseNoHeaders(dst) + if err != nil { + return dst, err + } + dst = h.AppendHeaders(dst) + return append(dst, strCRLF...), nil +} + +// AppendResponseNoHeaders appends the first line of the response containing protocol and status code/text: i.e: "HTTP/1.1 200 OK\r\n" +func (h *Header) AppendResponseNoHeaders(dst []byte) ([]byte, error) { proto := h.Protocol() - if h.flags.hasAny(flagOOMReached) { + if h.flags.HasAny(flagOOMReached) { return dst, errOOM } else if h.statusCode.len == 0 || h.statusText.len == 0 { return dst, errBadStatusCodeTxt @@ -429,10 +457,7 @@ func (h *Header) AppendResponse(dst []byte) ([]byte, error) { dst = append(dst, ' ') dst = append(dst, text...) dst = append(dst, strCRLF...) - - dst = h.AppendHeaders(dst) - - return append(dst, strCRLF...), nil + return dst, nil } // AppendHeaders appends headers to buffer. Use AppendRequest and AppendResponse over this. @@ -540,3 +565,63 @@ func CopyNormalizedHeaderValue(dst []byte, value []byte) (n int, modified bool) } return write, modified } + +// CopyDecodedPercentURL decodes percent-escapes in value into dst and returns bytes written. +// n < len(value) implies percent-escapes were decoded; the converse does not hold since +// '+' substitution preserves length. If plusAsSpace is set '+' decodes to ' ', +// which is correct for query and form-encoded data but NOT for path segments. +// On malformed escape returns n bytes written before the fault and a non-nil error. +// dst and value may only alias if &dst[0] == &value[0]. +func CopyDecodedPercentURL(dst, value []byte, plusAsSpace bool) (n int, err error) { + if len(dst) < len(value) { + panic("httpraw.CopyDecodedPercentURL: dst buffer shorter than value") + } + read := 0 + for { + escape := bytes.IndexByte(value[read:], '%') + if escape < 0 { + n += copyPlusDecoded(dst[n:], value[read:], plusAsSpace) + return n, nil + } + escape += read + n += copyPlusDecoded(dst[n:], value[read:escape], plusAsSpace) + if escape+2 >= len(value) { + return n, errBadPercentEncode // Truncated escape at end of value. + } + hi, okhi := unhexdigit(value[escape+1]) + lo, oklo := unhexdigit(value[escape+2]) + if !okhi || !oklo { + return n, errBadPercentEncode + } + // Write index n is always <= escape since decoding shrinks 3 bytes to 1, + // so writing here never clobbers an unread byte when dst aliases value. + dst[n] = hi<<4 | lo + n++ + read = escape + 3 + } +} + +// copyPlusDecoded copies src to dst replacing '+' with ' ' if plusAsSpace set. +func copyPlusDecoded(dst, src []byte, plusAsSpace bool) int { + n := copy(dst, src) + if plusAsSpace { + for i := 0; i < n; i++ { + if dst[i] == '+' { + dst[i] = ' ' + } + } + } + return n +} + +func unhexdigit(c byte) (byte, bool) { + switch { + case c >= '0' && c <= '9': + return c - '0', true + case c >= 'a' && c <= 'f': + return c - 'a' + 10, true + case c >= 'A' && c <= 'F': + return c - 'A' + 10, true + } + return 0, false +} diff --git a/http/httpraw/header_test.go b/http/httpraw/header_test.go index 2d94955..1e8f3ec 100644 --- a/http/httpraw/header_test.go +++ b/http/httpraw/header_test.go @@ -209,6 +209,67 @@ func TestCopyNormalizedHeaderValue(t *testing.T) { } } +func TestCopyDecodedPercentURL(t *testing.T) { + var tests = []struct { + value string + plusAsSpace bool + want string + wantErr bool + }{ + {value: "", want: ""}, + {value: "/plain/path", want: "/plain/path"}, + {value: "/a%20b", want: "/a b"}, + {value: "%41%42%43", want: "ABC"}, + {value: "%2f%2F", want: "//"}, // lower and upper case hex digits. + {value: "%25", want: "%"}, // escaped percent must not re-trigger decoding. + {value: "%2525", want: "%25"}, // decoded output is not re-scanned. + {value: "a+b", want: "a+b"}, // plus is literal in path segments. + {value: "a+b", plusAsSpace: true, want: "a b"}, + {value: "%20+%20", plusAsSpace: true, want: " "}, + {value: "/x?q=%E2%82%AC", want: "/x?q=\xe2\x82\xac"}, // multi-byte UTF-8 sequence. + // Malformed escapes must error, never pass through silently. + {value: "%zz", want: "", wantErr: true}, + {value: "ok%4", want: "ok", wantErr: true}, // truncated escape at end. + {value: "ok%", want: "ok", wantErr: true}, // bare percent at end. + {value: "a%2gb", want: "a", wantErr: true}, // second digit not hex. + {value: "a%g2b", want: "a", wantErr: true}, // first digit not hex. + } + dst := make([]byte, 256) + for _, test := range tests { + value := []byte(test.value) + n, err := CopyDecodedPercentURL(dst[:len(value)], value, test.plusAsSpace) + if test.wantErr && err == nil { + t.Errorf("%q: want error, got nil", test.value) + } else if !test.wantErr && err != nil { + t.Errorf("%q: unexpected error %s", test.value, err) + } + if got := string(dst[:n]); got != test.want { + t.Errorf("%q: want %q got %q", test.value, test.want, got) + } + // n' ' substitution preserves length. + if !test.wantErr && n < len(test.value) && test.want == test.value { + t.Errorf("%q: n=%d signals decoding but value unchanged", test.value, n) + } + if !test.wantErr && !test.plusAsSpace && test.want != test.value && n >= len(test.value) { + t.Errorf("%q: n=%d does not signal decoding of %q", test.value, n, test.want) + } + } +} + +// In-place decoding (dst aliasing value at offset 0) must yield the same result. +func TestCopyDecodedPercentURLInPlace(t *testing.T) { + const value, want = "/a%20b%2Fc%25", "/a b/c%" + buf := []byte(value) + n, err := CopyDecodedPercentURL(buf, buf, false) + if err != nil { + t.Fatal(err) + } + if got := string(buf[:n]); got != want { + t.Fatalf("want %q got %q", want, got) + } +} + func TestHeaderSetOverwrite(t *testing.T) { var h Header h.Reset(nil) diff --git a/http/httpraw/parse.go b/http/httpraw/parse.go index 82f4027..3b7ba7e 100644 --- a/http/httpraw/parse.go +++ b/http/httpraw/parse.go @@ -29,6 +29,7 @@ var ( errBadStatusCodeTxt = errors.New("invalid status code or text") errCookiesParsed = errors.New("cookies already parsed, reset before parsing again") errBufferTooLarge = errors.New("httpraw: buffer exceeds max size (offsets are uint16)") + errBadPercentEncode = errors.New("httpraw: invalid percent-encoding in URL") ) // maxBufLen bounds the header buffer. Offsets/lengths are stored as uint16 @@ -172,7 +173,7 @@ func (hb *headerBuf) scanUntilByte(c byte) []byte { return buf } -func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto headerSlice, flags flags, err error) { +func (hb *headerBuf) parseFirstLineRequest(initFlags Flags) (method, uri, proto headerSlice, flags Flags, err error) { debuglog("http:req:scan") hb.off = 0 // Parsing first line resets offset. hb.skipLeadingCRLF() @@ -206,7 +207,7 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto return method, uri, proto, flags, nil } -func (hb *headerBuf) parseFirstLineResponse(initFlags flags) (statusCode, statusText headerSlice, flags flags, err error) { +func (hb *headerBuf) parseFirstLineResponse(initFlags Flags) (statusCode, statusText headerSlice, flags Flags, err error) { debuglog("http:resp:scan") hb.off = 0 // Parsing first line resets offset. hb.skipLeadingCRLF() @@ -369,7 +370,7 @@ func (h *Header) reserve(need int) bool { return false } if need > hb.free() { - if h.flags.hasAny(flagNoBufferGrow) { + if h.flags.HasAny(flagNoBufferGrow) { h.flags |= flagOOMReached return false } @@ -519,9 +520,9 @@ func (hb *headerBuf) next(ss *scannerState) argsKV { // ConnectionClose returns true if 'Connection: close' header is set or if a invalid header was found. func (h *Header) ConnectionClose() bool { - closed := h.flags.hasAny(flagConnClose) || + closed := h.flags.HasAny(flagConnClose) || h.hasHeaderValue(headerConnection, strClose) || - (h.flags.hasAny(flagNoHTTP11) && !h.hasHeaderValue(headerConnection, "keep-alive")) + (h.flags.HasAny(flagNoHTTP11) && !h.hasHeaderValue(headerConnection, "keep-alive")) if closed { h.flags |= flagConnClose }