diff --git a/README.md b/README.md index 40e9f04..65db113 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Userspace networking primitives. - Zero scheduling required. No goroutines/channels use in Lneto. Can be run in event loop. - Heapless packet processing - [`httpraw`](https://github.com/soypat/lneto/tree/main/http/httpraw) is likely the most performant HTTP/1.1 processing package in the Go ecosystem. Based on [`fasthttp`](https://github.com/valyala/fasthttp) but simpler and more thoughtful memory use. + - [`httφ`](https://github.com/soypat/lneto/tree/main/http/httphi) - Heapless HTTP router and zero-copy response writing with Go's standard library API. - Lean memory footprint - HTTP header struct is 80 bytes with no runtime usage nor heap usage other than buffer - Entire Ethernet+IPv4+UDP+DHCP+DNS+NTP stack in ~2kB RAM. diff --git a/arp/stringers.go b/arp/stringers.go index cd8d0c7..376de7a 100644 --- a/arp/stringers.go +++ b/arp/stringers.go @@ -17,9 +17,9 @@ const _Operation_name = "requestreply" var _Operation_index = [...]uint8{0, 7, 12} func (i Operation) String() string { - i -= 1 - if i >= Operation(len(_Operation_index)-1) { - return "Operation(" + strconv.FormatInt(int64(i+1), 10) + ")" + idx := int(i) - 1 + if i < 1 || idx >= len(_Operation_index)-1 { + return "Operation(" + strconv.FormatInt(int64(i), 10) + ")" } - return _Operation_name[_Operation_index[i]:_Operation_index[i+1]] + return _Operation_name[_Operation_index[idx]:_Operation_index[idx+1]] } diff --git a/dhcp/dhcpv4/stringers.go b/dhcp/dhcpv4/stringers.go index 2141e9e..8dcce7b 100644 --- a/dhcp/dhcpv4/stringers.go +++ b/dhcp/dhcpv4/stringers.go @@ -106,10 +106,11 @@ const _Op_name = "undefinedrequestreply" var _Op_index = [...]uint8{0, 9, 16, 21} func (i Op) String() string { - if i >= Op(len(_Op_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_Op_index)-1 { return "Op(" + strconv.FormatInt(int64(i), 10) + ")" } - return _Op_name[_Op_index[i]:_Op_index[i+1]] + return _Op_name[_Op_index[idx]:_Op_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. @@ -131,10 +132,11 @@ const _MessageType_name = "undefineddiscoverofferrequestdeclineacknakreleaseinfo var _MessageType_index = [...]uint8{0, 9, 17, 22, 29, 36, 39, 42, 49, 55} func (i MessageType) String() string { - if i >= MessageType(len(_MessageType_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_MessageType_index)-1 { return "MessageType(" + strconv.FormatInt(int64(i), 10) + ")" } - return _MessageType_name[_MessageType_index[i]:_MessageType_index[i+1]] + return _MessageType_name[_MessageType_index[idx]:_MessageType_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. @@ -155,9 +157,9 @@ const _ClientState_name = "initselectingrequestingboundrenewingrebindinginit-reb var _ClientState_index = [...]uint8{0, 4, 13, 23, 28, 36, 45, 56, 65} func (i ClientState) String() string { - i -= 1 - if i >= ClientState(len(_ClientState_index)-1) { - return "ClientState(" + strconv.FormatInt(int64(i+1), 10) + ")" + idx := int(i) - 1 + if i < 1 || idx >= len(_ClientState_index)-1 { + return "ClientState(" + strconv.FormatInt(int64(i), 10) + ")" } - return _ClientState_name[_ClientState_index[i]:_ClientState_index[i+1]] + return _ClientState_name[_ClientState_index[idx]:_ClientState_index[idx+1]] } diff --git a/dhcp/dhcpv6/stringers.go b/dhcp/dhcpv6/stringers.go index c7c29bc..0145ead 100644 --- a/dhcp/dhcpv6/stringers.go +++ b/dhcp/dhcpv6/stringers.go @@ -28,11 +28,11 @@ const _MsgType_name = "solicitadvertiserequestconfirmrenewrebindreplyreleasedecl var _MsgType_index = [...]uint8{0, 7, 16, 23, 30, 35, 41, 46, 53, 60, 71, 85, 95, 105} func (i MsgType) String() string { - i -= 1 - if i >= MsgType(len(_MsgType_index)-1) { - return "MsgType(" + strconv.FormatInt(int64(i+1), 10) + ")" + idx := int(i) - 1 + if i < 1 || idx >= len(_MsgType_index)-1 { + return "MsgType(" + strconv.FormatInt(int64(i), 10) + ")" } - return _MsgType_name[_MsgType_index[i]:_MsgType_index[i+1]] + return _MsgType_name[_MsgType_index[idx]:_MsgType_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. @@ -51,11 +51,11 @@ const _ClientState_name = "initsolicitingrequestingboundrenewingrebinding" var _ClientState_index = [...]uint8{0, 4, 14, 24, 29, 37, 46} func (i ClientState) String() string { - i -= 1 - if i >= ClientState(len(_ClientState_index)-1) { - return "ClientState(" + strconv.FormatInt(int64(i+1), 10) + ")" + idx := int(i) - 1 + if i < 1 || idx >= len(_ClientState_index)-1 { + return "ClientState(" + strconv.FormatInt(int64(i), 10) + ")" } - return _ClientState_name[_ClientState_index[i]:_ClientState_index[i+1]] + return _ClientState_name[_ClientState_index[idx]:_ClientState_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. @@ -134,10 +134,11 @@ const _StatusCode_name = "successunspec-failno-addrs-availno-bindingnot-on-linku var _StatusCode_index = [...]uint8{0, 7, 18, 32, 42, 53, 66} func (i StatusCode) String() string { - if i >= StatusCode(len(_StatusCode_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_StatusCode_index)-1 { return "StatusCode(" + strconv.FormatInt(int64(i), 10) + ")" } - return _StatusCode_name[_StatusCode_index[i]:_StatusCode_index[i+1]] + return _StatusCode_name[_StatusCode_index[idx]:_StatusCode_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. @@ -153,9 +154,9 @@ const _DUIDType_name = "duid-lltduid-enduid-ll" var _DUIDType_index = [...]uint8{0, 8, 15, 22} func (i DUIDType) String() string { - i -= 1 - if i >= DUIDType(len(_DUIDType_index)-1) { - return "DUIDType(" + strconv.FormatInt(int64(i+1), 10) + ")" + idx := int(i) - 1 + if i < 1 || idx >= len(_DUIDType_index)-1 { + return "DUIDType(" + strconv.FormatInt(int64(i), 10) + ")" } - return _DUIDType_name[_DUIDType_index[i]:_DUIDType_index[i+1]] + return _DUIDType_name[_DUIDType_index[idx]:_DUIDType_index[idx+1]] } diff --git a/dns/stringers.go b/dns/stringers.go index f30e888..42f7947 100644 --- a/dns/stringers.go +++ b/dns/stringers.go @@ -119,10 +119,11 @@ const _RCode_name = "successformat errorserver failurename errornot implementedr var _RCode_index = [...]uint8{0, 7, 19, 33, 43, 58, 65} func (i RCode) String() string { - if i >= RCode(len(_RCode_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_RCode_index)-1 { return "RCode(" + strconv.FormatInt(int64(i), 10) + ")" } - return _RCode_name[_RCode_index[i]:_RCode_index[i+1]] + return _RCode_name[_RCode_index[idx]:_RCode_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. @@ -138,8 +139,9 @@ const _OpCode_name = "Standard queryInverse queryServer status request" var _OpCode_index = [...]uint8{0, 14, 27, 48} func (i OpCode) String() string { - if i >= OpCode(len(_OpCode_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_OpCode_index)-1 { return "OpCode(" + strconv.FormatInt(int64(i), 10) + ")" } - return _OpCode_name[_OpCode_index[i]:_OpCode_index[i+1]] + return _OpCode_name[_OpCode_index[idx]:_OpCode_index[idx+1]] } diff --git a/examples/berkeley-listener/berkeley_server.go b/examples/berkeley-listener/berkeley_server.go index f3cee5f..310dffb 100644 --- a/examples/berkeley-listener/berkeley_server.go +++ b/examples/berkeley-listener/berkeley_server.go @@ -288,7 +288,7 @@ func handleConnNet(conn net.Conn) error { } } method := string(hdr.Method()) - uri := string(hdr.RequestURI()) + uri := string(hdr.RequestTarget()) fmt.Printf("< %s %s\n", method, uri) var resp httpraw.Header @@ -370,7 +370,7 @@ func mockClient(stack *xnet.StackAsync, port uint16, subnet netip.Prefix) { var hdr httpraw.Header hdr.SetMethod("GET") - hdr.SetRequestURI("/") + hdr.SetRequestTarget("/") hdr.SetProtocol("HTTP/1.1") hdr.Set("Host", string(ipv4.AppendFormatAddr(nil, stack.Addr4()))) hdr.Set("User-Agent", "lneto-mock") diff --git a/examples/http-linux/conn-linux.go b/examples/http-linux/conn-linux.go deleted file mode 100644 index 9da6143..0000000 --- a/examples/http-linux/conn-linux.go +++ /dev/null @@ -1,96 +0,0 @@ -//go:build !tinygo && linux - -package main - -import ( - "net/netip" - "syscall" -) - -// Conn wraps an accepted TCP connection from a raw Linux socket file descriptor. -// It implements io.Reader/io.Writer/io.Closer over syscall.Read/Write/Close. -type Conn struct { - fd int - remote netip.AddrPort -} - -// Read reads bytes from the connection into b. -func (c *Conn) Read(b []byte) (int, error) { - if len(b) == 0 { - return 0, nil - } - n, err := syscall.Read(c.fd, b) - if err != nil { - return 0, err - } - if n == 0 { - return 0, syscall.ECONNRESET // Peer closed. - } - return n, nil -} - -// Write writes b to the connection, looping until all bytes are sent. -func (c *Conn) Write(b []byte) (int, error) { - total := 0 - for total < len(b) { - n, err := syscall.Write(c.fd, b[total:]) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -// Close closes the underlying file descriptor. -func (c *Conn) Close() error { - return syscall.Close(c.fd) -} - -// RemoteAddr returns the peer address of the connection. -func (c *Conn) RemoteAddr() netip.AddrPort { return c.remote } - -// Listener wraps a listening TCP socket bound to a local port. -type Listener struct { - fd int -} - -// Listen creates a listening TCP socket bound to port on all interfaces. -func Listen(port uint16) (*Listener, error) { - fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) - if err != nil { - return nil, err - } - // Allow quick rebind after restart. - if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil { - syscall.Close(fd) - return nil, err - } - addr := &syscall.SockaddrInet4{Port: int(port)} - if err = syscall.Bind(fd, addr); err != nil { - syscall.Close(fd) - return nil, err - } - if err = syscall.Listen(fd, syscall.SOMAXCONN); err != nil { - syscall.Close(fd) - return nil, err - } - return &Listener{fd: fd}, nil -} - -// Accept blocks until an incoming connection arrives and returns it as a Conn. -func (l *Listener) Accept(conn *Conn) error { - nfd, sa, err := syscall.Accept(l.fd) - if err != nil { - return err - } - - conn.fd = nfd - if sa4, ok := sa.(*syscall.SockaddrInet4); ok { - conn.remote = netip.AddrPortFrom(netip.AddrFrom4(sa4.Addr), uint16(sa4.Port)) - } - return nil -} - -// Close closes the listening socket. -func (l *Listener) Close() error { return syscall.Close(l.fd) } diff --git a/examples/http-linux/main-httplinux.go b/examples/http-linux/main-httplinux.go index b243fea..a4548d9 100644 --- a/examples/http-linux/main-httplinux.go +++ b/examples/http-linux/main-httplinux.go @@ -3,42 +3,77 @@ package main import ( + "log/slog" + "net" "os" "strconv" "sync/atomic" "time" - "github.com/soypat/lneto/http/httpraw" + "github.com/soypat/lneto/http/httphi" ) -const listenPort = 8080 +const ( + kB = 1 << 10 + listenPort = 8080 + bufferSizes = 2 * kB + // A browser sends around twenty header fields; a request carrying more + // than this is answered 431 rather than parsed into memory it was not + // given. Each field costs 8 bytes of table. + numHeaderFields = 32 + numGoroutines = 4 + readTimeout = 2 * time.Second +) func main() { if err := run(); err != nil { - println("Error: ", err) + println("Error:", err.Error()) os.Exit(1) } println("DONE") } func run() error { - ln, err := Listen(listenPort) + ln, err := net.Listen("tcp", ":"+strconv.Itoa(listenPort)) if err != nil { return err } defer ln.Close() - println("listening on port", listenPort) - conn := new(Conn) + print("listening on http://localhost:", listenPort, "\n") + + var mux httphi.MuxSlice + mux.Handle("GET /", homepage) + + var router httphi.Router + err = router.Configure(httphi.RouterConfig{ + FixedNumGoroutines: numGoroutines, + RequestHeaderBufferSize: bufferSizes, + RequestNumHeaderKVCap: numHeaderFields, + ResponseHeaderMinBufferSize: bufferSizes, + MaxAwaitingConns: 256, + Mux: &mux, + Logger: slog.Default(), + }) + if err != nil { + return err + } + defer router.Shutdown() + for { - err := ln.Accept(conn) + conn, err := ln.Accept() if err != nil { return err } - visits.Add(1) - if err := handle(conn); err != nil { - println("handle:", conn.RemoteAddr().String(), err.Error()) + // The connection owns the idle policy: a peer that opens a socket and + // then stalls fails its read instead of holding a router goroutine. + conn.SetReadDeadline(time.Now().Add(readTimeout)) + err = router.Handle(conn) + if err != nil { + // Every goroutine is busy and the queue is full. Dropping the + // connection is the backpressure: memory stays bounded. + slog.Warn("dropped connection", slog.String("remote", conn.RemoteAddr().String()), slog.String("err", err.Error())) + conn.Close() } - conn.Close() } } @@ -50,58 +85,22 @@ const ( htmlTail = `!` + `Sign my guestbook!` + `
Best viewed in Netscape Navigator` + // maxPage bounds the rendered page: both halves plus the visitor number. + maxPage = len(htmlHead) + 20 + len(htmlTail) ) -const maxHTTPHeader = 1024 +// visits counts served requests. Handlers run on the router's goroutines, so +// every visitor gets their own number. +var visits atomic.Uint64 -var ( - hdr httpraw.Header - httpbuf [maxHTTPHeader]byte - htmlbuf [512]byte - visits atomic.Uint64 -) +func homepage(exch *httphi.Exchange) { + var page [maxPage]byte + n := copy(page[:], htmlHead) + n += len(strconv.AppendUint(page[n:n], visits.Add(1), 10)) + n += copy(page[n:], htmlTail) -func handle(conn *Conn) error { - hdr.Reset(httpbuf[:0]) - hdr.EnableBufferGrowth(false) // Limit memory to buffer capacity. - const incomingIsResponse = false // We get HTTP requests from clients. - deadline := time.Now().Add(50 * time.Millisecond) - for time.Until(deadline) > 0 { - if _, err := hdr.ReadFromLimited(conn, maxHTTPHeader); err != nil { - return err - } - needmoredata, err := hdr.TryParse(incomingIsResponse) - if err != nil { - return err - } - if needmoredata { - continue - } - break - } - println("\n\n================\n\n", hdr.String()) - if time.Since(deadline) > 0 { - print("DEADLINE EXCEED: ", hdr.BufferParsed(), "/", hdr.BufferReceived(), " bytes parsed/read\n") - return nil - } - // Prepare tacky HTML response. - n := copy(htmlbuf[:], htmlHead) - n += len(strconv.AppendUint(htmlbuf[n:n], visits.Load(), 10)) - n += copy(htmlbuf[n:], htmlTail) - contentLen := n - hdr.Reset(httpbuf[:0]) - hdr.SetProtocol("HTTP/1.1") - hdr.SetStatus("200", "OK") - hdr.Set("Content-Type", "text/html") - hdr.SetInt("Content-Length", int64(contentLen), 10) - // Here we do some buffer juggling. We use remaining space - // of HTTP Header buffer to write the response that will be written over the wire. - respbuf := httpbuf[hdr.BufferUsed():] - header, err := hdr.AppendResponse(respbuf[:0]) - if err != nil { - return err - } - conn.Write(header) - _, err = conn.Write(htmlbuf[:contentLen]) - return err + exch.StageHeader("Content-Type", "text/html") + exch.StageHeaderInt("Content-Length", int64(n), 10) + exch.WriteHeader(int(httphi.StatusOK)) + exch.WriteBody(page[:n]) } diff --git a/examples/httpclient/main.go b/examples/httpclient/main.go index 66f0459..8fbd6be 100644 --- a/examples/httpclient/main.go +++ b/examples/httpclient/main.go @@ -27,7 +27,7 @@ func run() error { // Prepare GET request. var hdr httpraw.Header hdr.SetMethod("GET") - hdr.SetRequestURI("/") + hdr.SetRequestTarget("/") hdr.SetProtocol("HTTP/1.1") req, err := hdr.AppendRequest(nil) if err != nil { @@ -50,7 +50,7 @@ func run() error { }() fmt.Println(time.Now().Format("15:04:05.000"), "writing...") conn.Write(req) - hdr.Reset(nil) + hdr.Reset(nil, 0) // No preallocation. Both key/value and raw buffer will grow and allocate. var needMore bool = true for needMore { _, err = hdr.ReadFromLimited(conn, 1024) diff --git a/examples/httpserver/main.go b/examples/httpserver/main.go index 83dd463..7f55fff 100644 --- a/examples/httpserver/main.go +++ b/examples/httpserver/main.go @@ -282,7 +282,7 @@ func handleConnection(conn *tcp.Conn) error { } method := string(hdr.Method()) - uri := string(hdr.RequestURI()) + uri := string(hdr.RequestTarget()) fmt.Printf("< %s %s\n", method, uri) // Build response body. diff --git a/examples/xcurl/main.go b/examples/xcurl/main.go index 85cd425..000d991 100644 --- a/examples/xcurl/main.go +++ b/examples/xcurl/main.go @@ -307,7 +307,7 @@ func run() (err error) { timeHTTPCreate := timer("create HTTP GET request") var hdr httpraw.Header hdr.SetMethod("GET") - hdr.SetRequestURI("/") + hdr.SetRequestTarget("/") hdr.SetProtocol("HTTP/1.1") hdr.Set("Host", flagHostToResolve) hdr.Set("User-Agent", "lneto") diff --git a/http/httphi/README.md b/http/httphi/README.md new file mode 100644 index 0000000..1b6ac35 --- /dev/null +++ b/http/httphi/README.md @@ -0,0 +1,51 @@ +# httφ + +Heapless HTTP/1.1 router with `net/http`-shaped handlers. Benchmarked against `net/http` and friends at [**soypat/httpbench**](https://github.com/soypat/httpbench). + +## Why + +`net/http` allocates per request: `Request`, header map, response buffer, goroutine. Fine on a +server, fatal on a microcontroller. httφ pays that cost once, at `Router.Configure`: + +- Exchanges and goroutines are fixed there. Serving allocates nothing; memory does not grow with load. +- No free exchange means `Handle` refuses the connection unclosed, rather than allocating one more of everything. +- `Handle` takes an `io.ReadWriteCloser`, so the router runs over a raw socket, an [`lneto`](https://github.com/soypat/lneto) TCP stack or a test pipe. No listener, no OS, no clock. + +Parsing is [`httpraw`](../httpraw). + +## Example + +```go +var mux httphi.MuxSlice +mux.Handle("GET /", func(ex *httphi.Exchange) { + ex.WriteBody([]byte("hello world")) +}) + +var router httphi.Router +err := router.Configure(httphi.RouterConfig{ + FixedNumGoroutines: 4, // 4 workers, 4 exchanges, allocated here and never again. + MaxAwaitingConns: 8, // Queue depth. Full queue drops connections. + RequestHeaderBufferSize: 1024, + ResponseHeaderMinBufferSize: 32, // Shares the request buffer. + RequestNumHeaderKVCap: 32, + Backoff: func(uint) time.Duration { return time.Millisecond }, + Mux: &mux, +}) +if err != nil { + log.Fatal(err) +} +for { + conn, err := listener.Accept() // Accepting is the caller's job. + if err != nil { + log.Fatal(err) + } + if err = router.Handle(conn); err != nil { + conn.Close() // Refused: never blocks, never queues unboundedly. + } +} +``` + +`FixedNumGoroutines: -1` gives the unbounded flavor, a goroutine and an exchange per connection. + +Runnable server over raw Linux sockets, plus query, form and multipart handlers: +[`example_test.go`](./example_test.go). diff --git a/http/httphi/bench_test.go b/http/httphi/bench_test.go new file mode 100644 index 0000000..99fb33c --- /dev/null +++ b/http/httphi/bench_test.go @@ -0,0 +1,134 @@ +package httphi + +import ( + "io" + "testing" + + "github.com/soypat/lneto/http/httpraw" + "github.com/soypat/lneto/internal" +) + +// benchConn replays a fixed request and discards the response. It allocates +// nothing itself so benchmark alloc counts belong to the package under test. +type benchConn struct { + request string + read int + written int +} + +func (c *benchConn) rewind() { c.read, c.written = 0, 0 } + +func (c *benchConn) Read(b []byte) (int, error) { + if c.read >= len(c.request) { + return 0, io.EOF + } + n := copy(b, c.request[c.read:]) + c.read += n + return n, nil +} + +func (c *benchConn) Write(b []byte) (int, error) { + c.written += len(b) + return len(b), nil +} + +func (c *benchConn) Close() error { return nil } + +// benchBody is package level: converting a string literal to []byte inside the +// handler would allocate on every request and hide the router's own cost. +var benchBody = []byte("hello world") + +func benchExchange(b *testing.B, conn conn) *Exchange { + b.Helper() + const bufferSize = 1024 + const numHeaderCap = 2 + exch := new(Exchange) + exch.Configure(ExchangeConfig{ + RawBuf: make([]byte, 2*bufferSize), + RequestBufferLim: bufferSize, + NumHeaderKVCap: numHeaderCap, + }) + if !exch.Acquire(conn) { + b.Fatal("fresh exchange failed to acquire connection") + } + return exch +} + +// BenchmarkHandle measures a whole exchange: read, parse, mux and respond. +func BenchmarkHandle(b *testing.B) { + expect := []byte("123") + buf := make([]byte, 64) + for _, bb := range []struct { + name string + request string + handler HandlerFunc + }{ + { + name: "GETWithHeadersAndQuery", + request: "GET /?abc=123 HTTP/1.1\r\nHost: tinygo.org\r\nUser-Agent: bench\r\nAccept: */*\r\nConnection: close\r\n\r\n", + handler: func(ex *Exchange) { + ex.StageHeader("Content-Type", "text/plain") + ex.StageHeaderInt("Content-Length", int64(len(benchBody)), 10) + data, present := ex.RequestQueryAppend(buf[:0], "abc", true) + if !present || !internal.BytesEqual(data, expect) { + panic("invalid result") + } + ex.WriteBody(benchBody) + }, + }, + { + name: "NotFound", + request: "GET /nowhere HTTP/1.1\r\nHost: tinygo.org\r\n\r\n", + handler: nil, // Unregistered: exercises the 404 path. + }, + } { + b.Run(bb.name, func(b *testing.B) { + var mux MuxSlice + if bb.handler != nil { + mux.Handle("GET /", bb.handler) + } + conn := &benchConn{request: bb.request} + exch := benchExchange(b, conn) + b.ReportAllocs() + b.SetBytes(int64(len(bb.request))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + conn.rewind() + exch.Release() + exch.Acquire(conn) + Handle(exch, &mux, nopBackoff) + } + }) + } +} + +// benchForm is package level so the Form's pair slice is reused across requests, +// as a real handler holding one per goroutine would. +var benchForm httpraw.Form + +// BenchmarkRequestParseForm measures reading and parsing a urlencoded body into +// a buffer the caller owns. Nothing on the path may allocate. +func BenchmarkRequestParseForm(b *testing.B) { + const request = "POST /f HTTP/1.1\r\nHost: tinygo.org\r\n" + + "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 27\r\n\r\n" + + "user=gopher&msg=hello+world" + buf := make([]byte, 64) + var mux MuxSlice + mux.Handle("POST /f", func(ex *Exchange) { + err := ex.RequestParseForm(&benchForm, buf) + if err != nil || benchForm.Len() != 2 { + panic("invalid result") + } + }) + conn := &benchConn{request: request} + exch := benchExchange(b, conn) + b.ReportAllocs() + b.SetBytes(int64(len(request))) + b.ResetTimer() + for b.Loop() { + conn.rewind() + exch.Release() + exch.Acquire(conn) + Handle(exch, &mux, nopBackoff) + } +} diff --git a/http/httphi/example_test.go b/http/httphi/example_test.go new file mode 100644 index 0000000..1094469 --- /dev/null +++ b/http/httphi/example_test.go @@ -0,0 +1,110 @@ +package httphi_test + +import ( + "fmt" + "io" + "log" + "log/slog" + "net" + "os" + + "github.com/soypat/lneto/http/httphi" + "github.com/soypat/lneto/http/httpraw" +) + +// ExampleRouter_linux goes over how to setup a linux server using raw linux connections. +// See [ExampleMuxSlice_query_forms_multipart] on how to define handlers for common HTTP processing. +func ExampleRouter() { + // Chrome tends to send ~700 bytes on a typical landing page request. + const requestBuffer = 1024 + const numHeaderKV = requestBuffer / 32 // + var mux httphi.MuxSlice + mux.Handle("GET /", func(ex *httphi.Exchange) { + ex.WriteBody([]byte("hello world")) + }) + var router httphi.Router + err := router.Configure(httphi.RouterConfig{ + FixedNumGoroutines: -1, // Unbounded goroutines and allocations. + RequestHeaderBufferSize: requestBuffer, + ResponseHeaderMinBufferSize: 32, // Shared buffer with Request, not strictly necessary, especially if not sending headers. + RequestNumHeaderKVCap: numHeaderKV, + NormalizeOutgoingKeys: true, + Mux: &mux, + Logger: slog.Default(), + }) + if err != nil { + log.Fatal(err) + } + const port = ":8080" + listener, err := net.Listen("tcp", port) + if err != nil { + log.Fatal(err) + } + log.Printf("server up at http://localhost%s", port) + for { + conn, err := listener.Accept() + if err != nil { + log.Fatal(err) + } + err = router.Handle(conn) + if err != nil { + log.Println("httphi.Router failed to handle connection:", err) + } + } +} + +// ExampleMuxSlice_query_forms_multipart goes over how to define Handlers (HandleFunc). +// See [ExampleRouter_linux] for how to setup the server. +func ExampleMuxSlice_query_forms_multipart() { + + var mux httphi.MuxSlice + + mux.Handle("/users/{id}", func(ex *httphi.Exchange) { + userID := ex.PathValue("id") + fmt.Printf("someone requested data for user %s\n", userID) + }) + + mux.Handle("/query", func(ex *httphi.Exchange) { + // query parameter in URL. + const decodeQuery = true + const queryKey = "search" + valueRaw, present := ex.RequestQueryValue(queryKey) + if !present { + return + } + valueDecoded, present := ex.RequestQueryAppend(nil, queryKey, decodeQuery) + fmt.Printf("got query=%v %s=%s (raw:%s)\n", present, queryKey, valueDecoded, valueRaw) + }) + + mux.Handle("GET /form", func(ex *httphi.Exchange) { + // Request Body Form. + formbuf := make([]byte, 1024) + var form httpraw.Form + err := ex.RequestParseForm(&form, formbuf) + if err != nil { + ex.WriteHeader(httphi.StatusInternalServerError) + return + } + for i := range form.Len() { + k, v := form.Pair(i) + fmt.Printf("received form value %d: %s=%s\n", i, k, v) + } + }) + + mux.Handle("GET /file-upload", func(ex *httphi.Exchange) { + // File upload directly onto server using multipart. + formbuf := make([]byte, 1024) + _, err := ex.ReadMultiparts(nil, formbuf, func(hdr *httpraw.MultipartHeader) io.WriteCloser { + fp, err := os.Create(string(hdr.Filename)) + if err != nil { + return nil + } + return fp // Will write full file contents to fp. + }) + if err != nil { + ex.WriteHeader(httphi.StatusInternalServerError) + return + } + }) + +} diff --git a/http/httphi/exchange.go b/http/httphi/exchange.go new file mode 100644 index 0000000..59d182a --- /dev/null +++ b/http/httphi/exchange.go @@ -0,0 +1,707 @@ +package httphi + +import ( + "io" + "net" + "slices" + "strconv" + "sync/atomic" + + "github.com/soypat/lneto" + "github.com/soypat/lneto/http/httpraw" + "github.com/soypat/lneto/internal" +) + +// maxStatusLine bounds the response status line: "HTTP/1.1 " + 3 digit code + +// " " + longest [StatusText] + CRLF. +const maxStatusLine = len("HTTP/1.1 ") + 3 + 1 + len("Network Authentication Required") + 2 + +// Exchange is a single request-response cycle over a connection, playing the +// part of both http.Request and http.ResponseWriter: Request* methods read the +// request, [Exchange.StageHeader] and [Exchange.WriteBody] produce the response. +// A [Router] owns a fixed pool of them, which is what bounds its memory. +// +// Request and response share one buffer, the response header being written over +// the bytes that follow the parsed request header. Read the request body with +// [Exchange.ReadBody] before setting response headers. +type Exchange struct { + acquired atomic.Bool + gen atomic.Uint32 + respTopBuf [maxStatusLine]byte + respTopWritten uint8 + + rawbuf []byte + respHeaderOff uint16 + respHeaderLen uint16 + reqHdr httpraw.Header + pathValues []pathValue + + hijacked bool + rw conn + + matchedPattern string + + respRemains int + respErr error // Sticky: response is unrecoverable once a write fails. + headerWritten bool + normalizeKeys bool + nextFree *Exchange + readErr error +} + +// ExchangeConfig is the memory an [Exchange] is fixed to for the rest of its +// life by [Exchange.Configure]. A [Router] derives one per exchange from its +// [RouterConfig], which is what bounds the router's memory. +type ExchangeConfig struct { + // RawBuf is the single buffer holding the request header, the response + // header and any surplus body. See [Exchange.UnsafeRawBuffer]. + RawBuf []byte + // RequestBufferLim reserves the first bytes of RawBuf for the request + // header, the rest being the response. Configure panics if it exceeds RawBuf. + RequestBufferLim int + // NumHeaderKVCap is how many request header fields may be parsed. A request + // carrying more is answered 431, see [httpraw.ErrHeaderTooMany]. + NumHeaderKVCap int + // NormalizeOutgoingKeys normalizes staged response header keys as they are + // written, i.e: "content-type" becomes "Content-Type". + NormalizeOutgoingKeys bool + // NoRequestBufferGrowth holds the request header to RequestBufferLim rather + // than growing it. A header outgrowing it is answered 431, see [httpraw.ErrBufferExhausted]. + NoRequestBufferGrowth bool + // MaxPathValues is how many wildcards a single pattern may bind, read back with + // [Exchange.PathValue]. A pattern with more never matches, see [SetPathValues]. + MaxPathValues int +} + +// HijackRaw is a low-level implementation of http.Hijacker interface. +// A Hijack method is not exposed due to heap allocation implications and correctness concerns. +// Below is what an actual implementation may look like: +// +// func (exch *Exchange) Hijack() (net.Conn, *bufio.ReadWriter, error) { +// conn, ok := exch.rw.(net.Conn) +// if !ok { +// return nil, nil, errors.New("net.Conn not implemented") +// } +// _, data, err := exch.HijackRaw(nil) +// if err != nil { +// return nil, nil, err +// } +// var rd *bufio.ReadWriter +// if len(data) > 0 { +// rd = &bufio.ReadWriter{Reader: bufio.NewReader(bytes.NewReader(data))} +// } +// return conn, rd, nil +// } +func (exch *Exchange) HijackRaw(dstBody []byte) (conn, []byte, error) { + data, err := exch.remainingSurplusBody() + if err != nil { + return nil, nil, err + } + exch.hijacked = true + dstBody = append(dstBody, data...) + return exch.rw, dstBody, nil +} + +// Configure sets the memory the exchange works with for the rest of its life: +// rawbuf holds the request header, the response header and any surplus body, +// of which the first requestLim bytes are reserved for the request header. +// Panics if requestLim exceeds the buffer. Set normalizeKeys to normalize +// outgoing header keys, i.e: "content-type" to "Content-Type". +func (exch *Exchange) Configure(cfg ExchangeConfig) { + respSize := len(cfg.RawBuf) - cfg.RequestBufferLim + if respSize < 0 { + panic("request lim larger than buffer") + } + exch.rawbuf = cfg.RawBuf + exch.reqHdr.Reset(cfg.RawBuf[:0:cfg.RequestBufferLim], cfg.NumHeaderKVCap) + exch.reqHdr.ConfigBufferGrowth(!cfg.NoRequestBufferGrowth) + exch.normalizeKeys = cfg.NormalizeOutgoingKeys + internal.SliceReuse(&exch.pathValues, cfg.MaxPathValues) + exch.pathValues = exch.pathValues[:cfg.MaxPathValues] +} + +// Acquire claims the exchange for conn and resets it to serve a new request, +// reusing the buffer set by [Exchange.Configure]. Returns false if the exchange +// is already serving, in which case conn is untouched. +func (exch *Exchange) Acquire(conn conn) bool { + if !exch.acquired.CompareAndSwap(false, true) { + return false + } + exch.matchedPattern = "" + exch.gen.Add(1) + exch.readErr = nil + exch.respErr = nil + exch.hijacked = false + exch.respTopWritten = 0 + exch.respHeaderOff = 0 + exch.respHeaderLen = 0 + exch.respRemains = 0 + exch.rw = conn + exch.headerWritten = false + exch.nextFree = nil + clear(exch.pathValues) + exch.reqHdr.Reset(nil, 0) + return true +} + +// Release closes the exchange's connection and frees the exchange for a future +// [Exchange.Acquire]. The connection is left open if the handler took ownership +// of it with [Exchange.HijackRaw]. +func (exch *Exchange) Release() { + if !exch.hijacked { + exch.rw.Close() + } + exch.rw = nil + exch.gen.Add(1) + exch.acquired.Store(false) +} + +// UnsafeRawBuffer returns the contiguous buffer owned by [Exchange] being used for the request and response. +// +// Writing to it will mangle the entire request header+body and/or any staged response headers. +// Does not return the buffer used for the response first line so can be safely +// written to and used without modifying the staged response first line. +// +// Staging headers will write to this buffer so use mindfully. +// To access only the request header buffer portion use [httpraw.Header.BufferRaw] limited +// to [httpraw.Header.BufferParsed] as returned by [Exchange.RequestHeaderRaw]. +// Writing to this section will not change the contents read by [Exchange.ReadBody]. +// +// In [Router] context, the size of this buffer is influenced directly by [RouterConfig] HeaderBufferSize fields. +func (exch *Exchange) UnsafeRawBuffer() []byte { return exch.rawbuf } + +// StageHeader stages a response header field, written on the first +// [Exchange.FlushHeader], [Exchange.WriteHeader] or [Exchange.WriteBody]. +// Returns false and drops the field if the response buffer cannot fit it. +// Has no effect once the header has been written. +func (exch *Exchange) StageHeader(key, value string) (enoughMemory bool) { + if exch.headerWritten { + return false + } + off := int(exch.respHeaderOff) + int(exch.respHeaderLen) + free := len(exch.rawbuf) - off + // Field costs key+':'+value+CRLF, plus the CRLF [Exchange.FlushHeader] + // appends past the last field to close the header block. + if len(key)+len(value)+len(":\r\n")+len("\r\n") > free { + exch.respErr = lneto.ErrBufferFull // Omit writing header back to prevent incomplete response. + return false + } + n := copy(exch.rawbuf[off:], key) + if exch.normalizeKeys { + httpraw.NormalizeHeaderKey(exch.rawbuf[off : off+n]) + } + exch.rawbuf[off+n] = ':' + n++ + n += copy(exch.rawbuf[off+n:], value) + exch.rawbuf[off+n] = '\r' + exch.rawbuf[off+n+1] = '\n' + n += 2 + exch.respHeaderLen += uint16(n) + return true +} + +// StageHeaderInt is [Exchange.StageHeader] with an integer value, i.e: Content-Length. +// It formats the value directly into the response buffer without allocating. +// base must be in the range 10..36; lower bases are dropped, no HTTP header +// field value is written below base 10. +func (exch *Exchange) StageHeaderInt(key string, value int64, base int) (enoughMemory bool) { + if exch.headerWritten || base < 10 || base > 36 { + return false + } + off := int(exch.respHeaderOff) + int(exch.respHeaderLen) + free := len(exch.rawbuf) - off + if len(key)+internal.IntLen(value, base)+len(":\r\n")+len("\r\n") > free { + exch.respErr = lneto.ErrBufferFull // Omit writing header back to prevent incomplete response. + return false + } + n := copy(exch.rawbuf[off:], key) + if exch.normalizeKeys { + httpraw.NormalizeHeaderKey(exch.rawbuf[off : off+n]) + } + exch.rawbuf[off+n] = ':' + n++ + n += len(strconv.AppendInt(exch.rawbuf[off+n:off+n], value, base)) + exch.rawbuf[off+n] = '\r' + exch.rawbuf[off+n+1] = '\n' + n += 2 + exch.respHeaderLen += uint16(n) + return true +} + +// StageStatus prepares the status line for the given code without writing +// it, i.e: "HTTP/1.1 404 Not Found". Codes with no [StatusText] get an empty +// reason phrase. Has no effect once the header has been written. +func (exch *Exchange) StageStatus(code int) { + if code >= 1000 || exch.headerWritten { + return + } else if code == 200 { + // Common case. + exch.respTopWritten = uint8(copy(exch.respTopBuf[:], "HTTP/1.1 200 OK\r\n")) + return + } + n := copy(exch.respTopBuf[:], "HTTP/1.1 ") + n += len(strconv.AppendInt(exch.respTopBuf[n:n], int64(code), 10)) + text := StatusText(code) + exch.respTopBuf[n] = ' ' + n++ + n += copy(exch.respTopBuf[n:], text) + exch.respTopBuf[n] = '\r' + exch.respTopBuf[n+1] = '\n' + exch.respTopWritten = uint8(n + 2) +} + +// WriteHeader sends the status line for code along with the staged header +// fields. Only the first call reaches the wire, as in http.ResponseWriter. +func (exch *Exchange) WriteHeader(code int) { + if !exch.headerWritten { + exch.StageStatus(code) + exch.FlushHeader() + } +} + +// FlushHeader writes the status line and staged header fields to the connection +// and returns the bytes written, defaulting to a 200 status if none was staged. +// Does nothing if the header was already written. +func (exch *Exchange) FlushHeader() (int, error) { + if exch.respErr != nil { + return 0, exch.respErr + } else if exch.headerWritten { + return 0, nil + } + if exch.respTopWritten == 0 { + exch.StageStatus(200) + } + exch.headerWritten = true + ng, err := exch.rw.Write(exch.respTopBuf[:exch.respTopWritten]) + if err != nil { + exch.respErr = err + return ng, err + } + off := int(exch.respHeaderOff) + headers := exch.rawbuf[off : off+int(exch.respHeaderLen)+2] + headers[len(headers)-1] = '\n' + headers[len(headers)-2] = '\r' + ng2, err := exch.rw.Write(headers) + exch.respErr = err + return ng + ng2, err +} + +// ExchangeRW is an [io.ReadWriteCloser] view of an [Exchange] wrapping +// [Exchange.ReadBody] and [Exchange.WriteBody] methods. +// +// Exchanges are pooled and reused, so a handle records the exchange generation +// it was taken at and refuses to touch the connection once that exchange moves +// on to another request. Obtain one with [Exchange.ReadWriter]. +type ExchangeRW struct { + gen uint32 + exch *Exchange +} + +// IsValid returns true while the handle still refers to the request it was +// taken from, i.e: false once the exchange was released. +func (rw *ExchangeRW) IsValid() bool { + return rw.gen == rw.exch.gen.Load() && rw.exch.acquired.Load() +} + +func (rw *ExchangeRW) validate() error { + if !rw.IsValid() { + return net.ErrClosed + } + return nil +} + +// Write writes response body bytes. See [Exchange.WriteBody]. +// Fails with [net.ErrClosed] once the handle is no longer valid. +func (rw *ExchangeRW) Write(buf []byte) (int, error) { + if err := rw.validate(); err != nil { + return 0, err + } + return rw.exch.WriteBody(buf) +} + +// Read reads request body bytes. See [Exchange.ReadBody]. +// Fails with [net.ErrClosed] once the handle is no longer valid. +func (rw *ExchangeRW) Read(buf []byte) (int, error) { + if err := rw.validate(); err != nil { + return 0, err + } + return rw.exch.ReadBody(buf) +} + +// Close invalidates this handle so later reads and writes fail. It does not +// close the connection nor end the exchange, both of which the [Router] owns. +func (rw *ExchangeRW) Close() error { + if err := rw.validate(); err != nil { + return err + } + rw.gen-- + return nil +} + +// ReadWriter fills dst with a stream view of the exchange, valid until the +// exchange is released. The caller owns dst, so a handler may keep one and +// refill it every request without allocating. +func (exch *Exchange) ReadWriter(dst *ExchangeRW) { + dst.gen = exch.gen.Load() + dst.exch = exch +} + +// Write writes response body bytes, flushing the header first if the handler +// has not written it yet. Once a write to the connection fails the response is +// unrecoverable and every later write returns that same error, so a body never +// reaches the wire without its header. +func (exch *Exchange) WriteBody(buf []byte) (int, error) { + if exch.respErr != nil { + return 0, exch.respErr + } else if !exch.headerWritten { + _, err := exch.FlushHeader() + if err != nil { + return 0, err // Body must not reach the wire without its header. + } + } + if len(buf) == 0 { + return 0, nil + } + n, err := exch.rw.Write(buf) + exch.respErr = err + return n, err +} + +// ReadBody reads the request body into dst, starting with the bytes that +// arrived in the same read as the header and continuing from the connection. +// The exchange does not know the body's length: use Content-Length or the +// transfer encoding to know when to stop reading. +func (exch *Exchange) ReadBody(dst []byte) (n int, _ error) { + if exch.respRemains > 0 { + toRead, err := exch.remainingSurplusBody() + if err != nil { + return 0, err + } + n = copy(dst, toRead) + exch.respRemains -= n + // hand over what already arrived since conn might have + // exhausted data and could block indefinetely. + return n, nil + } + return exch.rw.Read(dst) +} + +func (exch *Exchange) remainingSurplusBody() ([]byte, error) { + _, err := exch.reqHdr.Body() + if err != nil { + return nil, err // Returns mangled buffer error if request header has been misused. + } + surplus := exch.rawbuf[exch.reqHdr.BufferParsed():exch.reqHdr.BufferReceived()] + toRead := surplus[len(surplus)-exch.respRemains:] + return toRead, nil +} + +// MuxPattern returns the pattern [Mux] matched to the request. +func (exch *Exchange) MuxPattern() string { + return exch.matchedPattern +} + +// RequestHeaderRaw returns the parsed request header for access beyond the +// Request* methods, such as [httpraw.Header.ForEach]. Valid until the exchange +// is released, and writing to it corrupts the response. +func (exch *Exchange) RequestHeaderRaw() *httpraw.Header { + return &exch.reqHdr +} + +// RequestParseCookie parses the request's key header field into dst, i.e: +// "Cookie". The caller owns dst and its buffer, so it may be reused between +// requests. +func (exch *Exchange) RequestParseCookie(dst *httpraw.Cookie, key string) error { + value := exch.RequestHeader(key) + return dst.ParseBytes(value) +} + +// RequestContentType returns the request's Content-Type field value as it +// appears on the wire, parameters included, nil if absent. Test it with +// [httpraw.MediaTypeIs] and pick parameters out with [httpraw.ContentParam]. +func (exch *Exchange) RequestContentType() []byte { + // Folded: field names are case insensitive and HTTP/2 mandates lowercase, so + // a proxy translating h2 to h1 sends "content-type", RFC 9110 5.1. + return exch.RequestHeaderRaw().GetFold("Content-Type") +} + +// RequestContentLength returns the body length declared by the request's +// Content-Length field. An absent field is signalled with present=false and no error. +// See [httpraw.Header.ContentLength]. +func (exch *Exchange) RequestContentLength() (_ int64, present bool, _ error) { + return exch.RequestHeaderRaw().ContentLength() +} + +// RequestParseForm reads the request body into buf and parses it as +// "application/x-www-form-urlencoded" into dst. buf is the only storage used and +// the only limit: a body longer than buf is refused with [lneto.ErrBufferFull] +// before a single byte is read, leaving the caller free to answer 413. Pairs are +// left as they arrived, call [httpraw.Form.Decode] to decode them in place. +// +// Unlike http.Request.ParseForm the query string is not folded in, reach it with +// [Exchange.RequestQuery] or [Exchange.RequestQueryAppend]. The body is consumed, so +// call this before [Exchange.ReadBody]. +// +// A request with no Content-Length has no body, RFC 9112 6.3, and yields an +// empty form. Use [Exchange.RequestContentLength] to tell that apart from a body +// that arrived empty. +func (exch *Exchange) RequestParseForm(dst *httpraw.Form, buf []byte) error { + if !httpraw.MediaTypeIs(exch.RequestContentType(), "application/x-www-form-urlencoded") { + return errNotFormEncoded + } else if exch.RequestHeaderRaw().GetFold("Transfer-Encoding") != nil { + // Chunked bodies are framed, so reading Content-Length bytes off the + // wire would parse chunk sizes as form data. httpraw does not decode them. + return errUnsupportedTransferCoding + } + + length, present, err := exch.RequestContentLength() + if !present { + dst.Reset(nil, 0) + return nil // No length is no body, RFC 9112 6.3. + } else if err != nil { + return err + } else if length > int64(len(buf)) { + return lneto.ErrShortBuffer // Refuse before reading, caller may answer 413. + } + buf = buf[:length] + for read := 0; read < len(buf); { + n, err := exch.ReadBody(buf[read:]) + read += n + if n == 0 { + if err == nil { + err = io.ErrNoProgress + } else if err == io.EOF { + break + } + return err + } + } + dst.Reset(buf, 0) + return dst.Parse() +} + +// RequestMultipart returns a parser prepared from the boundary parameter of the +// request's Content-Type field. It reads no body: multipart parts declare no +// length, so the caller drives the loop with a buffer it owns and decides per +// part what to keep and when a part has grown too large. See +// [Exchange.ReadMultiparts] for that loop already written. +func (exch *Exchange) RequestMultipart() (mp httpraw.Multipart, err error) { + contentType := exch.RequestContentType() + if !httpraw.MediaTypeIs(contentType, "multipart/form-data") { + return mp, errNotMultipart + } + return mp, mp.SetContentType(contentType) +} + +// MultipartSink is a part of a multipart body together with the writer its +// content was streamed to, as appended by [Exchange.ReadMultiparts]. +type MultipartSink struct { + // Header identifies the part. Name and Filename are copies, so they + // outlive the read buffer; PartView does not, see [httpraw.MultipartHeader]. + Header httpraw.MultipartHeader + // Sink received the part's content and was closed when the part ended, + // nil for a part newSink chose to discard. + Sink io.WriteCloser +} + +// ReadMultiparts streams the request's "multipart/form-data" body, writing each +// part to a sink newSink returns for it and appending the pair to dst. buf is the +// only storage used and content is never held whole, so a part of any length +// streams through a buffer the caller sized. dst is appended to and returned, so +// a handler may hand back the slice of a previous request to reuse its parts. +// +// newSink is called once per part, before any of its content is read, and picks +// what to do with it from hdr.Name and hdr.Filename: return a writer to keep the +// part, or nil to discard its content and keep only the header. Each sink is +// closed as soon as its part ends, so Close reports the part arrived whole; on +// error the sink of the part being read is left open for the caller to deal with. +// +// A part header that does not fit buf is refused with [lneto.ErrShortBuffer], +// since reading more can never complete it, leaving the caller free to answer +// 413. The body is consumed, so call this before [Exchange.ReadBody]. +func (exch *Exchange) ReadMultiparts(dst []MultipartSink, buf []byte, newSink func(hdr *httpraw.MultipartHeader) io.WriteCloser) (_ []MultipartSink, _ error) { + mp, err := exch.RequestMultipart() + if err != nil { + return dst, err + } else if newSink == nil || len(buf) <= len("\r\n--")+len(mp.Boundary) { + // A buffer that cannot outgrow a delimiter never makes progress. + return dst, lneto.ErrInvalidConfig + } + buflen := 0 + for { + // Slot for the next part, given back when the body turns out to be + // over, so its Name and Filename buffers stay available for reuse. + part := internal.SliceReclaim(&dst) + var parsed int + for { + parsed, err = mp.NextHeader(&part.Header, buf[:buflen]) + if err != nil { + dst = dst[:len(dst)-1] + if err == io.EOF { + err = nil // Closing delimiter, body done. + } + return dst, err + } else if parsed > 0 { + break // Delimiter and header block complete. + } else if buflen == len(buf) { + dst = dst[:len(dst)-1] + return dst, lneto.ErrShortBuffer // Header longer than buf. + } + // A read that both delivers and fails, as the last of the body + // followed by a hangup does, may still hold what the parser is + // waiting for: take the data and let the error surface on the + // next read. + n, readErr := exch.ReadBody(buf[buflen:]) + buflen += n + if n == 0 && readErr != nil { + dst = dst[:len(dst)-1] + return dst, readErr + } + } + part.Sink = newSink(&part.Header) + buflen = copy(buf, buf[parsed:buflen]) + for { + bodyLen, restOff, done := mp.NextBody(buf[:buflen]) + if bodyLen > 0 && part.Sink != nil { + _, err = part.Sink.Write(buf[:bodyLen]) + if err != nil { + return dst, err + } + } + buflen = copy(buf, buf[restOff:buflen]) + if done { + break // Buffer now starts at the next part's delimiter. + } + n, readErr := exch.ReadBody(buf[buflen:]) + buflen += n + if n == 0 && readErr != nil { + return dst, readErr // Body ended mid part. + } + } + if part.Sink != nil { + if err = part.Sink.Close(); err != nil { + return dst, err + } + } + } +} + +// RequestHeader returns the value of the first request header field matching +// key, or nil if absent. Key matching is case sensitive. +func (exch *Exchange) RequestHeader(key string) []byte { + header := exch.RequestHeaderRaw() + return header.Get(key) +} + +// RequestTarget returns the request-target (URI) of the request line, i.e: +// "/search?q=go". See [httpraw.Header.RequestTarget]. +func (exch *Exchange) RequestTarget() []byte { + return exch.RequestHeaderRaw().RequestTarget() +} + +// RequestPath returns the request-target (URI) up to the query string. This is +// what the [Mux] matches on, i.e: "/search" for a request to "/search?q=go". +func (exch *Exchange) RequestPath() []byte { + return exch.RequestHeaderRaw().RequestPath() +} + +// RequestQuery returns the request's query string as it appears on the wire. +// Iterate it with [httpraw.NextQueryPair]. See [httpraw.Header.RequestQuery]. +func (exch *Exchange) RequestQuery() []byte { + return exch.RequestHeaderRaw().RequestQuery() +} + +// RequestQueryValue returns an undecoded view of the first query parameter +// matching key and reports whether it was present. Keys are matched decoded, so +// key "a b" finds "a%20b" and "a+b"; a parameter whose key is a malformed +// escape is skipped. A parameter with no value ("?debug") and one with an empty +// value ("?debug=") are both present with a zero length view. +// +// The view aliases the request buffer, so copy it to outlive the handler or use +// [Exchange.RequestQueryAppend] to decode it out. +func (exch *Exchange) RequestQueryValue(key string) (rawValue []byte, present bool) { + const plusAsSpace = true // Query strings are form encoded, unlike paths. + rawkey, rawval, rest := httpraw.NextQueryPair(exch.RequestQuery()) + for ; rawkey != nil; rawkey, rawval, rest = httpraw.NextQueryPair(rest) { + // Compare raw first: a key needing no decoding is the common case, and + // the decoding compare walks the key an escape at a time. + if b2s(rawkey) == key || httpraw.EqualDecodedPercentURL(rawkey, key, plusAsSpace) { + return rawval, true + } + } + return nil, false +} + +// RequestQueryAppend appends the value of the first query parameter matching key to +// dst and reports whether the parameter was present, matching keys as +// [Exchange.RequestQueryValue] does. A parameter with no value ("?debug") and +// one with an empty value ("?debug=") are both present with nothing appended. +// +// Values are appended raw unless decoded is set, in which case percent escapes +// and '+' are decoded. A parameter whose value fails to decode is reported +// absent, dst being left as it was rather than holding half a decode. +func (exch *Exchange) RequestQueryAppend(dst []byte, key string, decoded bool) (valueAppended []byte, present bool) { + const plusAsSpace = true // Query strings are form encoded, unlike paths. + rawval, present := exch.RequestQueryValue(key) + if !present || len(rawval) == 0 { + return dst, present + } + if !decoded { + return append(dst, rawval...), true + } + base := len(dst) + dst = slices.Grow(dst, len(rawval)) + n, err := httpraw.CopyDecodedPercentURL(dst[base:base+len(rawval)], rawval, plusAsSpace) + if err != nil { + return dst[:base], false // Do not hand back half a decode. + } + return dst[:base+n], true +} + +// PathValue returns the segment the request path bound to the wildcard named +// key, or nil if the matched pattern has no such wildcard. It plays the part of +// http.Request.PathValue. See [SetPathValues] for the pattern syntax and for +// which segments a wildcard binds. +// +// sm.Handle("GET /users/{id}", func(exch *httphi.Exchange) { +// id := exch.PathValue("id") // "42" on a GET /users/42. +// }) +func (exch *Exchange) PathValue(key string) []byte { + for i := range exch.pathValues { + if exch.pathValues[i].Key == key { + return exch.pathValues[i].Value + } else if exch.pathValues[i].Key == "" { + break // No more keys set. + } + } + return nil +} + +// PathValueAppend acceses the result of [Exchange.PathValue] and appends it to dst. +// If decoded is set to true the result will be URL-percent decoded. An error is returned if URL-percent decoding fails. +func (exch *Exchange) PathValueAppend(dst []byte, key string, decoded bool) ([]byte, error) { + const plusAsSpace = true + rawValue := exch.PathValue(key) + if !decoded || len(rawValue) == 0 { + return append(dst, rawValue...), nil + } + base := len(dst) + dst = slices.Grow(dst, len(rawValue)) + n, err := httpraw.CopyDecodedPercentURL(dst[base:base+len(rawValue)], rawValue, plusAsSpace) + if err != nil { + return dst[:base], err // Do not hand back half a decode. + } + return dst[:base+n], nil +} + +// RequestMethod returns the request line's method, i.e: "GET". See +// [MethodFromBytes] to compare it against a [Method]. +func (exch *Exchange) RequestMethod() []byte { + return exch.RequestHeaderRaw().Method() +} + +// RequestConnectionClose returns true if the client asked for the connection to +// be closed after this exchange with a "Connection: close" header field. +func (exch *Exchange) RequestConnectionClose() bool { + return exch.RequestHeaderRaw().ConnectionClose() +} diff --git a/http/httphi/exchange_test.go b/http/httphi/exchange_test.go new file mode 100644 index 0000000..cf908f2 --- /dev/null +++ b/http/httphi/exchange_test.go @@ -0,0 +1,1472 @@ +package httphi + +import ( + "context" + "errors" + "io" + "slices" + "strconv" + "strings" + + "testing" + "time" + + "github.com/soypat/lneto/http/httpraw" + + "github.com/soypat/lneto" +) + +func nopBackoff(consecutiveBackoffs uint) time.Duration { return lneto.BackoffFlagNop } + +// defaultNumHeaderKVCap is the field table tests get unless they set their own: +// room for a realistic request, the sizing being what the test is about only +// where it says so. +const defaultNumHeaderKVCap = 32 + +// newExchange returns an Exchange acquired on conn, ready to serve a request. +func newExchange(t *testing.T, conn conn, cfg ExchangeConfig) *Exchange { + t.Helper() + exch := new(Exchange) + if cfg.NumHeaderKVCap == 0 { + cfg.NumHeaderKVCap = defaultNumHeaderKVCap + } + exch.Configure(cfg) + if !exch.Acquire(conn) { + t.Fatal("fresh exchange failed to acquire connection") + } + return exch +} + +// serve runs a single exchange to completion on the calling goroutine. +func serve(t *testing.T, request string, mux Mux) *rwconn { + t.Helper() + const bufferSize = 1024 + conn := newConn(request) + conn.Hangup() // Whole request already pending, nothing more will arrive. + exch := newExchange(t, conn, ExchangeConfig{ + RawBuf: make([]byte, 2*bufferSize), + RequestBufferLim: bufferSize, + }) + err := Handle(exch, mux, nopBackoff) + if err != nil { + t.Fatalf("Handle(%q): %s", request, err) + } + return conn +} + +// WriteHeader must emit a complete status line terminated in CRLF followed by +// the end-of-headers CRLF, for every status code including the longest text. +func TestExchangeWriteHeader(t *testing.T) { + var buf [128]byte + for _, test := range []struct { + code int + want string + }{ + {code: 200, want: "HTTP/1.1 200 OK\r\n\r\n"}, + {code: 404, want: "HTTP/1.1 404 Not Found\r\n\r\n"}, + {code: 500, want: "HTTP/1.1 500 Internal Server Error\r\n\r\n"}, + // Longest status text in status.go: worst case for the status line buffer. + {code: 511, want: "HTTP/1.1 511 Network Authentication Required\r\n\r\n"}, + } { + t.Run(strconv.Itoa(test.code), func(t *testing.T) { + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: buf[:], RequestBufferLim: 64}) + exch.WriteHeader(test.code) + if got := conn.ViewWritten(); got != test.want { + t.Errorf("want %q, got %q", test.want, got) + } + }) + } +} + +// Status line is written once: a second WriteHeader must not reach the wire. +func TestExchangeWriteHeaderOnce(t *testing.T) { + var buf [128]byte + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: buf[:], RequestBufferLim: 64}) + exch.WriteHeader(404) + exch.WriteHeader(500) + const want = "HTTP/1.1 404 Not Found\r\n\r\n" + if got := conn.ViewWritten(); got != want { + t.Errorf("want %q, got %q", want, got) + } +} + +// Write with no prior WriteHeader must flush a 200 header ahead of the body. +func TestExchangeWriteFlushesHeader(t *testing.T) { + var buf [128]byte + const body = "hello" + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: buf[:], RequestBufferLim: 64}) + n, err := exch.WriteBody([]byte(body)) + if err != nil { + t.Fatal(err) + } + if n != len(body) { + t.Errorf("want %d bytes written, got %d", len(body), n) + } + const want = "HTTP/1.1 200 OK\r\n\r\n" + body + if got := conn.ViewWritten(); got != want { + t.Errorf("want %q, got %q", want, got) + } +} + +func TestExchangeSetHeader(t *testing.T) { + for _, test := range []struct { + name string + normalize bool + set [][2]string + want string // Header block emitted after the status line. + }{ + {name: "none", want: "\r\n"}, + {name: "single", set: [][2]string{{"Content-Type", "text/plain"}}, want: "Content-Type:text/plain\r\n\r\n"}, + { + name: "multiple", + set: [][2]string{{"Content-Type", "text/plain"}, {"Content-Length", "5"}}, + want: "Content-Type:text/plain\r\nContent-Length:5\r\n\r\n", + }, + { + name: "normalized key", + normalize: true, + set: [][2]string{{"content-TYPE", "text/plain"}}, + want: "Content-Type:text/plain\r\n\r\n", + }, + { + name: "key kept verbatim when not normalizing", + set: [][2]string{{"content-TYPE", "text/plain"}}, + want: "content-TYPE:text/plain\r\n\r\n", + }, + {name: "empty value", set: [][2]string{{"X-Empty", ""}}, want: "X-Empty:\r\n\r\n"}, + } { + t.Run(test.name, func(t *testing.T) { + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 128), RequestBufferLim: 64, NormalizeOutgoingKeys: test.normalize}) + for _, kv := range test.set { + if !exch.StageHeader(kv[0], kv[1]) { + t.Fatalf("SetHeader(%q,%q) reported insufficient memory", kv[0], kv[1]) + } + } + exch.WriteHeader(200) + got, found := strings.CutPrefix(conn.ViewWritten(), "HTTP/1.1 200 OK\r\n") + if !found { + t.Fatalf("want 200 status line, got %q", conn.ViewWritten()) + } + if got != test.want { + t.Errorf("want header block %q, got %q", test.want, got) + } + }) + } +} + +// SetHeader must refuse to write past the buffer and say so, never panic nor +// emit a truncated field. +func TestExchangeSetHeaderOOM(t *testing.T) { + const bufferSize = 32 + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*bufferSize), RequestBufferLim: bufferSize}) + if exch.StageHeader("X-Big", strings.Repeat("v", 4*bufferSize)) { + t.Fatal("want insufficient memory reported for oversized header value") + } + exch.WriteHeader(200) + if got := conn.ViewWritten(); strings.Contains(got, "X-Big") { + t.Errorf("dropped header must not appear in response, got %q", got) + } +} + +// Request line and fields the handler observes, over a spread of well formed +// and awkward but legal request headers. +func TestHandleRequestFields(t *testing.T) { + for _, test := range []struct { + name string + request string + wantMethod string + wantURI string + wantHost string + }{ + { + name: "minimal", + request: "GET / HTTP/1.1\r\nHost: h\r\n\r\n", + wantMethod: "GET", wantURI: "/", wantHost: "h", + }, + { + name: "query string", + request: "GET /search?q=go&n=1 HTTP/1.1\r\nHost: h\r\n\r\n", + wantMethod: "GET", wantURI: "/search?q=go&n=1", wantHost: "h", + }, + { + name: "no fields", + request: "GET /x HTTP/1.1\r\n\r\n", + wantMethod: "GET", wantURI: "/x", wantHost: "", + }, + { + name: "post", + request: "POST /submit HTTP/1.1\r\nHost: h\r\nContent-Length: 0\r\n\r\n", + wantMethod: "POST", wantURI: "/submit", wantHost: "h", + }, + { + name: "extension method", + request: "FROBNICATE / HTTP/1.1\r\nHost: h\r\n\r\n", + wantMethod: "FROBNICATE", wantURI: "/", wantHost: "h", + }, + } { + t.Run(test.name, func(t *testing.T) { + var gotMethod, gotURI, gotHost string + var sm MuxSlice + route, _, _ := strings.Cut(test.wantURI, "?") // Mux matches on path. + sm.Handle(route, func(ex *Exchange) { + gotMethod = string(ex.RequestMethod()) + gotURI = string(ex.RequestTarget()) + gotHost = string(ex.RequestHeader("Host")) + ex.WriteHeader(200) + }) + serve(t, test.request, &sm) + + if gotMethod != test.wantMethod { + t.Errorf("want method %q, got %q", test.wantMethod, gotMethod) + } + if gotURI != test.wantURI { + t.Errorf("want URI %q, got %q", test.wantURI, gotURI) + } + if gotHost != test.wantHost { + t.Errorf("want Host %q, got %q", test.wantHost, gotHost) + } + }) + } +} + +// Malformed request headers must fail the exchange, never reach a handler. +func TestHandleMalformedRequest(t *testing.T) { + for _, test := range []struct { + name string + request string + }{ + {name: "no colon in field", request: "GET / HTTP/1.1\r\nBadFieldNoColon\r\n\r\n"}, + {name: "no protocol", request: "GET /\r\nHost: h\r\n\r\n"}, + {name: "empty request line", request: "\r\n\r\n"}, + {name: "truncated header", request: "GET / HTTP/1.1\r\nHost: h\r\n"}, + } { + t.Run(test.name, func(t *testing.T) { + var handled bool + var sm MuxSlice + sm.Handle("/", func(ex *Exchange) { handled = true }) + conn := newConn(test.request) + conn.Hangup() + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) + if err := Handle(exch, &sm, nopBackoff); err == nil { + t.Error("want error on malformed request, got nil") + } + if handled { + t.Error("handler must not run on malformed request") + } + }) + } +} + +// A request-line naming a version this package does not speak must be answered +// 505 without reaching a handler, RFC 9112 2.6. The third token is not +// validated by the parser, so anything that is not HTTP/1.0 or HTTP/1.1 lands +// here: bogus versions, non-HTTP tokens, and the tail of a request-target that +// contained a space and got split across the URI/protocol boundary. +func TestHandleUnsupportedProtocol(t *testing.T) { + for _, test := range []struct { + name string + request string + }{ + {name: "future major", request: "GET / HTTP/6.9\r\nHost: h\r\n\r\n"}, + {name: "future major keepalive", request: "GET / HTTP/6.9\r\nHost: h\r\nConnection: keep-alive\r\n\r\n"}, + {name: "http2", request: "GET / HTTP/2.0\r\nHost: h\r\n\r\n"}, + {name: "long minor", request: "GET / HTTP/1.10\r\nHost: h\r\n\r\n"}, + {name: "lowercase", request: "GET / http/1.1\r\nHost: h\r\n\r\n"}, + {name: "not http", request: "GET / BANANA\r\nHost: h\r\n\r\n"}, + {name: "space in target", request: "GET /a b HTTP/1.1\r\nHost: h\r\n\r\n"}, + } { + t.Run(test.name, func(t *testing.T) { + var handled bool + var sm MuxSlice + sm.Handle("/", func(ex *Exchange) { handled = true }) + sm.Handle("/a", func(ex *Exchange) { handled = true }) + conn := newConn(test.request) + conn.Hangup() + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) + if err := Handle(exch, &sm, nopBackoff); err == nil { + t.Error("want error on unsupported protocol, got nil") + } + if handled { + t.Error("handler must not run on unsupported protocol") + } + const want = "HTTP/1.1 505 HTTP Version Not Supported\r\n\r\n" + if got := conn.ViewWritten(); got != want { + t.Errorf("want %q, got %q", want, got) + } + }) + } +} + +// HTTP/1.0 predates HTTP/1.1 but is still served: only the connection is not +// kept alive. It must not be swept up by the 505 gate. +func TestHandleHTTP10Served(t *testing.T) { + var sm MuxSlice + sm.Handle("/", func(ex *Exchange) { ex.WriteHeader(200) }) + conn := serve(t, "GET / HTTP/1.0\r\nHost: h\r\n\r\n", &sm) + const want = "HTTP/1.1 200 OK\r\n\r\n" + if got := conn.ViewWritten(); got != want { + t.Errorf("want %q, got %q", want, got) + } +} + +// No registered handler must yield 404, not an empty response. +func TestHandleNoHandler(t *testing.T) { + var sm MuxSlice + sm.Handle("GET /", func(ex *Exchange) { t.Error("handler must not run") }) + conn := serve(t, "GET /nowhere HTTP/1.1\r\nHost: h\r\n\r\n", &sm) + const want = "HTTP/1.1 404 Not Found\r\n\r\n" + if got := conn.ViewWritten(); got != want { + t.Errorf("want %q, got %q", want, got) + } +} + +// A handler that writes nothing must still produce a valid response. +func TestHandleSilentHandler(t *testing.T) { + var sm MuxSlice + sm.Handle("/", func(ex *Exchange) {}) + conn := serve(t, "GET / HTTP/1.1\r\nHost: h\r\n\r\n", &sm) + const want = "HTTP/1.1 200 OK\r\n\r\n" + if got := conn.ViewWritten(); got != want { + t.Errorf("want %q, got %q", want, got) + } +} + +// Body bytes arriving in the same segment as the header must be readable. +var _ io.ReadWriteCloser = (*ExchangeRW)(nil) + +// ExchangeRW writes the response body and reads the request body, so it may be +// handed to code that wants an io.ReadWriter. +func TestExchangeRW(t *testing.T) { + const body = "hello" + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*128), RequestBufferLim: 128}) + var rw ExchangeRW + exch.ReadWriter(&rw) + + n, err := io.WriteString(&rw, body) + if err != nil { + t.Fatal(err) + } + if n != len(body) { + t.Errorf("want %d bytes written, got %d", len(body), n) + } + const want = "HTTP/1.1 200 OK\r\n\r\n" + body + if got := conn.ViewWritten(); got != want { + t.Errorf("want %q, got %q", want, got) + } +} + +// The exchange is pooled and reused: a handle kept past the request it was +// taken from must fail instead of reaching the next request's connection. +func TestExchangeRWOutlivesExchange(t *testing.T) { + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*128), RequestBufferLim: 128}) + var rw ExchangeRW + exch.ReadWriter(&rw) + if !rw.IsValid() { + t.Fatal("want a fresh handle to be valid") + } + exch.Release() + + if rw.IsValid() { + t.Error("want handle invalidated by release") + } + if _, err := rw.Write([]byte("late")); err == nil { + t.Error("want error writing through a released exchange, got nil") + } + if _, err := rw.Read(make([]byte, 4)); err == nil { + t.Error("want error reading through a released exchange, got nil") + } + if got := conn.ViewWritten(); strings.Contains(got, "late") { + t.Errorf("late write reached the connection: %q", got) + } +} + +func TestExchangeReadBody(t *testing.T) { + const body = "message body" + var got string + var readErr error + var sm MuxSlice + sm.Handle("POST /", func(ex *Exchange) { + dst := make([]byte, len(body)) + n, err := ex.ReadBody(dst) + got, readErr = string(dst[:n]), err + ex.WriteHeader(200) + }) + serve(t, "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 12\r\n\r\n"+body, &sm) + + if readErr != nil { + t.Fatal(readErr) + } + if got != body { + t.Errorf("want body %q, got %q", body, got) + } +} + +// SetHeader must budget every byte it writes: colon, CRLF, and the CRLF that +// FlushHeader appends after the last field. Buffers that fit all but the last +// byte must be refused, never overrun. +func TestExchangeStageOKAndFail(t *testing.T) { + // Long enough that the buffers under test clear the smallest a header + // buffer may be, while still being sized to the byte around the field. + const key, value = "X-A-Header-Field-Key", "a-header-field-value" + const field = len(key) + len(value) + len(":\r\n") + const numHeaderCap = 4 + for _, bufLen := range []int{field + 2, field + 1, field} { + t.Run("buffer"+strconv.Itoa(bufLen), func(t *testing.T) { + conn := newConn("") + exch := new(Exchange) + exch.Configure(ExchangeConfig{ + RawBuf: make([]byte, bufLen), + RequestBufferLim: bufLen, + NumHeaderKVCap: numHeaderCap, + }) + if !exch.Acquire(conn) { + t.Fatal("fresh exchange failed to acquire connection") + } + set := exch.StageHeader(key, value) + n, err := exch.FlushHeader() + + want := "HTTP/1.1 200 OK\r\n" + if set { + want += key + ":" + value + "\r\n" + want += "\r\n" + } else { + if err != lneto.ErrBufferFull || n != 0 { + t.Fatal("expected buffer full and no data written:", err, n) + } + want = "" + } + if got := conn.ViewWritten(); got != want { + t.Errorf("want %q, got %q", want, got) + } + if wantSet := bufLen >= field+2; set != wantSet { + t.Errorf("want SetHeader=%v, got %v", wantSet, set) + } + }) + } +} + +// Handle never closes the connection, on any outcome: the caller owns it so +// that error policy and connection reuse stay the caller's decision. +func TestHandleLeavesConnOpen(t *testing.T) { + var sm MuxSlice + sm.Handle("GET /", staticPage(t, "ok")) + for _, test := range []struct { + name string + request string + }{ + {name: "served", request: "GET / HTTP/1.1\r\nHost: h\r\n\r\n"}, + {name: "404", request: "GET /nowhere HTTP/1.1\r\nHost: h\r\n\r\n"}, + {name: "rejected no http version", request: "GET /\r\nHost: h\r\n\r\n"}, + {name: "rejected parse error", request: "GET / HTTP/1.1\r\nBadFieldNoColon\r\n\r\n"}, + } { + t.Run(test.name, func(t *testing.T) { + conn := newConn(test.request) + conn.Hangup() + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) + Handle(exch, &sm, nopBackoff) + if conn.IsClosed() { + t.Errorf("Handle closed the connection for %q", test.request) + } + }) + } +} + +// Hijacking hands the connection to the handler, so Release must not close it. +// Ownership must not carry over: the next connection the exchange serves is +// the router's again and must be closed on Release. +func TestExchangeHijackOwnership(t *testing.T) { + var sm MuxSlice + var hijackErr error + sm.Handle("GET /", func(ex *Exchange) { + _, _, hijackErr = ex.HijackRaw(nil) + }) + first := newConn("GET / HTTP/1.1\r\nHost: h\r\n\r\n") + first.Hangup() + exch := newExchange(t, first, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) + if err := Handle(exch, &sm, nopBackoff); err != nil { + t.Fatal(err) + } + if hijackErr != nil { + t.Fatal(hijackErr) + } + exch.Release() + if first.IsClosed() { + t.Error("hijacked connection must stay open after Release") + } + + second := newConn("") + if !exch.Acquire(second) { + t.Fatal("released exchange must be acquirable") + } + exch.Release() + if !second.IsClosed() { + t.Error("connection must be closed on Release: hijack of a previous request must not carry over") + } +} + +// Idle peer policy belongs to the connection: Handle keeps retrying an empty +// read until the conn itself reports failure, so a stalled peer ends the +// exchange through the conn's deadline instead of pinning the exchange. +func TestHandleIdlePeerEndsOnConnDeadline(t *testing.T) { + var sm MuxSlice + sm.Handle("/", func(ex *Exchange) { t.Error("handler must not run on partial request") }) + conn := newConn("GET / HTTP") // Peer stalls mid request line, never hangs up. + conn.SetDeadline(time.Now().Add(10 * time.Millisecond)) + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) + + done := make(chan error, 1) + go func() { done <- Handle(exch, &sm, nopBackoff) }() + select { + case err := <-done: + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("want connection deadline error, got %v", err) + } + case <-time.After(time.Second): + t.Fatal("Handle ignored the connection deadline") + } +} + +// A body must never reach the wire without its header: if flushing the header +// fails, Write must report the failure and send nothing. +func TestExchangeWriteHeaderFlushFails(t *testing.T) { + const body = "body" + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*128), RequestBufferLim: 128}) + conn.FailWrites(1) // Status line write fails, body write would succeed. + + n, err := exch.WriteBody([]byte(body)) + if err == nil { + t.Error("want error when header flush fails, got nil") + } + if n != 0 { + t.Errorf("want 0 bytes written, got %d", n) + } + if got := conn.ViewWritten(); got != "" { + t.Errorf("want nothing on the wire, got %q", got) + } + // Writes after a failed header stay failed: the response is unrecoverable, + // a body without its header would corrupt the stream. + if _, err = exch.WriteBody([]byte(body)); err == nil { + t.Error("want error on write after failed header flush, got nil") + } + if got := conn.ViewWritten(); got != "" { + t.Errorf("want nothing on the wire, got %q", got) + } +} + +func TestExchangeSetHeaderInt(t *testing.T) { + for _, test := range []struct { + value int64 + base int + want string // Header block emitted after the status line. + }{ + {value: 1234, base: 10, want: "N:1234\r\n\r\n"}, + {value: 0, base: 10, want: "N:0\r\n\r\n"}, + {value: -42, base: 10, want: "N:-42\r\n\r\n"}, + {value: 255, base: 16, want: "N:ff\r\n\r\n"}, + {value: 9223372036854775807, base: 10, want: "N:9223372036854775807\r\n\r\n"}, + {value: -9223372036854775808, base: 10, want: "N:-9223372036854775808\r\n\r\n"}, + {value: 1, base: 2, want: "\r\n"}, // Below base 10, dropped. + {value: 1, base: 37, want: "\r\n"}, // Above base 36, dropped. + } { + name := strconv.FormatInt(test.value, 10) + "_base" + strconv.Itoa(test.base) + t.Run(name, func(t *testing.T) { + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*256), RequestBufferLim: 256}) + exch.StageHeaderInt("N", test.value, test.base) + exch.WriteHeader(200) + got, _ := strings.CutPrefix(conn.ViewWritten(), "HTTP/1.1 200 OK\r\n") + if got != test.want { + t.Errorf("want %q, got %q", test.want, got) + } + }) + } +} + +// SetHeaderInt must format into the response buffer without allocating. +func TestExchangeSetHeaderIntNoAlloc(t *testing.T) { + exch := newExchange(t, newConn(""), ExchangeConfig{RawBuf: make([]byte, 2*256), RequestBufferLim: 256}) + allocs := testing.AllocsPerRun(100, func() { + exch.StageHeaderInt("Content-Length", 1234567890, 10) + }) + if allocs != 0 { + t.Errorf("SetHeaderInt allocated %v times, want 0", allocs) + } +} + +// The Mux matches on the request path: a query string must not defeat routing. +func TestHandleMuxOnPath(t *testing.T) { + var sm MuxSlice + var gotPath, gotQuery string + sm.Handle("GET /search", func(ex *Exchange) { + gotPath = string(ex.RequestPath()) + rawkey, rawval, rest := httpraw.NextQueryPair(ex.RequestQuery()) + for rawkey != nil { + gotQuery += string(rawkey) + "=" + string(rawval) + ";" + rawkey, rawval, rest = httpraw.NextQueryPair(rest) + } + ex.WriteHeader(200) + }) + conn := serve(t, "GET /search?q=go&n=1 HTTP/1.1\r\nHost: h\r\n\r\n", &sm) + + if gotPath != "/search" { + t.Errorf("want path %q, got %q", "/search", gotPath) + } + if gotQuery != "q=go;n=1;" { + t.Errorf("want query %q, got %q", "q=go;n=1;", gotQuery) + } + if got := conn.ViewWritten(); !strings.HasPrefix(got, "HTTP/1.1 200 OK\r\n") { + t.Errorf("want the handler to have run, got %q", got) + } +} + +func TestExchangeAppendQuery(t *testing.T) { + for _, test := range []struct { + uri string + key string + decoded bool + want string + wantPresent bool + }{ + {uri: "/x?q=go", key: "q", want: "go", wantPresent: true}, + {uri: "/x?q=go&n=1", key: "n", want: "1", wantPresent: true}, + {uri: "/x?a=1&a=2", key: "a", want: "1", wantPresent: true}, // First match wins. + {uri: "/x?q=go", key: "nope", want: "", wantPresent: false}, // Absent. + {uri: "/x", key: "q", want: "", wantPresent: false}, // No query at all. + {uri: "/x?debug&q=go", key: "debug", want: "", wantPresent: true}, // Flag: present, no value. + {uri: "/x?q=", key: "q", want: "", wantPresent: true}, // Present, empty. + // Decoding is opt-in and applies to the value only. + {uri: "/x?q=hello%20world", key: "q", want: "hello%20world", wantPresent: true}, + {uri: "/x?q=hello%20world", key: "q", decoded: true, want: "hello world", wantPresent: true}, + {uri: "/x?q=a+b", key: "q", want: "a+b", wantPresent: true}, + {uri: "/x?q=a+b", key: "q", decoded: true, want: "a b", wantPresent: true}, + // Keys are matched decoded: "a b" cannot appear raw. + {uri: "/x?a%20b=c", key: "a b", want: "c", wantPresent: true}, + {uri: "/x?a+b=c", key: "a b", want: "c", wantPresent: true}, + // Malformed escapes: a bad key is skipped, a bad value is not returned. + {uri: "/x?%zz=1&q=go", key: "q", want: "go", wantPresent: true}, + {uri: "/x?q=%zz", key: "q", decoded: true, want: "", wantPresent: false}, + {uri: "/x?q=%zz", key: "q", want: "%zz", wantPresent: true}, // Undecoded, passed through. + } { + // The path is the same for every case: name them by what differs, and + // keep the '/' out so the name stays a single -run element. + name := strings.TrimPrefix(test.uri, "/x") + if name == "" { + name = "noquery" + } + name += "_" + test.key + if test.decoded { + name += "_decoded" + } + t.Run(name, func(t *testing.T) { + var sm MuxSlice + var got string + var present bool + sm.Handle("/x", func(ex *Exchange) { + var value []byte + value, present = ex.RequestQueryAppend(nil, test.key, test.decoded) + got = string(value) + }) + serve(t, "GET "+test.uri+" HTTP/1.1\r\nHost: h\r\n\r\n", &sm) + + if present != test.wantPresent { + t.Errorf("want present=%v, got %v", test.wantPresent, present) + } + if got != test.want { + t.Errorf("want %q, got %q", test.want, got) + } + }) + } +} + +// AppendQuery appends to dst, leaving what was already there untouched, and +// does not allocate when dst has the capacity. +func TestExchangeAppendQueryReusesBuffer(t *testing.T) { + var sm MuxSlice + var got string + var allocs float64 + dst := make([]byte, 0, 64) + sm.Handle("/x", func(ex *Exchange) { + var value []byte + allocs = testing.AllocsPerRun(50, func() { + value, _ = ex.RequestQueryAppend(dst[:len("prefix:")], "q", true) + }) + got = string(value) // Conversion allocates, keep it out of the measurement. + }) + copy(dst[:cap(dst)], "prefix:") + serve(t, "GET /x?q=hello%20world HTTP/1.1\r\nHost: h\r\n\r\n", &sm) + + if got != "prefix:hello world" { + t.Errorf("want %q, got %q", "prefix:hello world", got) + } + if allocs != 0 { + t.Errorf("AppendQuery allocated %v times into a buffer with capacity, want 0", allocs) + } +} + +// formString renders a form as "key=value" pairs joined by '|', a pair with no +// value shown as the bare key. +func formString(f *httpraw.Form) string { + var sb strings.Builder + for i := 0; i < f.Len(); i++ { + if i > 0 { + sb.WriteByte('|') + } + key, value := f.Pair(i) + sb.Write(key) + if value != nil { + sb.WriteByte('=') + sb.Write(value) + } + } + return sb.String() +} + +const formType = "Content-Type: application/x-www-form-urlencoded\r\n" + +// formPair is one key/value pair of a form body. flag sends the key bare, with +// no '=', which parses back as a nil value: distinct from a present but empty +// one, unlike http.FormValue. +type formPair struct { + key, value string + flag bool +} + +// appendForm renders pairs as a urlencoded body, joining them with '&'. +func appendForm(dst []byte, pairs []formPair) []byte { + for i, pair := range pairs { + if i > 0 { + dst = append(dst, '&') + } + dst = append(dst, pair.key...) + if !pair.flag { + dst = append(dst, '=') + dst = append(dst, pair.value...) + } + } + return dst +} + +func TestExchangeRequestParseForm(t *testing.T) { + for _, test := range []struct { + name string + formVals []formPair + // wantVals defaults to formVals: set it only where what comes back out + // differs from what went in, as decoding makes it. + wantVals []formPair + target string // Request target, defaults to "/f". + contentType string // Media type, defaults to application/x-www-form-urlencoded. + noContentType bool // Send no Content-Type field at all. + noContentLength bool // Send no Content-Length field: no body at all, RFC 9112 6.3. + extraHeaders string // Header fields sent verbatim, each CRLF terminated. + callDecode bool + bufsize int // Defaults to 64. + wantErr error + }{ + { + name: "pairs", + formVals: []formPair{{key: "a", value: "1"}, {key: "b", value: "2"}, {key: "c", value: "3"}}, + }, { + name: "long value", + formVals: []formPair{{key: "a", value: ""}, {key: "k", value: strings.Repeat("k", 20)}}, + }, { + // A flag and an empty value stay distinguishable, unlike http.FormValue. + name: "flag and empty", + formVals: []formPair{{key: "a", flag: true}, {key: "b", value: ""}}, + }, { + // Decoding is the caller's call: untouched without it. + name: "left encoded", + formVals: []formPair{{key: "n", value: "a%20b"}}, + }, { + // Decode reaches both keys and values. + name: "decoded", + formVals: []formPair{{key: "a+b", value: "c%20d"}, {key: "e", value: "f%2B"}}, + wantVals: []formPair{{key: "a b", value: "c d"}, {key: "e", value: "f+"}}, + callDecode: true, + }, { + name: "media type parameters", + formVals: []formPair{{key: "a", value: "1"}}, + contentType: "application/x-www-form-urlencoded; charset=utf-8", + }, { + // Only the body is parsed: the query string is not folded in. + name: "query not folded", + formVals: []formPair{{key: "a", value: "1"}}, + target: "/f?q=go", + }, { + name: "no content length", + noContentLength: true, + }, { + name: "wrong media type", + formVals: []formPair{{key: "a", value: "1"}}, + contentType: "text/plain", + wantErr: errNotFormEncoded, + }, { + name: "no media type", + formVals: []formPair{{key: "a", value: "1"}}, + noContentType: true, + wantErr: errNotFormEncoded, + }, { + // The coding is refused on the field alone, so the body stays off. + name: "chunked", + noContentLength: true, + extraHeaders: "Transfer-Encoding: chunked\r\n", + wantErr: errUnsupportedTransferCoding, + }, { + name: "body larger than buffer", + formVals: []formPair{{key: "a", value: "1"}, {key: "b", value: "2"}, {key: "c", value: "3"}}, + bufsize: 4, + wantErr: lneto.ErrShortBuffer, + }, + } { + t.Run(test.name, func(t *testing.T) { + bufSize := test.bufsize + if bufSize == 0 { + bufSize = 64 + } + target := test.target + if target == "" { + target = "/f" + } + contentType := test.contentType + if contentType == "" { + contentType = "application/x-www-form-urlencoded" + } + wantVals := test.wantVals + if wantVals == nil { + wantVals = test.formVals + } + body := appendForm(nil, test.formVals) + + var builder strings.Builder + builder.WriteString("POST ") + builder.WriteString(target) + builder.WriteString(" HTTP/1.1\r\nHost: h\r\n") + if !test.noContentType { + builder.WriteString("Content-Type: ") + builder.WriteString(contentType) + builder.WriteString("\r\n") + } + builder.WriteString(test.extraHeaders) + if !test.noContentLength { + builder.WriteString("Content-Length: ") + builder.WriteString(strconv.Itoa(len(body))) + builder.WriteString("\r\n") + } + builder.WriteString("\r\n") + builder.Write(body) + + var form httpraw.Form + var gotErr error + var sm MuxSlice + sm.Reset(1) + sm.Handle("/f", func(exch *Exchange) { + gotErr = exch.RequestParseForm(&form, make([]byte, bufSize)) + if gotErr == nil && test.callDecode { + gotErr = form.Decode() + } + }) + serve(t, builder.String(), &sm) + + if gotErr != test.wantErr { + t.Fatalf("want error %v, got %v", test.wantErr, gotErr) + } else if test.wantErr != nil { + return // Nothing is promised about the form on failure. + } + if form.Len() != len(wantVals) { + t.Fatalf("want %d pairs parsed, got %d: %q", len(wantVals), form.Len(), formString(&form)) + } + for i, want := range wantVals { + key, value := form.Pair(i) + if b2s(key) != want.key { + t.Errorf("pair %d: want key %q, got %q", i, want.key, key) + } + switch { + case want.flag && value != nil: + t.Errorf("pair %d: want no value, got %q", i, value) + case !want.flag && value == nil: + t.Errorf("pair %d: want value %q, got no value", i, want.value) + case !want.flag && b2s(value) != want.value: + t.Errorf("pair %d: want value %q, got %q", i, want.value, value) + } + } + }) + } +} + +// A body arriving after the header, in its own segment, must still be parsed whole. +func TestExchangeRequestParseFormSplit(t *testing.T) { + conn := newConn("POST /f HTTP/1.1\r\nHost: h\r\n" + formType + "Content-Length: 11\r\n\r\na=1&") + conn.AddSegment("b=2&c=3") + conn.Hangup() + var form httpraw.Form + var gotErr error + var sm MuxSlice + sm.Reset(1) + sm.Handle("/f", func(exch *Exchange) { + gotErr = exch.RequestParseForm(&form, make([]byte, 64)) + }) + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) + if err := Handle(exch, &sm, nopBackoff); err != nil { + t.Fatal(err) + } + if gotErr != nil { + t.Fatal(gotErr) + } + if got := formString(&form); got != "a=1|b=2|c=3" { + t.Errorf("want %q, got %q", "a=1|b=2|c=3", got) + } +} + +// Decode is the caller's call, and it must reach both keys and values. +func TestExchangeRequestParseFormDecode(t *testing.T) { + var form httpraw.Form + var sm MuxSlice + sm.Reset(1) + sm.Handle("/f", func(exch *Exchange) { + if err := exch.RequestParseForm(&form, make([]byte, 64)); err != nil { + t.Error(err) + } else if err = form.Decode(); err != nil { + t.Error(err) + } + }) + serve(t, "POST /f HTTP/1.1\r\nHost: h\r\n"+formType+"Content-Length: 16\r\n\r\na+b=c%20d&e=f%2B", &sm) + if got := formString(&form); got != "a b=c d|e=f+" { + t.Errorf("want %q, got %q", "a b=c d|e=f+", got) + } +} + +// partBuffer is a sink that keeps a part's content in memory and records that +// [Exchange.ReadMultiparts] closed it. +type partBuffer struct { + content []byte + closed bool +} + +func (p *partBuffer) Write(b []byte) (int, error) { + if p.closed { + return 0, errors.New("write to closed part sink") + } + p.content = append(p.content, b...) + return len(b), nil +} + +func (p *partBuffer) Close() error { p.closed = true; return nil } + +// multipartPart is one part of a multipart/form-data body. discard makes the +// sink factory refuse it, exercising [Exchange.ReadMultiparts]' discard path. +type multipartPart struct { + name, filename, content string + discard bool +} + +// appendMultipart renders parts as a multipart/form-data body delimited by +// boundary, closed off with the terminating delimiter. +func appendMultipart(dst []byte, boundary string, parts []multipartPart) []byte { + for _, part := range parts { + dst = append(dst, "--"+boundary+"\r\n"...) + dst = append(dst, `Content-Disposition: form-data; name="`+part.name+`"`...) + if part.filename != "" { + dst = append(dst, `; filename="`+part.filename+`"`...) + } + dst = append(dst, "\r\n\r\n"...) + dst = append(dst, part.content...) + dst = append(dst, "\r\n"...) + } + return append(dst, "--"+boundary+"--\r\n"...) +} + +// serveMultipart serves request to a handler that streams its multipart body +// with [Exchange.ReadMultiparts] over a buffer of bufSize bytes. Bytes past +// each offset in segmentAt are delivered on later reads, so the parser must +// compact and read more to see them. discard names the parts whose sink is +// refused. +func serveMultipart(t *testing.T, request string, bufSize int, discard []string, segmentAt []int) ([]MultipartSink, error) { + t.Helper() + prev := 0 + for _, off := range segmentAt { + if off < prev || off > len(request) { + t.Fatalf("segment offset %d out of order or past the %d byte request", off, len(request)) + } + prev = off + } + conn := newConn(request[:firstOr(segmentAt, len(request))]) + for i, off := range segmentAt { + end := len(request) + if i+1 < len(segmentAt) { + end = segmentAt[i+1] + } + conn.AddSegment(request[off:end]) + } + conn.Hangup() + var parts []MultipartSink + var gotErr error + var sm MuxSlice + sm.Reset(1) + sm.Handle("/f", func(exch *Exchange) { + newSink := func(hdr *httpraw.MultipartHeader) io.WriteCloser { + if slices.Contains(discard, string(hdr.Name)) { + return nil // Discard this part's content. + } + return new(partBuffer) + } + parts, gotErr = exch.ReadMultiparts(parts, make([]byte, bufSize), newSink) + }) + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) + if err := Handle(exch, &sm, nopBackoff); err != nil { + t.Fatal(err) + } + return parts, gotErr +} + +func firstOr(s []int, or int) int { + if len(s) == 0 { + return or + } + return s[0] +} + +// checkParts asserts the part header and streamed content of every sink, that a +// discarded part has no sink at all, and that no sink was left open, which would +// hide a part that never ended. +func checkParts(t *testing.T, got []MultipartSink, want []multipartPart) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("want %d parts, got %d: %q", len(want), len(got), partsString(t, got)) + } + for i, want := range want { + part := &got[i] + if string(part.Header.Name) != want.name { + t.Errorf("part %d: want name %q, got %q", i, want.name, part.Header.Name) + } + if string(part.Header.Filename) != want.filename { + t.Errorf("part %d: want filename %q, got %q", i, want.filename, part.Header.Filename) + } + if want.discard { + if part.Sink != nil { + t.Errorf("part %d: want no sink for a discarded part, got one", i) + } + continue + } + if part.Sink == nil { + t.Errorf("part %d: want content %q, got no sink", i, want.content) + continue + } + sink := part.Sink.(*partBuffer) + if !sink.closed { + t.Errorf("part %d (%q): sink left open", i, want.name) + } + if string(sink.content) != want.content { + t.Errorf("part %d (%q): want content %q, got %q", i, want.name, want.content, sink.content) + } + } +} + +// partsString renders parts as "name=content" joined by '|', a file part shown +// as "name(filename)=content" and a discarded one as "name=". Fails the +// test if a sink was left open, which would hide a part that never ended. +func partsString(t *testing.T, parts []MultipartSink) string { + t.Helper() + var sb strings.Builder + for i := range parts { + if i > 0 { + sb.WriteByte('|') + } + part := &parts[i] + sb.Write(part.Header.Name) + if len(part.Header.Filename) > 0 { + sb.WriteByte('(') + sb.Write(part.Header.Filename) + sb.WriteByte(')') + } + sb.WriteByte('=') + if part.Sink == nil { + sb.WriteString("") + continue + } + sink := part.Sink.(*partBuffer) + if !sink.closed { + t.Errorf("part %q: sink left open", part.Header.Name) + } + sb.Write(sink.content) + } + return sb.String() +} + +// mpTeaser is content that teases the parser with delimiter prefixes that never +// complete, so a compaction that fails to hold the tail back drops part of it. +var mpTeaser = strings.Repeat("\r\n--xy", 16) + strings.Repeat("A", 100) + "\r\n--xyy" + +func TestExchangeReadMultiparts(t *testing.T) { + for _, test := range []struct { + name string + boundary string // Defaults to "xyz". + parts []multipartPart // The body sent. + // wantParts defaults to parts: set it only where what comes back out + // differs from what went in. A wantErr case reports no parts at all. + wantParts []multipartPart + // segmentAt are offsets into the body where a later read begins. The + // request header always arrives in the first read. + segmentAt []int + bufsize int // Defaults to 128. + wantErr error + }{ + { + // Names, filenames and content of every part, over a body split so + // that a part straddles two reads and the parser must compact and + // read more. The boundary opens with dashes of its own, which the + // delimiter's leading "--" must not be confused with. + name: "parts and files", + boundary: "--xyz", + parts: []multipartPart{ + {name: "caption", content: "hi there"}, + {name: "photo", filename: "beach.png", content: "\x89PNG\r\n\x00"}, + }, + segmentAt: []int{89}, // Inside the second part's header. + }, { + // A part longer than the buffer must come out whole: every + // compaction has to keep the tail NextBody held back, or content + // that looks like the start of a delimiter is dropped. The header's + // Name must survive those reads too. + name: "part larger than buffer", + parts: []multipartPart{{name: "blob", content: mpTeaser}}, + segmentAt: []int{0, 30}, // Header alone, then 30 bytes of body. + bufsize: 64, + }, { + // A nil sink discards a part's content without losing its place in + // the body: the parts around it must still arrive whole. + name: "discards part", + parts: []multipartPart{ + {name: "keep", content: "kept"}, + {name: "huge", filename: "big.bin", content: strings.Repeat("Z", 200), discard: true}, + {name: "also", content: "kept too"}, + }, + bufsize: 96, + }, { + // A part header that does not fit the buffer cannot be completed by + // reading more, so the caller is told instead of spinning. + name: "header larger than buffer", + parts: []multipartPart{{name: strings.Repeat("n", 64), content: "v"}}, + bufsize: 32, + wantErr: lneto.ErrShortBuffer, + }, { + // A buffer too small to ever outgrow a delimiter is a caller error, + // refused before any of the body is read. + name: "buffer unusable", + parts: []multipartPart{{name: "a", content: "v"}}, + bufsize: len("\r\n--xyz"), + wantErr: lneto.ErrInvalidConfig, + }, + } { + t.Run(test.name, func(t *testing.T) { + boundary := test.boundary + if boundary == "" { + boundary = "xyz" + } + bufSize := test.bufsize + if bufSize == 0 { + bufSize = 128 + } + wantParts := test.wantParts + if wantParts == nil { + wantParts = test.parts + } + var discard []string + for _, part := range test.parts { + if part.discard { + discard = append(discard, part.name) + } + } + head := "POST /f HTTP/1.1\r\nHost: h\r\nContent-Type: multipart/form-data; boundary=" + boundary + "\r\n\r\n" + body := appendMultipart(nil, boundary, test.parts) + segmentAt := make([]int, len(test.segmentAt)) + for i, off := range test.segmentAt { + if off < 0 || off >= len(body) { + t.Fatalf("segment offset %d is not inside the %d byte body", off, len(body)) + } + segmentAt[i] = len(head) + off + } + + parts, err := serveMultipart(t, head+string(body), bufSize, discard, segmentAt) + if err != test.wantErr { + t.Fatalf("want error %v, got %v", test.wantErr, err) + } + if test.wantErr != nil { + // Nothing parsed is promised on failure, and a part reported + // for a header that never parsed is a part the caller cannot + // act on. + if len(parts) != 0 { + t.Errorf("want no parts reported, got %d: %q", len(parts), partsString(t, parts)) + } + return + } + checkParts(t, parts, wantParts) + }) + } +} + +// A request that is not multipart, or whose boundary is missing, must be refused. +func TestExchangeRequestParseMultipartRejects(t *testing.T) { + for _, test := range []struct { + contentType string + wantErr bool + }{ + {contentType: "multipart/form-data; boundary=xyz"}, + {contentType: "application/x-www-form-urlencoded", wantErr: true}, + {contentType: "multipart/form-data", wantErr: true}, // Boundary is required. + {contentType: "", wantErr: true}, + } { + name := test.contentType + if name == "" { + name = "no content type" + } + t.Run(name, func(t *testing.T) { + var gotErr error + var sm MuxSlice + sm.Reset(1) + sm.Handle("/f", func(exch *Exchange) { + _, gotErr = exch.RequestMultipart() + }) + request := "POST /f HTTP/1.1\r\nHost: h\r\n" + if test.contentType != "" { + request += "Content-Type: " + test.contentType + "\r\n" + } + serve(t, request+"\r\n", &sm) + if (gotErr != nil) != test.wantErr { + t.Errorf("want error %v, got %v", test.wantErr, gotErr) + } + }) + } +} + +// A request from a browser carries around twenty header fields. Serving one +// must not depend on how many fields the parser happens to have room for: the +// exchange's buffer is the memory the caller granted, and the field table comes +// out of it. +func TestHandleBrowserSizedRequest(t *testing.T) { + const wantMode = "navigate" + request := "GET /echo HTTP/1.1\r\nHost: lneto.test\r\n" + + "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\n" + + "Accept-Language: en-US,en;q=0.5\r\nAccept-Encoding: gzip, deflate, br\r\n" + + "Upgrade-Insecure-Requests: 1\r\nSec-Fetch-Dest: document\r\nSec-Fetch-Site: none\r\n" + + "Sec-Ch-Ua: \"Chromium\";v=\"120\"\r\nCache-Control: max-age=0\r\nDnt: 1\r\n" + + "Referer: https://lneto.test/index.html\r\nCookie: session=abcdef0123456789; theme=dark\r\n" + + "X-Trace: 0123456789abcdef\r\nX-Client: bench\r\nX-Seq: 42\r\nX-Tag: alpha\r\n" + + "X-Nonce: cafebabe\r\nX-Mode: " + wantMode + "\r\n\r\n" + + var gotMode string + var fields int + var sm MuxSlice + sm.Reset(1) + sm.Handle("GET /echo", func(exch *Exchange) { + gotMode = string(exch.RequestHeader("X-Mode")) + exch.RequestHeaderRaw().ForEach(func(key, value []byte) bool { + fields++ + return true + }) + }) + conn := newConn(request) + conn.Hangup() + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*8192), RequestBufferLim: 8192}) + if err := Handle(exch, &sm, nopBackoff); err != nil { + t.Fatalf("serving a browser sized request: %s", err) + } + if gotMode != wantMode { + t.Errorf("last header field read back as %q, want %q", gotMode, wantMode) + } + const sent = 19 + if fields < sent { + t.Errorf("handler saw %d header fields, request carried %d", fields, sent) + } +} + +// A request larger than the exchange has room for must be answered, not +// dropped: the peer learns its request was too large instead of seeing the +// connection go away. Either bound may be the one it ran into, and both are +// reported to the caller. +func TestHandleRequestTooLargeAnswers431(t *testing.T) { + var request strings.Builder + request.WriteString("GET /echo HTTP/1.1\r\nHost: lneto.test\r\n") + for i := range 512 { + request.WriteByte('H') + request.WriteString(strconv.Itoa(i)) + request.WriteString(":v\r\n") + } + request.WriteString("\r\n") + + for _, test := range []struct { + name string + cfg ExchangeConfig + wantErr error + }{ + { + // Room for every byte of the block, but not for its fields. The + // field table only bounds the request when growth is refused: + // otherwise it is the size the parser starts from, not a limit. + name: "more fields than the table holds", + cfg: ExchangeConfig{ + RawBuf: make([]byte, 16*1024), RequestBufferLim: 8 * 1024, + NumHeaderKVCap: 16, NoRequestBufferGrowth: true, + }, + wantErr: httpraw.ErrHeaderTooMany, + }, + { + // Room for the fields, but not for the bytes they arrive in. + name: "more bytes than the buffer holds", + cfg: ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024, NumHeaderKVCap: 1024, NoRequestBufferGrowth: true}, + wantErr: httpraw.ErrBufferExhausted, + }, + } { + t.Run(test.name, func(t *testing.T) { + var served bool + var sm MuxSlice + sm.Reset(1) + sm.Handle("GET /echo", func(exch *Exchange) { served = true }) + conn := newConn(request.String()) + conn.Hangup() + exch := newExchange(t, conn, test.cfg) + err := Handle(exch, &sm, nopBackoff) + if !errors.Is(err, test.wantErr) { + t.Fatalf("want %v reported to the caller, got %v", test.wantErr, err) + } + if served { + t.Fatal("handler ran on a request the parser could not hold") + } + got := conn.ViewWritten() + if !strings.HasPrefix(got, "HTTP/1.1 431 ") { + t.Errorf("want a 431 answer, got %q", firstLine(got)) + } + }) + } +} + +func firstLine(s string) string { + if before, _, ok := strings.Cut(s, "\r\n"); ok { + return before + } + return s +} + +// A body that already arrived alongside the request header must be handed over +// without touching the connection again. A peer that sent a whole request and +// is waiting for its answer sends nothing more, so a read for bytes already in +// hand blocks until the connection's deadline, or forever without one. +func TestExchangeReadBodyDoesNotReadPastWhatArrived(t *testing.T) { + const body = "message body" + dst := make([]byte, 64) // Deliberately larger than the body. + var got string + var readErr error + var sm MuxSlice + sm.Reset(1) + sm.Handle("POST /", func(ex *Exchange) { + n, err := ex.ReadBody(dst) + got, readErr = string(dst[:n]), err + ex.WriteHeader(200) + }) + // The peer is still there, waiting to be answered: a read for bytes it is + // not going to send blocks, exactly as it does on a socket. + conn := &blockingConn{request: "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 12\r\n\r\n" + body} + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) + done := make(chan struct{}) + go func() { + defer close(done) + Handle(exch, &sm, nopBackoff) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("ReadBody blocked waiting for a body that had already arrived") + } + if readErr != nil { + t.Fatal(readErr) + } + if got != body { + t.Errorf("want body %q, got %q", body, got) + } +} + +// blockingConn delivers a request and then blocks on reads, the way a peer +// awaiting its answer does. Writes are discarded. +type blockingConn struct { + request string + read int + blocked chan struct{} +} + +func (c *blockingConn) Read(b []byte) (int, error) { + if c.read >= len(c.request) { + if c.blocked == nil { + c.blocked = make(chan struct{}) + } + <-c.blocked // Nothing more is coming, and nothing unblocks this. + return 0, io.EOF + } + n := copy(b, c.request[c.read:]) + c.read += n + return n, nil +} + +func (c *blockingConn) Write(b []byte) (int, error) { return len(b), nil } +func (c *blockingConn) Close() error { return nil } + +// Field names are case insensitive, RFC 9110 5.1, and HTTP/2 mandates lowercase +// ones, so an h2-to-h1 proxy sends "content-type". The field must be found +// whatever the case, or form parsing refuses a body it should accept. +func TestExchangeRequestContentTypeFolded(t *testing.T) { + for _, name := range []string{"Content-Type", "content-type", "CONTENT-TYPE", "cOnTeNt-TyPe"} { + t.Run(name, func(t *testing.T) { + const formType = "application/x-www-form-urlencoded" + var gotType string + var gotErr error + var form httpraw.Form + var sm MuxSlice + sm.Reset(1) + sm.Handle("/f", func(exch *Exchange) { + gotType = string(exch.RequestContentType()) + gotErr = exch.RequestParseForm(&form, make([]byte, 64)) + }) + serve(t, "POST /f HTTP/1.1\r\nHost: h\r\n"+name+": "+formType+"\r\nContent-Length: 3\r\n\r\na=1", &sm) + + if gotType != formType { + t.Fatalf("want Content-Type %q, got %q", formType, gotType) + } + if gotErr != nil { + t.Fatalf("want form parsed, got %v", gotErr) + } + if key, value := form.Pair(0); b2s(key) != "a" || b2s(value) != "1" { + t.Errorf("want a=1, got %q=%q", key, value) + } + }) + } +} + +// The Transfer-Encoding guard stops chunk framing being read as form data. A +// lowercase field name must not slip past it: with a Content-Length alongside, +// the chunk sizes and their CRLFs land inside the parsed pairs. +func TestExchangeRequestParseFormFoldedTransferEncoding(t *testing.T) { + for _, name := range []string{"Transfer-Encoding", "transfer-encoding", "TRANSFER-ENCODING"} { + t.Run(name, func(t *testing.T) { + const body = "3\r\na=1\r\n0\r\n\r\n" + var gotErr error + var form httpraw.Form + var sm MuxSlice + sm.Reset(1) + sm.Handle("/f", func(exch *Exchange) { + gotErr = exch.RequestParseForm(&form, make([]byte, 64)) + }) + serve(t, "POST /f HTTP/1.1\r\nHost: h\r\nContent-Type: application/x-www-form-urlencoded\r\n"+ + name+": chunked\r\nContent-Length: "+strconv.Itoa(len(body))+"\r\n\r\n"+body, &sm) + + if gotErr != errUnsupportedTransferCoding { + t.Fatalf("want %v, got %v with %d pairs %q", errUnsupportedTransferCoding, gotErr, form.Len(), formString(&form)) + } + }) + } +} diff --git a/http/httphi/httphi_fuzz_test.go b/http/httphi/httphi_fuzz_test.go new file mode 100644 index 0000000..209f452 --- /dev/null +++ b/http/httphi/httphi_fuzz_test.go @@ -0,0 +1,419 @@ +package httphi + +import ( + "bytes" + "io" + "strings" + "testing" + + "github.com/soypat/lneto/http/httpraw" +) + +// Fuzz targets in this file are written so that a stored corpus keeps its +// meaning as the tests grow. Go's corpus files are positional and typed, so an +// input is only reproducible while the code that decodes it stays fixed. Five +// rules keep that true, and edits to this file must obey them: +// +// 1. A target's signature is frozen: func(t *testing.T, ctrl uint64, data []byte). +// Adding, removing or reordering a parameter invalidates every stored entry. +// 2. Control decisions come from ctrl only, wire bytes from data only. Never +// branch on data, never put payload in ctrl: the two axes mutate apart. +// 3. ctrl is a bit field read through absolute shifts and masks named below. +// New knobs claim unused high bits and are only ever appended, never +// renumbered, and the zero value of a knob must decode to the behaviour that +// existed before it was added, so old entries keep replaying as they did. +// 4. No PRNG, no cursor. No rand, no internal.Prand64, and no helper that +// "reads the next N bits" while advancing a position: inserting one draw +// ahead of another shifts every later decision, which is the same corpus +// invalidation a PRNG causes. +// 5. Nothing ambient: no wall clock, no goroutines, no map iteration order. +// Targets drive [Handle] on the calling goroutine; [Router] is not fuzzed +// here precisely because it serves on goroutines of its own. +const ( + // Bits 0..3 index segSizes: how the request is split across reads. + ctlSegShift, ctlSegMask = 0, 0xf + // Bits 4..7 and 8..11 size the request and response halves of the buffer. + ctlReqBufShift, ctlRespBufShift, ctlBufMask = 4, 8, 0xf + // Bits 12..13 size the request header field table. + ctlKVCapShift, ctlKVCapMask = 12, 0x3 + // Bit 14 normalizes outgoing header keys. + ctlNormalizeShift, ctlNormalizeMask = 14, 0x1 + // Bits 15..16 make that many leading writes to the connection fail. + ctlFailWriteShift, ctlFailWriteMask = 15, 0x3 + // Bits 17..20 select what the handler does, see the op constants. + ctlHandlerOpsShift, ctlHandlerOpsMask = 17, 0xf + // Bits 21..23 count the header fields the handler stages. + ctlStageCountShift, ctlStageCountMask = 21, 0x7 + // Bit 24 asks AppendQuery for decoded values. + ctlDecodeShift, ctlDecodeMask = 24, 0x1 + // Bits 25..28 size the scratch buffer handed to form and multipart parsing. + ctlScratchShift, ctlScratchMask = 25, 0xf + // Bit 29 makes the multipart sink discard part content. + ctlDiscardShift, ctlDiscardMask = 29, 0x1 + // Next knob starts at bit 30. +) + +// Handler operations, selected by the ctlHandlerOps field. Values are frozen: +// a new operation takes the next free bit within the field. +const ( + opStageHeaders = 1 << iota + opReadBody + opWriteBody + opHijack +) + +// segSizes are the chunk sizes a request may be delivered in, index 0 meaning +// "all at once". Splitting a request mid-CRLF or before a colon is what drives +// [httpraw.Header.TryParse]'s resumption path. Entries may be appended, never +// changed: an existing index must keep splitting exactly as it does today. +var segSizes = [16]int{0, 1, 2, 3, 5, 7, 11, 16, 23, 37, 64, 101, 173, 256, 509, 1024} + +// ctlField reads a knob out of ctrl. Absolute shift, no cursor, so knobs are +// independent of one another and of the order they are read in. +func ctlField(ctrl uint64, shift, mask uint64) uint64 { + return (ctrl >> shift) & mask +} + +// maxFuzzInput bounds the wire data a target accepts. The filter depends only +// on the input, so it classifies an entry the same way on every run. +const maxFuzzInput = 16 << 10 + +// fuzzExchange returns an exchange acquired on a connection preloaded with data, +// both sized and segmented by ctrl. +func fuzzExchange(t *testing.T, ctrl uint64, data []byte) (*Exchange, *rwconn) { + t.Helper() + reqBuf := minRequestHeaderBuffer + int(ctlField(ctrl, ctlReqBufShift, ctlBufMask))*8 + respBuf := minResponseHeaderBuffer + int(ctlField(ctrl, ctlRespBufShift, ctlBufMask))*8 + kvCap := 1 + int(ctlField(ctrl, ctlKVCapShift, ctlKVCapMask))*8 + + conn := newConn("") + seg := segSizes[ctlField(ctrl, ctlSegShift, ctlSegMask)] + if seg <= 0 || seg >= len(data) { + conn.AddReadable(data) + } else { + for off := 0; off < len(data); off += seg { + conn.AddSegment(string(data[off:min(off+seg, len(data))])) + } + } + // Always hang up: a drained connection that never reports EOF reads (0,nil) + // forever and [Handle] would back off in an unbounded loop. + conn.Hangup() + if n := ctlField(ctrl, ctlFailWriteShift, ctlFailWriteMask); n > 0 { + conn.FailWrites(int(n)) + } + exch := newExchange(t, conn, ExchangeConfig{ + RawBuf: make([]byte, reqBuf+respBuf), + RequestBufferLim: reqBuf, + NumHeaderKVCap: kvCap, + NormalizeOutgoingKeys: ctlField(ctrl, ctlNormalizeShift, ctlNormalizeMask) != 0, + NoRequestBufferGrowth: true, + }) + return exch, conn +} + +// checkRequestView asserts the request-target views agree with one another. +func checkRequestView(t *testing.T, exch *Exchange) { + t.Helper() + target, path, query := exch.RequestTarget(), exch.RequestPath(), exch.RequestQuery() + if !bytes.HasPrefix(target, path) { + t.Fatalf("path %q is not a prefix of target %q", path, target) + } + if len(query) > 0 && !bytes.HasSuffix(target, query) { + t.Fatalf("query %q is not a suffix of target %q", query, target) + } + if len(path)+len(query) > len(target) { + t.Fatalf("path %q and query %q exceed target %q", path, query, target) + } +} + +// checkResponse asserts that whatever reached the wire is a response a peer +// could parse: a status line this package produced, followed by a header block +// that terminates, and that [httpraw] reads back what it wrote. +func checkResponse(t *testing.T, written string) { + t.Helper() + if written == "" { + return // Hijacked, or a request refused before anything was staged. + } + const proto = "HTTP/1.1 " + if !strings.HasPrefix(written, proto) { + t.Fatalf("response does not open with a status line: %q", written) + } + if len(written) < len(proto)+4 { + t.Fatalf("status line truncated: %q", written) + } + for _, c := range []byte(written[len(proto) : len(proto)+3]) { + if c < '0' || c > '9' { + t.Fatalf("status code is not three digits: %q", written) + } + } + if written[len(proto)+3] != ' ' { + t.Fatalf("status code not followed by a space: %q", written) + } + if !strings.Contains(written, "\r\n\r\n") { + t.Fatalf("header block never terminated: %q", written) + } + var resp httpraw.Header + const asResponse = true + if err := resp.ParseBytes(asResponse, []byte(written)); err != nil { + t.Fatalf("response does not parse back: %s in %q", err, written) + } +} + +// handler2Path is a second registration so lookup does not always match on the +// first entry of the mux. +const handler2Path = "/fuzz" + +// fuzzMux returns a mux serving handler on the paths the seed corpus requests. +func fuzzMux(handler HandlerFunc) *MuxSlice { + var mux MuxSlice + mux.Handle("/", handler) + mux.Handle(handler2Path, handler) + return &mux +} + +// FuzzHandleRequest drives a whole exchange: a request off the wire through +// [Handle], a handler staging and writing a response, and back out to the peer. +func FuzzHandleRequest(f *testing.F) { + addSeeds(f) + f.Fuzz(func(t *testing.T, ctrl uint64, data []byte) { + if len(data) > maxFuzzInput { + return + } + exch, conn := fuzzExchange(t, ctrl, data) + ops := ctlField(ctrl, ctlHandlerOpsShift, ctlHandlerOpsMask) + stage := ctlField(ctrl, ctlStageCountShift, ctlStageCountMask) + + Handle(exch, fuzzMux(func(exch *Exchange) { + checkRequestView(t, exch) + if ops&opStageHeaders != 0 { + // Literal fields: staging request bytes would test the caller's + // escaping, not this package's framing. + for i := range stage { + exch.StageHeader("X-Fuzz", "value") + exch.StageHeaderInt("X-Fuzz-Int", int64(i), 10) + } + } + if ops&opReadBody != 0 { + var body [64]byte + for range 64 { + n, err := exch.ReadBody(body[:]) + if n == 0 && err != nil { + break + } + } + } + if ops&opWriteBody != 0 { + exch.WriteBody([]byte("fuzz body")) + } + if ops&opHijack != 0 { + exch.HijackRaw(nil) + } + }), nopBackoff) + + if ctlField(ctrl, ctlFailWriteShift, ctlFailWriteMask) == 0 { + // A refused write leaves a partial response on purpose, so the + // wire is only well formed when every write got through. + checkResponse(t, conn.ViewWritten()) + } + }) +} + +// FuzzQueryAndForm drives the request-target query and the form-encoded body, +// both of which decode percent escapes in place over caller memory. +func FuzzQueryAndForm(f *testing.F) { + addSeeds(f) + f.Fuzz(func(t *testing.T, ctrl uint64, data []byte) { + if len(data) > maxFuzzInput { + return + } + exch, _ := fuzzExchange(t, ctrl, data) + decoded := ctlField(ctrl, ctlDecodeShift, ctlDecodeMask) != 0 + scratchLen := 8 + int(ctlField(ctrl, ctlScratchShift, ctlScratchMask))*16 + + Handle(exch, fuzzMux(func(exch *Exchange) { + // Every pair the iterator yields must be reachable by name, or the + // two views of the query string disagree. + const maxPairs = 64 + key, _, rest := httpraw.NextQueryPair(exch.RequestQuery()) + for pairs := 0; key != nil && pairs < maxPairs; pairs++ { + dec := make([]byte, len(key)) + n, err := httpraw.CopyDecodedPercentURL(dec, key, true) + if err == nil { + if n > len(key) { + t.Fatalf("decoding key %q grew it to %d bytes", key, n) + } + if _, present := exch.RequestQueryAppend(nil, string(dec[:n]), decoded); !present { + t.Fatalf("query pair %q absent from AppendQuery", key) + } + } + key, _, rest = httpraw.NextQueryPair(rest) + } + + var form httpraw.Form + buf := make([]byte, scratchLen) + if err := exch.RequestParseForm(&form, buf); err != nil { + return + } + total := 0 + lens := make([]int, form.Len()) + for i := range form.Len() { + k, v := form.Pair(i) + lens[i] = len(k) + len(v) + total += lens[i] + } + if total > len(buf) { + t.Fatalf("form pairs span %d bytes of a %d byte buffer", total, len(buf)) + } + if err := form.Decode(); err != nil { + return + } + // Decoding replaces escapes in place, so no pair may grow. + for i := range form.Len() { + k, v := form.Pair(i) + if len(k)+len(v) > lens[i] { + t.Fatalf("pair %d grew from %d to %d bytes on decode", i, lens[i], len(k)+len(v)) + } + } + }), nopBackoff) + }) +} + +// countSink counts what a multipart part streamed into it and whether the part +// was closed off. +type countSink struct { + written int + closed bool +} + +func (c *countSink) Write(b []byte) (int, error) { + c.written += len(b) + return len(b), nil +} + +func (c *countSink) Close() error { + c.closed = true + return nil +} + +// FuzzMultipart drives [Exchange.ReadMultiparts], which streams a body of +// unknown length through a buffer the caller sized. +func FuzzMultipart(f *testing.F) { + addSeeds(f) + f.Fuzz(func(t *testing.T, ctrl uint64, data []byte) { + if len(data) > maxFuzzInput { + return + } + exch, _ := fuzzExchange(t, ctrl, data) + discard := ctlField(ctrl, ctlDiscardShift, ctlDiscardMask) != 0 + bufLen := 16 + int(ctlField(ctrl, ctlScratchShift, ctlScratchMask))*16 + + Handle(exch, fuzzMux(func(exch *Exchange) { + var sinks []*countSink + _, err := exch.ReadMultiparts(nil, make([]byte, bufLen), func(hdr *httpraw.MultipartHeader) io.WriteCloser { + if discard { + return nil + } + sink := new(countSink) + sinks = append(sinks, sink) + return sink + }) + if err != nil { + return // Sinks are left for the caller to deal with on error. + } + total := 0 + for _, sink := range sinks { + if !sink.closed { + t.Fatal("ReadMultiparts returned with a part left open") + } + total += sink.written + } + if total > len(data) { + t.Fatalf("parts streamed %d bytes out of a %d byte request", total, len(data)) + } + }), nopBackoff) + }) +} + +// addSeeds adds the shared seed corpus. Every seed pins ctrl to zero so its +// meaning never moves: knobs added later decode their zero value to the +// behaviour the seed was recorded under. +func addSeeds(f *testing.F) { + f.Helper() + const ( + formType = "Content-Type: application/x-www-form-urlencoded\r\n" + mpType = "Content-Type: multipart/form-data; boundary=b0undary\r\n" + ) + seeds := []string{ + // Well formed traffic, so the fuzzer has somewhere to mutate from. + "GET / HTTP/1.1\r\nHost: h\r\n\r\n", + "GET /fuzz?q=go&n=1 HTTP/1.1\r\nHost: h\r\n\r\n", + "POST /fuzz HTTP/1.1\r\nHost: h\r\n" + formType + "Content-Length: 11\r\n\r\na=1&b=2&c=3", + "POST /fuzz HTTP/1.1\r\nHost: h\r\n" + mpType + "Content-Length: 76\r\n\r\n--b0undary\r\nContent-Disposition: form-data; name=\"f\"\r\n\r\nbody\r\n--b0undary--\r\n", + + // Framing the RFC leaves room to disagree over, which is where request + // smuggling lives: two lengths, a length plus a coding, and a coding + // this package does not decode. + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 3\r\nContent-Length: 4\r\n\r\nabcd", + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 3\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n0\r\n\r\n", + "POST / HTTP/1.1\r\nHost: h\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n", + + // Field names are case insensitive, RFC 9110 5.1, so a lookup that + // misses one of these reads the request differently than the peer wrote it. + "POST / HTTP/1.1\r\nHost: h\r\ncontent-length: 4\r\n\r\nbody", + "POST / HTTP/1.1\r\nHost: h\r\nCONTENT-LENGTH: 4\r\n\r\nbody", + "POST /fuzz HTTP/1.1\r\nHost: h\r\ncontent-type: application/x-www-form-urlencoded\r\ncontent-length: 3\r\n\r\na=1", + + // Content-Length values that are not a bare digit string, RFC 9112 6.2. + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: +5\r\n\r\nbody!", + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: -1\r\n\r\nbody", + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 1 2\r\n\r\nbody", + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length:\r\n\r\nbody", + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 9223372036854775808\r\n\r\nbody", + + // Line ending and field syntax edges. + "GET / HTTP/1.1\nHost: h\n\n", + "GET / HTTP/1.1\r\nHost: h\rX: y\r\n\r\n", + "GET / HTTP/1.1\r\nHost: h\r\n Continued: fold\r\n\r\n", + "GET / HTTP/1.1\r\nHost: h\r\nX: va\x00lue\r\n\r\n", + "GET / HTTP/1.1\r\nHost: h\r\nNoColon\r\n\r\n", + "GET / HTTP/1.1\r\nHost: h\r\n: novalue\r\n\r\n", + "GET / HTTP/1.1\r\nHost: h\r\nX:\r\n\r\n", + "GET / HTTP/1.1\r\nHost: h\r\n\r\n\r\n", + + // Request lines this package tolerates or must refuse. + "GET http://h/abs HTTP/1.1\r\nHost: h\r\n\r\n", + "GET /\r\n\r\n", // HTTP/0.9 simple request, no version. + " GET / HTTP/1.1\r\nHost: h\r\n\r\n", + "GET / HTTP/1.1\r\nHost: h\r\n\r\n", + "/ HTTP/1.1\r\nHost: h\r\n\r\n", + "GET / HTTP/9.9\r\nHost: h\r\n\r\n", + + // Percent escapes, including the truncated and the malformed. + "GET /a%20b?a%20b=c%20d HTTP/1.1\r\nHost: h\r\n\r\n", + "GET /?q=%2 HTTP/1.1\r\nHost: h\r\n\r\n", + "GET /?q=%zz HTTP/1.1\r\nHost: h\r\n\r\n", + "GET /%00?a=%00 HTTP/1.1\r\nHost: h\r\n\r\n", + "GET /?a=b=c&&=v&debug HTTP/1.1\r\nHost: h\r\n\r\n", + "GET /?a%3db=1&a+b=2 HTTP/1.1\r\nHost: h\r\n\r\n", + + // Multipart bodies that end early or never open a part. + "POST /fuzz HTTP/1.1\r\nHost: h\r\n" + mpType + "Content-Length: 12\r\n\r\n--b0undary\r\n", + "POST /fuzz HTTP/1.1\r\nHost: h\r\n" + mpType + "Content-Length: 14\r\n\r\n--b0undary--\r\n", + "POST /fuzz HTTP/1.1\r\nHost: h\r\nContent-Type: multipart/form-data\r\nContent-Length: 4\r\n\r\nbody", + + // Sizes that crowd the buffers: a long target, many fields, and a body + // arriving in the same read as the header it follows. + "GET /" + strings.Repeat("a", 512) + " HTTP/1.1\r\nHost: h\r\n\r\n", + "GET / HTTP/1.1\r\n" + strings.Repeat("X: y\r\n", 200) + "\r\n", + "GET / HTTP/1.1\r\n" + strings.Repeat("k", 512) + ": v\r\n\r\n", + "POST / HTTP/1.1\r\nHost: h\r\nContent-Length: 4\r\n\r\nbodyTRAILING", + + // Nothing, and nothing that resembles a request at all. + "", + "\r\n\r\n", + "\x00\x00\x00\x00", + } + for _, seed := range seeds { + f.Add(uint64(0), []byte(seed)) + } +} diff --git a/http/httphi/mux.go b/http/httphi/mux.go new file mode 100644 index 0000000..7ca7e6f --- /dev/null +++ b/http/httphi/mux.go @@ -0,0 +1,313 @@ +package httphi + +import ( + "bytes" + "io" + "strings" + "unsafe" + + "github.com/soypat/lneto" + "github.com/soypat/lneto/http/httpraw" + "github.com/soypat/lneto/internal" +) + +// Handle is a extremely low-level HTTP handling method used internally in [Router]. +// Requires exchange to be acquired and configured. Will panic if any argument is nil. +// Handle does not close the connection on any outcome: the caller owns it. +// backoff can be set for dealing with non-blocking connections. If backoff set to nil +// then a zero-length-read will result in Handle returning [io.ErrNoProgress]. +func Handle(exch *Exchange, mux Mux, backoff lneto.BackoffStrategy) error { + if !exch.acquired.Load() { + return lneto.ErrBadState + } + reqhdr := &exch.reqHdr + reqhdr.Reset(nil, 0) // Assume exchange has been configured and reuse memory. + var consecutiveBackoffs uint + for { + n, err := reqhdr.ReadFromLimited(exch.rw, reqhdr.BufferFree()) + if err != nil { + exch.readErr = err + exch.handleError(err) + return err + } else if n == 0 { + if backoff == nil { + return io.ErrNoProgress + } + backoff.Do(consecutiveBackoffs) + consecutiveBackoffs++ + continue + } + consecutiveBackoffs = 0 + const asRequest = false + needMore, err := reqhdr.TryParse(asRequest) + if needMore { + continue // Request header split across reads, accumulate the rest. + } else if err != nil { + exch.handleError(err) + return err + } + break // Done! + } + // Setup Exchange fields necessary for correct functioning. + parsed := reqhdr.BufferParsed() + exch.respRemains = reqhdr.BufferReceived() - parsed + exch.respHeaderOff = uint16(parsed) + exch.respHeaderLen = 0 + proto := b2s(reqhdr.Protocol()) + if len(proto) == 0 { + // HTTP/0.9 not tolerated RFC 9112 3. + exch.WriteHeader(int(StatusBadRequest)) + return errNoRequestProto + } else if proto != "HTTP/1.1" && proto != "HTTP/1.0" { + // RFC 9112 2.6. + exch.WriteHeader(int(StatusHTTPVersionNotSupported)) + return errBadRequestProto + } + // Mux on the request path: the query string is the handler's business. + path := reqhdr.RequestPath() + meth := reqhdr.Method() + matchedPattern, handler := mux.LookupHandler(MethodFromBytes(meth), path, exch.pathValues) + if handler != nil { + exch.matchedPattern = matchedPattern + handler(exch) + if !exch.hijacked { + exch.FlushHeader() + } + } else { + exch.WriteHeader(404) + } + return nil +} + +func (exch *Exchange) handleError(err error) { + if err == httpraw.ErrHeaderTooMany || err == httpraw.ErrBufferExhausted || exch.reqHdr.BufferFree() == 0 { + // The peer is owed an answer: no larger buffer is coming, so + // say so instead of dropping the connection, RFC 6585 5. + exch.StageHeader("Content-Length", "0") + exch.WriteHeader(int(StatusRequestHeaderFieldsTooLarge)) + } +} + +// HandlerFunc serves a single request, playing the part of http.Handler. +// The exchange is only valid for the duration of the call: it is released to +// the router's pool on return, so a handler must not retain it nor any slice it +// handed out. +type HandlerFunc func(ex *Exchange) + +// Mux resolves a request to the handler that serves it. [Handle] calls +// LookupHandler with the request-target's path, not the whole target, and +// replies 404 when it returns nil. +type Mux interface { + // LookupHandler matches the requestPath and method to a handler and returns it and the + // pattern it matched. dstPathVals are set to non-zero values by Mux and can later be accessed by [Exchange.PathValue] + // requestPath is a buffer owned by the [Exchange] usually and should not be held after LookupHandler returns. + LookupHandler(get Method, requestPath []byte, dstPathVals []pathValue) (matchedPattern string, handler HandlerFunc) +} + +// MuxSlice is a [Mux] backed by a slice of registered endpoints, matched by +// exact path. Lookup is linear in the number of registrations. +type MuxSlice struct { + // TODO: binary search worth it? + _handlers []struct { + method Method + path string + handler HandlerFunc + setPathVal bool + } +} + +type pathValue struct { + Key string // owned by mux. + Value []byte // points to raw exchange buffer. +} + +// pathSeparator is shared so [SetPathValues] never converts a literal per call. +var pathSeparator = []byte{'/'} + +// SetPathValues matches requestPath against pattern and binds its wildcards +// into dstPathVals, read back with [Exchange.PathValue]. Wildcards are whole +// segments as per http.ServeMux: "{name}" takes one non-empty segment, +// "{name...}" the rest including slashes, "{$}" only the path's end, and a +// trailing slash is an anonymous "{...}". i.e: "/b/{bucket}/o/{obj...}". +// +// Unlike ServeMux, segments are compared and bound raw, so "/users/{id}" binds +// "x%2Fy" and not "x/y". Which paths match is unaffected. Bound values alias +// requestPath rather than copy it. +func SetPathValues(dstPathVals []pathValue, pattern string, requestPath []byte) (matched, pathValSliceTooShort bool) { + if len(pattern) == 0 || pattern[0] != '/' || len(requestPath) == 0 || requestPath[0] != '/' { + return false, false + } + pattern, requestPath = pattern[1:], requestPath[1:] + n := 0 + for { + if len(pattern) == 0 { + // Nothing left after a slash: an anonymous "..." taking the rest, + // which is why "/files/" matches "/files/a/b" and "/" matches all. + return true, false + } + patSeg, patRest, patMore := strings.Cut(pattern, "/") + reqSeg, reqRest, reqMore := bytes.Cut(requestPath, pathSeparator) + name, isMulti, isWildcard := pathWildcard(patSeg) + switch { + case isWildcard && name == "$": + // Matches the end of the path and nothing else, so it must be the + // last segment of the pattern and leave no path behind. + return !patMore && len(requestPath) == 0, false + + case isWildcard && isMulti: + // Takes the remainder including slashes, possibly empty. + if name != "" { + if n >= len(dstPathVals) { + return false, true + } + dstPathVals[n] = pathValue{Key: name, Value: requestPath} + n++ + } + return true, false + + case isWildcard: + if len(reqSeg) == 0 { + return false, false // One segment means a non-empty one. + } + if n >= len(dstPathVals) { + return false, true + } + dstPathVals[n] = pathValue{Key: name, Value: reqSeg} + n++ + + default: + if b2s(reqSeg) != patSeg { + return false, false + } + } + if patMore != reqMore { + // One side has a further segment and the other does not, so + // "/health" misses "/health/" and "/files/" misses "/files". + return false, false + } else if !patMore { + return true, false // Both spent on the same segment. + } + pattern, requestPath = patRest, reqRest + } +} + +// pathWildcard picks apart a "{name}" or "{name...}" pattern segment. It +// reports ok false for a literal segment, so "/b_{bucket}" is literal text and +// not a wildcard, matching ServeMux's rule that wildcards be whole segments. +func pathWildcard(segment string) (name string, isMulti, ok bool) { + if len(segment) < 2 || segment[0] != '{' || segment[len(segment)-1] != '}' { + return "", false, false + } + name = segment[1 : len(segment)-1] + if rest, found := strings.CutSuffix(name, "..."); found { + return rest, true, true + } + return name, false, true +} + +// Reset discards all registered handlers, reusing the backing array and growing +// it to fit capacity registrations. +func (sm *MuxSlice) Reset(capacity int) { + internal.SliceReuse(&sm._handlers, capacity) +} + +// LookupHandler returns the handler registered for request path, or nil if none matches. +// The first registration matching both method and uri wins. +func (sm *MuxSlice) LookupHandler(method Method, path []byte, dstPathVals []pathValue) (matched string, _ HandlerFunc) { + for _, endpoint := range sm._handlers { + if endpoint.method != MethUndefined && endpoint.method != method { + continue + } + // Method matches. + if endpoint.setPathVal { + if ok, _ := SetPathValues(dstPathVals, endpoint.path, path); ok { + return endpoint.path, endpoint.handler + } + } else if b2s(path) == endpoint.path { + return endpoint.path, endpoint.handler + } + } + return "", nil +} + +// Handle registers handler for reg, either a bare path matching any method or a +// method and path separated by a space, i.e: "/health" or "GET /health". +// Handle does not check for duplicate registrations: the first one added wins. +func (sm *MuxSlice) Handle(optMethodAndPath string, handler HandlerFunc) { + v := internal.SliceReclaim(&sm._handlers) + method := MethUndefined + methodOrURL, url, methodFound := strings.Cut(optMethodAndPath, " ") + if methodFound { + method = MethodFrom(methodOrURL) + } else { + url = methodOrURL + } + v.method = method + v.path = url + v.handler = handler +} + +// Method is a HTTP request method, parsed by [MethodFrom]. +type Method uint8 + +const ( + MethUndefined Method = iota // undefined + MethGet // GET + // lol. + MethHead // HEAD + MethPost // POST + MethPut // PUT + // RFC 5789 + MethPatch // PATCH + MethDelete // DELETE + MethConnect // CONNECT + MethOptions // OPTIONS + MethTrace // TRACE + MethUnknown // unknown +) + +// MethodFrom returns the [Method] matching meth, [MethUndefined] if meth is +// empty and [MethUnknown] if it names a method this package does not know. +// Comparison is case sensitive: methods are uppercase, RFC 9110 9.1. +func MethodFrom(meth string) (res Method) { + if len(meth) == 0 { + return MethUndefined + } + switch meth { + case "GET": + res = MethGet + case "HEAD": + res = MethHead + case "POST": + res = MethPost + case "PUT": + res = MethPut + case "PATCH": + res = MethPatch + case "DELETE": + res = MethDelete + case "CONNECT": + res = MethConnect + case "OPTIONS": + res = MethOptions + case "TRACE": + res = MethTrace + default: + res = MethUnknown + } + return res +} + +// MethodFromBytes is a [MethodFrom] wrapper with bytes argument instead of string. +func MethodFromBytes(meth []byte) (res Method) { + if len(meth) == 0 { + return MethUndefined + } + return MethodFrom(b2s(meth)) +} + +// b2s converts byte slice to a string without memory allocation. +// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . +func b2s(b []byte) string { + return unsafe.String(unsafe.SliceData(b), len(b)) +} diff --git a/http/httphi/mux_test.go b/http/httphi/mux_test.go new file mode 100644 index 0000000..23d5135 --- /dev/null +++ b/http/httphi/mux_test.go @@ -0,0 +1,222 @@ +package httphi + +import ( + "strings" + "testing" +) + +// SetPathValues must agree with net/http.ServeMux on which patterns match which +// paths and what each wildcard binds to. Every case below was taken from a run +// against a real ServeMux, so this table is an oracle, not a guess. +// +// The one documented deviation is percent-decoding: ServeMux unescapes segments +// before matching and binding, this does not. See TestSetPathValuesEscaping. +func TestSetPathValues(t *testing.T) { + for _, test := range []struct { + pattern string + path string + match bool + want string // "name=value" pairs joined by "|", in bind order. + }{ + // Single segment wildcard binds exactly one non-empty segment. + {pattern: "/users/{id}", path: "/users/42", match: true, want: "id=42"}, + {pattern: "/users/{id}", path: "/users/42/x", match: false}, + {pattern: "/users/{id}", path: "/users/", match: false}, + {pattern: "/users/{id}", path: "/users", match: false}, + {pattern: "/users/{id}/edit", path: "/users/42/edit", match: true, want: "id=42"}, + {pattern: "/{a}/{b}", path: "/x/y", match: true, want: "a=x|b=y"}, + + // "..." swallows the remainder, slashes included, and may bind empty. + {pattern: "/b/{bucket}/o/{obj...}", path: "/b/bk/o/a/b/c", match: true, want: "bucket=bk|obj=a/b/c"}, + {pattern: "/b/{bucket}/o/{obj...}", path: "/b/bk/o/", match: true, want: "bucket=bk|obj="}, + {pattern: "/b/{bucket}/o/{obj...}", path: "/b/bk/o", match: false}, + {pattern: "/files/{p...}", path: "/files/", match: true, want: "p="}, + {pattern: "/files/{p...}", path: "/files", match: false}, + + // {$} matches only the end of the path. + {pattern: "/{$}", path: "/", match: true}, + {pattern: "/{$}", path: "/x", match: false}, + {pattern: "/a/{$}", path: "/a/", match: true}, + {pattern: "/a/{$}", path: "/a", match: false}, + {pattern: "/a/{$}", path: "/a/b", match: false}, + + // A trailing slash is an anonymous "..." wildcard, binding nothing. + {pattern: "/files/", path: "/files/a/b", match: true}, + {pattern: "/files/", path: "/files/", match: true}, + {pattern: "/files/", path: "/files", match: false}, + {pattern: "/", path: "/anything/at/all", match: true}, + + // Literal patterns match exactly, trailing slash included. + {pattern: "/health", path: "/health", match: true}, + {pattern: "/health", path: "/health/", match: false}, + + // An empty segment never satisfies a single wildcard. + {pattern: "/a/{x}/b", path: "/a//b", match: false}, + } { + t.Run(test.pattern+"__"+test.path, func(t *testing.T) { + vals := make([]pathValue, 8) + match, tooShort := SetPathValues(vals, test.pattern, []byte(test.path)) + if tooShort { + t.Fatal("8 slots must be enough for these patterns") + } + if match != test.match { + t.Fatalf("want match=%v, got %v", test.match, match) + } + if !match { + return + } + if got := renderPathValues(vals); got != test.want { + t.Errorf("want %q, got %q", test.want, got) + } + }) + } +} + +// Bound values must alias the request path buffer rather than copy it: the +// exchange owns that memory and a copy would allocate per request. +func TestSetPathValuesAliasesRequestBuffer(t *testing.T) { + path := []byte("/users/42/edit") + vals := make([]pathValue, 4) + match, _ := SetPathValues(vals, "/users/{id}/edit", path) + if !match { + t.Fatal("want match") + } + if string(vals[0].Value) != "42" { + t.Fatalf("want id=42, got %q", vals[0].Value) + } + // Mutating the request buffer must show through the bound value. + path[7] = '9' + if string(vals[0].Value) != "92" { + t.Errorf("value must alias the request buffer, got %q", vals[0].Value) + } +} + +// A destination too small to hold every wildcard must say so rather than bind a +// partial set or write out of range. +func TestSetPathValuesSliceTooShort(t *testing.T) { + match, tooShort := SetPathValues(make([]pathValue, 1), "/{a}/{b}", []byte("/x/y")) + if !tooShort { + t.Error("want pathValSliceTooShort for 2 wildcards in 1 slot") + } + if match { + t.Error("want match=false when values could not be bound") + } + // A pattern that binds nothing needs no slots at all. + match, tooShort = SetPathValues(nil, "/health", []byte("/health")) + if !match || tooShort { + t.Errorf("want match with no slots needed, got match=%v tooShort=%v", match, tooShort) + } +} + +// Percent escapes are compared and bound raw. ServeMux unescapes segment by +// segment, so "/users/x%2Fy" binds id="x/y" there and id="x%2Fy" here. Matching +// agrees either way; only the bound bytes differ. +func TestSetPathValuesEscaping(t *testing.T) { + vals := make([]pathValue, 4) + if match, _ := SetPathValues(vals, "/a%2Fb/{x}", []byte("/a%2Fb/v")); !match { + t.Error("want literal escape in pattern to match the same bytes in path") + } + vals = make([]pathValue, 4) + match, _ := SetPathValues(vals, "/users/{id}", []byte("/users/x%2Fy")) + if !match { + t.Fatal("want match") + } + if got := string(vals[0].Value); got != "x%2Fy" { + t.Errorf("want raw %q, got %q", "x%2Fy", got) + } +} + +// renderPathValues joins the bound pairs for comparison, stopping at the first +// unused slot. +func renderPathValues(vals []pathValue) string { + var sb strings.Builder + for _, v := range vals { + if v.Key == "" { + break + } + if sb.Len() > 0 { + sb.WriteByte('|') + } + sb.WriteString(v.Key) + sb.WriteByte('=') + sb.Write(v.Value) + } + return sb.String() +} + +// Matching a request must not allocate: keys alias the mux's pattern and values +// alias the request buffer, so nothing is copied per request. +func TestSetPathValuesNoAlloc(t *testing.T) { + vals := make([]pathValue, 8) + path := []byte("/b/bk/o/a/b/c") + allocs := testing.AllocsPerRun(100, func() { + SetPathValues(vals, "/b/{bucket}/o/{obj...}", path) + }) + if allocs != 0 { + t.Fatalf("SetPathValues allocated %v times, want 0", allocs) + } +} + +// pathValueMux binds one wildcard pattern, standing in for a [Mux] that +// supports them until MuxSlice sets setPathVal. +type pathValueMux struct { + pattern string + handler HandlerFunc +} + +func (m *pathValueMux) LookupHandler(method Method, path []byte, dst []pathValue) (string, HandlerFunc) { + if ok, _ := SetPathValues(dst, m.pattern, path); ok { + return m.pattern, m.handler + } + return "", nil +} + +// A wildcard bound by one request must not be readable by the next request the +// same pooled exchange serves. A literal pattern binds nothing, so it never +// overwrites the previous request's slots, and the values it would leak alias a +// buffer the new request has already overwritten. +func TestExchangePathValueClearedBetweenRequests(t *testing.T) { + exch := new(Exchange) + exch.Configure(ExchangeConfig{ + RawBuf: make([]byte, 2048), RequestBufferLim: 1024, + NumHeaderKVCap: defaultNumHeaderKVCap, MaxPathValues: 4, + }) + + // First request binds id=42 off a wildcard pattern. + var gotFirst string + wildcard := &pathValueMux{pattern: "/users/{id}", handler: func(e *Exchange) { + gotFirst = string(e.PathValue("id")) + e.WriteHeader(200) + }} + conn := newConn("GET /users/42 HTTP/1.1\r\nHost: h\r\n\r\n") + conn.Hangup() + if !exch.Acquire(conn) { + t.Fatal("fresh exchange failed to acquire") + } + if err := Handle(exch, wildcard, nopBackoff); err != nil { + t.Fatalf("first request: %s", err) + } + exch.Release() + if gotFirst != "42" { + t.Fatalf("want id=42 bound on the first request, got %q", gotFirst) + } + + // Second request matches a literal pattern, which binds nothing at all. + var leaked []byte + var sm MuxSlice + sm.Handle("/health", func(e *Exchange) { + leaked = e.PathValue("id") + e.WriteHeader(200) + }) + conn2 := newConn("GET /health HTTP/1.1\r\nHost: h\r\n\r\n") + conn2.Hangup() + if !exch.Acquire(conn2) { + t.Fatal("released exchange failed to re-acquire") + } + if err := Handle(exch, &sm, nopBackoff); err != nil { + t.Fatalf("second request: %s", err) + } + if leaked != nil { + t.Errorf("want no path value on a literal route, got id=%q from the previous request", leaked) + } +} diff --git a/http/httphi/router.go b/http/httphi/router.go new file mode 100644 index 0000000..bed1f06 --- /dev/null +++ b/http/httphi/router.go @@ -0,0 +1,403 @@ +package httphi + +import ( + "errors" + "io" + "log/slog" + "math" + "sync" + "sync/atomic" + "time" + + "github.com/soypat/lneto" + "github.com/soypat/lneto/internal" +) + +//go:generate stringer -type Method -linecomment -output stringers.go + +// reconfigureWait bounds how long [Router.Configure] waits for the previous +// generation to stop serving before reusing its exchange buffers. +const reconfigureWait = 10 * time.Millisecond + +var ( + errNoRequestProto = errors.New("httphi: request line with no HTTP version") + errBadRequestProto = errors.New("httphi: unsupported HTTP version in request line") + errBusyExchanges = errors.New("httphi: exchanges still serving, cannot reuse their buffers") + errRouterTornDown = errors.New("httphi: router torn down, configure it before serving") + + errNotFormEncoded = errors.New("httphi: request body is not application/x-www-form-urlencoded") + errNotMultipart = errors.New("httphi: request body is not multipart/form-data") + errUnsupportedTransferCoding = errors.New("httphi: transfer coding not decoded, read the body directly") +) + +type conn = io.ReadWriteCloser + +// Router serves HTTP connections handed to it with [Router.Handle], routing +// each request to a handler found through its [Mux]. It plays the part of +// http.Server minus the listening: accepting connections is the caller's job, +// which is what lets the same router run over a TCP stack, a socket or a test +// pipe. +// +// A Router owns the exchanges and goroutines that serve connections and sizes +// both at [Router.Configure] time, so serving load costs no allocation and +// bounded memory. Connections arriving with nothing left to serve them are +// refused rather than queued, see [Router.Handle]. +// +// Methods are safe for concurrent use. The zero value is not usable: configure +// it first. +type Router struct { + mu sync.Mutex + gen atomic.Uint32 + numGoro int + reqBuf int + respBuf int + reqNumHeaderCap int + maxPathValues int + normalizeKeys bool + pendingConns chan job + mux Mux + + globbuf []byte + exchs []Exchange + freeList *Exchange + + log *slog.Logger +} + +// job is a connection waiting on an exchange for a worker goroutine to serve it. +type job struct { + exch *Exchange +} + +// RouterConfig configures a [Router]. See [Router.Configure]. +type RouterConfig struct { + // FixedNumGoroutines must be set to either -1 (freely allocate new goroutines) or to the number of goroutines + // to spawn on [Router.Configure] being called. + FixedNumGoroutines int + // RequestHeaderBufferSize determines the buffer allocated + // for processing request HTTP headers including request-target (URI), protocol and key/value pairs. + RequestHeaderBufferSize int + // ResponseHeaderMinBufferSize determines buffer allocated for processing response headers. + // Response buffer will reuse unused request memory so this is not a strict limit. + // "HTTP/1.1 200 OK\r\n" does not count towards this memory, only actual Headers key/value pairs use this memory. + // After memory is fully consumed [Exchange.StageHeader] will not append more headers. + ResponseHeaderMinBufferSize int + // Number of request header key/value pairs to parse before failing and returning [StatusRequestHeaderFieldsTooLarge]. + RequestNumHeaderKVCap int + // Sets maximum number of PathValue pairs that can be set on an exchange. Accessed via [Exchange.PathValue]. + MaxPathValues int + + // NormalizeOutgoingKeys normalizes response header field keys as they are + // staged, i.e: "content-type" becomes "Content-Type". + NormalizeOutgoingKeys bool + // MaxAwaitingConns is the depth of the queue connections wait in for a free + // goroutine. [Router.Handle] drops connections once it is full. Required and + // must be non-zero when running a fixed number of goroutines, unused otherwise. + MaxAwaitingConns int + + // Mux resolves each request's method and path to the handler serving it. Required. + Mux Mux + // Logger receives failed exchanges. Optional, nil disables logging. + Logger *slog.Logger +} + +const ( + // minRequestHeaderBuffer is the smallest request buffer [httpraw.Header] + // accepts with buffer growth disabled, which is how exchanges are configured. + minRequestHeaderBuffer = 32 + // minResponseHeaderBuffer is the room [Exchange.FlushHeader] needs for the + // CRLF closing the header block, written even when no field was staged. + minResponseHeaderBuffer = len("\r\n") + // maxExchangeBuffer bounds an exchange's whole buffer: [Exchange] indexes it + // with uint16 offsets, so a larger one would be addressed truncated. + maxExchangeBuffer = math.MaxUint16 +) + +// Validate returns a non-nil error if the configuration cannot be used to +// configure a [Router]. +func (cfg RouterConfig) Validate() error { + workerMode := cfg.workerMode() + switch { + case cfg.Mux == nil, + !workerMode && cfg.FixedNumGoroutines != -1, + workerMode && cfg.MaxAwaitingConns <= 0, + cfg.RequestNumHeaderKVCap <= 0, + cfg.RequestHeaderBufferSize < minRequestHeaderBuffer, + cfg.ResponseHeaderMinBufferSize < minResponseHeaderBuffer, + cfg.ResponseHeaderMinBufferSize > maxExchangeBuffer, + cfg.RequestHeaderBufferSize > maxExchangeBuffer-cfg.ResponseHeaderMinBufferSize: + return lneto.ErrInvalidConfig + } + if workerMode { + // Buffer sizes are bounded by maxExchangeBuffer above so the sum cannot + // overflow; the products below allocate and can. + exchBuf := cfg.RequestHeaderBufferSize + cfg.ResponseHeaderMinBufferSize + if cfg.FixedNumGoroutines > math.MaxInt/exchBuf || + cfg.FixedNumGoroutines > math.MaxInt/cfg.RequestNumHeaderKVCap { + return lneto.ErrInvalidConfig + } + } + return nil +} + +func (cfg RouterConfig) workerMode() bool { + return cfg.FixedNumGoroutines > 0 +} + +// Shutdown stops the router once in-flight exchanges finish; until +// [Router.Configure] is called again, connections are refused with a +// non-nil error. Configure calls it before installing a new generation. +func (r *Router) Shutdown() { + r.mu.Lock() + defer r.mu.Unlock() + r.shutdownLocked() +} + +// shutdownLocked is [Router.Shutdown] but without locking requirement. +func (r *Router) shutdownLocked() { + r.gen.Add(1) + if r.pendingConns != nil { + close(r.pendingConns) + r.pendingConns = nil + } +} + +// Configure prepares the router to serve connections, tearing down the previous +// generation of goroutines and exchanges first. In worker mode it spawns +// [RouterConfig.FixedNumGoroutines] goroutines and allocates their exchange +// buffers up front, so the router's memory use does not grow with load. +// +// Configure may be called on a serving router, but since the exchange buffers +// are reused it waits for connections in flight to finish and fails with a +// non-nil error rather than reconfigure buffers still being served from. +func (r *Router) Configure(cfg RouterConfig) error { + if err := cfg.Validate(); err != nil { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + r.shutdownLocked() + gen := r.gen.Load() + numgoro := cfg.FixedNumGoroutines + workerMode := cfg.workerMode() + + r.reqNumHeaderCap = cfg.RequestNumHeaderKVCap + r.reqBuf = cfg.RequestHeaderBufferSize + r.respBuf = cfg.ResponseHeaderMinBufferSize + r.mux = cfg.Mux + r.log = cfg.Logger + r.maxPathValues = cfg.MaxPathValues + r.normalizeKeys = cfg.NormalizeOutgoingKeys + // Freelist entries were sized by the outgoing configuration: recycling one + // would serve a request with buffer limits cfg never asked for. + r.freeList = nil + if !workerMode { + r.numGoro = 0 + r.pendingConns = nil + return nil + } + if workerMode { + jobqueue := make(chan job, cfg.MaxAwaitingConns) + if gen > 1 { + // Exchange buffers below are reused: the previous generation must be + // done serving before they may be handed to the new one. + err := r.awaitIdleExchangesLocked(reconfigureWait) + if err != nil { + return err + } + } + internal.SliceReuse(&r.exchs, numgoro) + r.exchs = r.exchs[:numgoro] + rawBuflen := cfg.RequestHeaderBufferSize + cfg.ResponseHeaderMinBufferSize + internal.SliceReuse(&r.globbuf, numgoro*rawBuflen) + for i := range numgoro { + // TODO exchange buffer alloc + goff := i * rawBuflen + // r.globbuf[goff:goff+rawBuflen], cfg.RequestHeaderBufferSize, cfg.RequestNumHeaderCap, cfg.NormalizeOutgoingKeys + r.exchs[i].Configure(ExchangeConfig{ + RawBuf: r.globbuf[goff : goff+rawBuflen], + RequestBufferLim: cfg.RequestHeaderBufferSize, + NumHeaderKVCap: cfg.RequestNumHeaderKVCap, + NormalizeOutgoingKeys: cfg.NormalizeOutgoingKeys, + NoRequestBufferGrowth: true, // Hard memory limit. + MaxPathValues: cfg.MaxPathValues, + }) + go r.goroWorker(gen, jobqueue, cfg.Mux) + } + r.pendingConns = jobqueue + r.numGoro = numgoro + } + return nil +} + +// awaitIdleExchangesLocked waits up to maxWait for exchanges of the previous +// generation to finish serving so their buffers may be reused. Requires r.mu +// held; the lock is released while waiting since [Router.freeExch] needs it to +// free the exchanges being waited on. +func (r *Router) awaitIdleExchangesLocked(maxWait time.Duration) error { + const pollInterval = time.Millisecond + for waited := time.Duration(0); ; waited += pollInterval { + busy := false + for i := range r.exchs { + if r.exchs[i].acquired.Load() { + busy = true + break + } + } + if !busy { + return nil + } else if waited >= maxWait { + return errBusyExchanges + } + r.mu.Unlock() + time.Sleep(pollInterval) + r.mu.Lock() + } +} + +// Handle takes ownership of conn and serves one exchange on it, closing it when +// done. It does not block on the exchange: the connection is handed to a +// goroutine and Handle returns immediately. +// +// Handle returns [lneto.ErrExhausted] when no exchange is free, +// [lneto.ErrPacketDrop] when the queue of connections awaiting a goroutine is +// full, and an error when the router's goroutines have been torn down. On every +// one of them conn is left untouched and unclosed for the caller to dispose of: +// refusing connections is how a router with fixed memory applies backpressure. +func (r *Router) Handle(conn io.ReadWriteCloser) error { + // Exchange acquisition and the configuration it is served with must be read + // under the same lock: [Router.Configure] may run concurrently. + r.mu.Lock() + numGoro, mux := r.numGoro, r.mux + gen := r.gen.Load() // Generation whose buffers the exchange below is sized by. + if numGoro > 0 && r.pendingConns == nil { + // Goroutines torn down: refuse before claiming an exchange. + r.mu.Unlock() + return errRouterTornDown + } + exch := r.getExchLocked(conn) + if exch == nil { + r.mu.Unlock() + return lneto.ErrExhausted + } else if numGoro == 0 { + r.mu.Unlock() + go r.goroHandle(gen, exch, mux) + return nil + } + // Enqueue under the lock: [Router.Configure] closes pendingConns while + // holding it, so an unlocked send could land on a closed channel. The send + // never blocks, so holding the lock cannot stall a worker. + var enqueued bool + select { + case r.pendingConns <- job{exch: exch}: + enqueued = true + default: + // pendingConns cannot store another Conn, we drop and return error. + exch.acquired.Store(false) // release. + } + r.mu.Unlock() + if enqueued { + return nil + } + return lneto.ErrPacketDrop +} + +func (r *Router) goroWorker(gen uint32, queue chan job, mux Mux) { + for job := range queue { + exch := job.exch + if exch == nil { + panic("httplo: unreachable nil job") + } else if gen != r.gen.Load() { + // Not released with freeExch since generation torn down, + // new buffer may have been allocated for Exchanges. + exch.Release() + continue + } + r.goroHandle(gen, exch, mux) + } +} + +func (r *Router) goroHandle(gen uint32, exch *Exchange, mux Mux) { + defer r.freeExch(gen, exch) + err := Handle(exch, mux, nil) + if err != nil { + if exch.readErr != nil { + r.error("goroHandle:ReadFromLimited", slog.String("err", err.Error())) + } else { + r.error("goroHandle:TryParse?", slog.String("err", err.Error())) + } + } +} + +// freeExch releases exch and offers it to the freelist for reuse. gen is the +// generation exch was acquired under: an exchange outliving its generation is +// dropped, its buffers being sized by a configuration the router no longer +// serves and possibly carved out of a globbuf it no longer owns. +func (r *Router) freeExch(gen uint32, exch *Exchange) { + const freelistMaxDepth = 5 + r.mu.Lock() + if gen != r.gen.Load() { + exch.Release() + r.mu.Unlock() + return + } + depth := 0 + for node := r.freeList; node != nil && depth < freelistMaxDepth; node = node.nextFree { + depth++ + } + if depth < freelistMaxDepth { + // Push at head: appending at the tail would drop every node past the + // depth limit instead of dropping the exchange we cannot store. + exch.nextFree = r.freeList + r.freeList = exch + } else { + exch.nextFree = nil // Freelist full, exchange is dropped. + } + exch.Release() + r.mu.Unlock() +} + +// getExchLocked returns an exchange acquired on conn. Requires r.mu held. +func (r *Router) getExchLocked(conn conn) (exch *Exchange) { + if r.freeList != nil { + // Successor must be read before Acquire: Acquire clears nextFree, so + // popping afterwards would truncate the freelist to the popped node. + next := r.freeList.nextFree + if r.freeList.Acquire(conn) { + exch = r.freeList + r.freeList = next + return exch + } + } + // Unbounded mode is stored as zero, see [Router.Configure]. + workerMode := r.numGoro > 0 + if workerMode { + for i := range r.exchs { + if r.exchs[i].Acquire(conn) { + return &r.exchs[i] + } + } + } else { + // Unbounded growth mode when r.numGoro==0. + exch := new(Exchange) + exch.Configure(ExchangeConfig{ + RawBuf: make([]byte, r.respBuf+r.reqBuf), + RequestBufferLim: r.reqBuf, + NumHeaderKVCap: r.reqNumHeaderCap, + NormalizeOutgoingKeys: r.normalizeKeys, + NoRequestBufferGrowth: true, + MaxPathValues: r.maxPathValues, + }) + exch.Acquire(conn) // Fresh exchange, CAS cannot fail. + return exch + } + return nil +} + +func (r *Router) error(msg string, attrs ...slog.Attr) { + internal.LogAttrs(r.log, slog.LevelError, msg, attrs...) +} + +func (r *Router) info(msg string, attrs ...slog.Attr) { + internal.LogAttrs(r.log, slog.LevelInfo, msg, attrs...) +} diff --git a/http/httphi/router_test.go b/http/httphi/router_test.go new file mode 100644 index 0000000..db78e2e --- /dev/null +++ b/http/httphi/router_test.go @@ -0,0 +1,501 @@ +package httphi + +import ( + "bytes" + "context" + "io" + "net" + "strings" + "sync" + "testing" + "time" +) + +// rwconn is a in-memory conn. The router handles connections on another +// goroutine so every field is guarded; onClose lets tests await the handler. +// +// A drained rwconn reads (0,nil) as a live socket with no data pending would, +// so a partial request can be completed with AddReadable mid-handling. Tests +// that need the peer to hang up call Hangup. +type rwconn struct { + mu sync.Mutex + readable bytes.Buffer + segments []string + written bytes.Buffer + closed bool + hangup bool + failWr int + onClose chan struct{} + deadline time.Time +} + +// newConn returns a conn preloaded with request and whose Close is observable +// with [rwconn.AwaitClose]. +func newConn(request string) *rwconn { + r := &rwconn{onClose: make(chan struct{})} + r.AddReadable([]byte(request)) + return r +} + +// AddSegment queues data delivered on a later read, once everything already +// pending has been read. Models a request split over several TCP segments +// without depending on goroutine scheduling. +func (r *rwconn) AddSegment(b string) { + r.mu.Lock() + defer r.mu.Unlock() + r.segments = append(r.segments, b) +} + +// FailWrites makes the next n writes fail, as a conn refusing further data +// would. Later writes succeed. +func (r *rwconn) FailWrites(n int) { + r.mu.Lock() + defer r.mu.Unlock() + r.failWr = n +} + +// Hangup makes reads past the pending data return [io.EOF], as a peer that +// closed its side of the connection would. +func (r *rwconn) Hangup() { + r.mu.Lock() + defer r.mu.Unlock() + r.hangup = true +} + +func (r *rwconn) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + if !r.closed { + r.closed = true + if r.onClose != nil { + close(r.onClose) + } + } + return nil +} + +// AwaitClose blocks until the connection is closed by its handler or timeout elapses. +func (r *rwconn) AwaitClose(t *testing.T, timeout time.Duration) { + t.Helper() + select { + case <-r.onClose: + case <-time.After(timeout): + t.Fatal("timed out awaiting connection close by handler") + } +} + +func (r *rwconn) Read(b []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return 0, net.ErrClosed + } else if r.deadlineExceeded() { + return 0, context.DeadlineExceeded + } else if r.readable.Len() == 0 { + if len(r.segments) > 0 { + r.readable.WriteString(r.segments[0]) + r.segments = r.segments[1:] + return r.readable.Read(b) + } + if r.hangup { + return 0, io.EOF + } + return 0, nil // No data pending, handler backs off and retries. + } + return r.readable.Read(b) +} +func (r *rwconn) Write(b []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return 0, net.ErrClosed + } else if r.deadlineExceeded() { + return 0, context.DeadlineExceeded + } else if r.failWr > 0 { + r.failWr-- + return 0, io.ErrShortWrite + } + return r.written.Write(b) +} +func (r *rwconn) deadlineExceeded() bool { + return !r.deadline.IsZero() && time.Since(r.deadline) > 0 +} +func (r *rwconn) AddReadable(b []byte) { + r.mu.Lock() + defer r.mu.Unlock() + r.readable.Write(b) +} + +// SetDeadline makes reads and writes past t fail, as a conn with a read +// deadline set would. +func (r *rwconn) SetDeadline(t time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + r.deadline = t +} + +// IsClosed reports whether the connection was closed by its handler. +func (r *rwconn) IsClosed() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.closed +} + +func (r *rwconn) ViewWritten() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.written.String() +} + +var _ Mux = (*MuxSlice)(nil) + +func configSynchronousRouter(t *testing.T, router *Router, bufferSize int, mux Mux) { + err := router.Configure(RouterConfig{ + FixedNumGoroutines: -1, + Mux: mux, + RequestHeaderBufferSize: bufferSize, + RequestNumHeaderKVCap: 16, + ResponseHeaderMinBufferSize: bufferSize, + }) + if err != nil { + t.Fatal(err) + } +} + +func TestRouterGet(t *testing.T) { + const bufferSize = 1024 + const expectResponse = "its time" + var ( + sm MuxSlice + router Router + ) + sm.Handle("GET /", staticPage(t, expectResponse)) + configSynchronousRouter(t, &router, bufferSize, &sm) + + conn := newConn("GET / HTTP/1.1\r\nHost: tinygo.org\r\n\r\n") + err := router.Handle(conn) + if err != nil { + t.Fatal(err) + } + conn.AwaitClose(t, time.Second) + + got := conn.ViewWritten() + if !strings.HasPrefix(got, "HTTP/1.1 200 OK\r\n") { + t.Errorf("want 200 status line, got %q", got) + } + if !strings.HasSuffix(got, expectResponse) { + t.Errorf("want body %q at end of response, got %q", expectResponse, got) + } +} + +// The handler observes the request line and header fields the router parsed. +func TestRouterRequestVisibleToHandler(t *testing.T) { + const bufferSize = 1024 + var ( + sm MuxSlice + router Router + ) + var gotMethod, gotURI, gotHost string + sm.Handle("GET /index.html", func(ex *Exchange) { + gotMethod = string(ex.RequestMethod()) + gotURI = string(ex.RequestTarget()) + gotHost = string(ex.RequestHeader("Host")) + ex.WriteHeader(200) + }) + configSynchronousRouter(t, &router, bufferSize, &sm) + + conn := newConn("GET /index.html HTTP/1.1\r\nHost: tinygo.org\r\n\r\n") + if err := router.Handle(conn); err != nil { + t.Fatal(err) + } + conn.AwaitClose(t, time.Second) + + if gotMethod != "GET" { + t.Errorf("want method %q, got %q", "GET", gotMethod) + } + if gotURI != "/index.html" { + t.Errorf("want URI %q, got %q", "/index.html", gotURI) + } + if gotHost != "tinygo.org" { + t.Errorf("want Host %q, got %q", "tinygo.org", gotHost) + } +} + +// Router must route on method and URI, and must not invoke a handler for +// requests it has no registration for. +func TestRouterMux(t *testing.T) { + const bufferSize = 1024 + for _, test := range []struct { + name string + request string + want string // Response body the matched handler must have written. + wantNoHandler bool // No registration matches: the router must answer 404 itself. + }{ + {name: "get root", request: "GET / HTTP/1.1\r\nHost: h\r\n\r\n", want: "root"}, + {name: "get page", request: "GET /page HTTP/1.1\r\nHost: h\r\n\r\n", want: "page"}, + {name: "any method", request: "DELETE /any HTTP/1.1\r\nHost: h\r\n\r\n", want: "any"}, + {name: "method mismatch", request: "POST / HTTP/1.1\r\nHost: h\r\n\r\n", wantNoHandler: true}, + {name: "unknown uri", request: "GET /nowhere HTTP/1.1\r\nHost: h\r\n\r\n", wantNoHandler: true}, + } { + t.Run(test.name, func(t *testing.T) { + var ( + sm MuxSlice + router Router + ) + sm.Handle("GET /", staticPage(t, "root")) + sm.Handle("GET /page", staticPage(t, "page")) + sm.Handle("/any", staticPage(t, "any")) // No method: matches any. + configSynchronousRouter(t, &router, bufferSize, &sm) + + conn := newConn(test.request) + if err := router.Handle(conn); err != nil { + t.Fatal(err) + } + conn.AwaitClose(t, time.Second) + + got := conn.ViewWritten() + _, body, found := strings.Cut(got, "\r\n\r\n") + if !found { + t.Fatalf("header block never terminated: %q", got) + } + if test.wantNoHandler { + if !strings.HasPrefix(got, "HTTP/1.1 404 ") { + t.Errorf("want a 404 answer, got %q", got) + } + if body != "" { + t.Errorf("no handler must run, got body %q", body) + } + return + } + if !strings.HasPrefix(got, "HTTP/1.1 200 OK\r\n") { + t.Errorf("want a 200 answer, got %q", got) + } + if body != test.want { + t.Errorf("want body %q, got %q", test.want, body) + } + }) + } +} + +// A request arriving in pieces (TCP segmentation) must still be handled. +func TestRouterSplitRequest(t *testing.T) { + const bufferSize = 1024 + const expectResponse = "split ok" + var ( + sm MuxSlice + router Router + ) + sm.Handle("GET /", staticPage(t, expectResponse)) + configSynchronousRouter(t, &router, bufferSize, &sm) + + conn := newConn("GET / HTTP/1.1\r\nHo") + conn.AddSegment("st: tinygo.org\r\n\r") + conn.AddSegment("\n") // Final CRLF lands in its own segment. + if err := router.Handle(conn); err != nil { + t.Fatal(err) + } + conn.AwaitClose(t, time.Second) + + if got := conn.ViewWritten(); !strings.HasSuffix(got, expectResponse) { + t.Errorf("want body %q, got response %q", expectResponse, got) + } +} + +func staticPage(t *testing.T, page string) HandlerFunc { + return func(ex *Exchange) { + var rw ExchangeRW // Streaming APIs take the ReadWriter view. + ex.ReadWriter(&rw) + n, err := io.WriteString(&rw, page) + // Handler runs on the router goroutine: Error, never Fatal. + if err != nil { + t.Error(err) + } else if n != len(page) { + t.Error("expected written ", len(page), "got", n) + } + } +} + +// An exchange freed by the outgoing generation carries that generation's +// buffers. Recycling it under a new configuration serves the request with +// buffer limits the new [RouterConfig] never asked for. +func TestRouterReconfigureDropsStaleExchanges(t *testing.T) { + const smallBuf, largeBuf = 256, 1024 + var ( + sm MuxSlice + router Router + ) + bufsize := make(chan int, 2) + sm.Handle("GET /", func(ex *Exchange) { bufsize <- len(ex.UnsafeRawBuffer()) }) + serve := func(want int) { + t.Helper() + conn := newConn("GET / HTTP/1.1\r\nHost: h\r\n\r\n") + conn.Hangup() + if err := router.Handle(conn); err != nil { + t.Fatal(err) + } + if got := <-bufsize; got != want { + t.Errorf("want exchange buffer %d, got %d", want, got) + } + conn.AwaitClose(t, time.Second) // Exchange hits the freelist on close. + } + + configSynchronousRouter(t, &router, smallBuf, &sm) + serve(2 * smallBuf) + configSynchronousRouter(t, &router, largeBuf, &sm) + serve(2 * largeBuf) +} + +// Configure writes the fields Handle reads; concurrent use must not race. +func TestRouterConfigureHandleRace(t *testing.T) { + const bufferSize = 1024 + var ( + sm MuxSlice + router Router + ) + sm.Handle("GET /", staticPage(t, "ok")) + configSynchronousRouter(t, &router, bufferSize, &sm) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + configSynchronousRouter(t, &router, bufferSize, &sm) + }() + go func() { + defer wg.Done() + conn := newConn("GET / HTTP/1.1\r\nHost: h\r\n\r\n") + if err := router.Handle(conn); err != nil { + t.Error(err) + } + }() + wg.Wait() +} + +// Reconfiguring a running router tears down the job queue that Handle may be +// sending a connection on. Connections may be dropped, but never panic. +// A torn down router has nothing left to serve with: it must say so instead of +// dropping the connection as if it were merely busy. +func TestRouterHandleAfterTeardown(t *testing.T) { + var ( + sm MuxSlice + router Router + ) + sm.Handle("GET /", staticPage(t, "ok")) + err := router.Configure(RouterConfig{ + FixedNumGoroutines: 2, + MaxAwaitingConns: 4, + Mux: &sm, + RequestHeaderBufferSize: 512, + RequestNumHeaderKVCap: 16, + ResponseHeaderMinBufferSize: 512, + }) + if err != nil { + t.Fatal(err) + } + router.Shutdown() + + conn := newConn("GET / HTTP/1.1\r\nHost: h\r\n\r\n") + if err = router.Handle(conn); err != errRouterTornDown { + t.Errorf("want errRouterTornDown, got %v", err) + } + if conn.IsClosed() { + t.Error("refused connection must be left for the caller to dispose of") + } +} + +// Tearing down a generation abandons the connections queued for it. They were +// taken ownership of by Handle, so they must be closed and their exchanges +// released: an exchange left claimed by a torn down generation is a buffer the +// router can never reconfigure again. +func TestRouterTeardownReleasesQueuedConns(t *testing.T) { + var ( + sm MuxSlice + router Router + ) + const numGoro = 2 + sm.Handle("GET /", staticPage(t, "ok")) + cfg := RouterConfig{ + FixedNumGoroutines: numGoro, + MaxAwaitingConns: 4, + Mux: &sm, + RequestHeaderBufferSize: 512, + RequestNumHeaderKVCap: 16, + ResponseHeaderMinBufferSize: 512, + } + // Handing connections over and tearing down immediately leaves them queued + // for workers that will never serve them. Rounds bound the scheduling luck + // needed; a single leaked exchange also fails every later Configure. + var conns [numGoro]*rwconn + for range 30 { + // errBusyExchanges is legitimate backpressure while the previous + // generation drops its connections, but it must not outlive it. + var err error + for range 100 { + if err = router.Configure(cfg); err == nil { + break + } + time.Sleep(time.Millisecond) + } + if err != nil { + t.Fatal("reconfigure after teardown:", err) + } + for i := range conns { + conns[i] = newConn("GET / HTTP/1.1\r\nHost: h\r\n\r\n") + conns[i].Hangup() + if err := router.Handle(conns[i]); err != nil { + t.Fatal(err) + } + } + router.Shutdown() + for i := range conns { + // Handle took ownership of the connection: served or dropped, the + // router closes it. + conns[i].AwaitClose(t, time.Second) + } + } +} + +func TestRouterConfigureDuringWorkerHandle(t *testing.T) { + var ( + sm MuxSlice + router Router + ) + sm.Handle("GET /", staticPage(t, "ok")) + cfg := RouterConfig{ + FixedNumGoroutines: 2, + MaxAwaitingConns: 4, + Mux: &sm, + RequestHeaderBufferSize: 512, + RequestNumHeaderKVCap: 16, + ResponseHeaderMinBufferSize: 512, + } + if err := router.Configure(cfg); err != nil { + t.Fatal(err) + } + defer router.Shutdown() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for range 300 { + conn := newConn("GET / HTTP/1.1\r\nHost: h\r\n\r\n") + conn.Hangup() + router.Handle(conn) // Drops are fine, panics are not. + } + }() + go func() { + defer wg.Done() + // Each Configure sleeps 5ms tearing down the previous generation, keep + // the count low and let the Handle loop supply the concurrency. + for range 20 { + // errBusyExchanges is legitimate backpressure: the previous + // generation was still serving when the buffers were needed. + if err := router.Configure(cfg); err != nil && err != errBusyExchanges { + t.Error(err) + return + } + } + }() + wg.Wait() +} diff --git a/http/httphi/status.go b/http/httphi/status.go new file mode 100644 index 0000000..a45ded6 --- /dev/null +++ b/http/httphi/status.go @@ -0,0 +1,273 @@ +package httphi + +// StatusText returns a text for the HTTP status code. It returns the empty +// string if the code is unknown. +func StatusText(code int) string { + switch status(code) { + case StatusContinue: + return "Continue" + case StatusSwitchingProtocols: + return "Switching Protocols" + case StatusProcessing: + return "Processing" + case StatusEarlyHints: + return "Early Hints" + case StatusOK: + return "OK" + case StatusCreated: + return "Created" + case StatusAccepted: + return "Accepted" + case StatusNonAuthoritativeInfo: + return "Non-Authoritative Information" + case StatusNoContent: + return "No Content" + case StatusResetContent: + return "Reset Content" + case StatusPartialContent: + return "Partial Content" + case StatusMultiStatus: + return "Multi-Status" + case StatusAlreadyReported: + return "Already Reported" + case StatusIMUsed: + return "IM Used" + case StatusMultipleChoices: + return "Multiple Choices" + case StatusMovedPermanently: + return "Moved Permanently" + case StatusFound: + return "Found" + case StatusSeeOther: + return "See Other" + case StatusNotModified: + return "Not Modified" + case StatusUseProxy: + return "Use Proxy" + case StatusTemporaryRedirect: + return "Temporary Redirect" + case StatusPermanentRedirect: + return "Permanent Redirect" + case StatusBadRequest: + return "Bad Request" + case StatusUnauthorized: + return "Unauthorized" + case StatusPaymentRequired: + return "Payment Required" + case StatusForbidden: + return "Forbidden" + case StatusNotFound: + return "Not Found" + case StatusMethodNotAllowed: + return "Method Not Allowed" + case StatusNotAcceptable: + return "Not Acceptable" + case StatusProxyAuthRequired: + return "Proxy Authentication Required" + case StatusRequestTimeout: + return "Request Timeout" + case StatusConflict: + return "Conflict" + case StatusGone: + return "Gone" + case StatusLengthRequired: + return "Length Required" + case StatusPreconditionFailed: + return "Precondition Failed" + case StatusRequestEntityTooLarge: + return "Request Entity Too Large" + case StatusRequestURITooLong: + return "Request URI Too Long" + case StatusUnsupportedMediaType: + return "Unsupported Media Type" + case StatusRequestedRangeNotSatisfiable: + return "Requested Range Not Satisfiable" + case StatusExpectationFailed: + return "Expectation Failed" + case StatusTeapot: + return "I'm a teapot" + case StatusMisdirectedRequest: + return "Misdirected Request" + case StatusUnprocessableEntity: + return "Unprocessable Entity" + case StatusLocked: + return "Locked" + case StatusFailedDependency: + return "Failed Dependency" + case StatusTooEarly: + return "Too Early" + case StatusUpgradeRequired: + return "Upgrade Required" + case StatusPreconditionRequired: + return "Precondition Required" + case StatusTooManyRequests: + return "Too Many Requests" + case StatusRequestHeaderFieldsTooLarge: + return "Request Header Fields Too Large" + case StatusUnavailableForLegalReasons: + return "Unavailable For Legal Reasons" + case StatusInternalServerError: + return "Internal Server Error" + case StatusNotImplemented: + return "Not Implemented" + case StatusBadGateway: + return "Bad Gateway" + case StatusServiceUnavailable: + return "Service Unavailable" + case StatusGatewayTimeout: + return "Gateway Timeout" + case StatusHTTPVersionNotSupported: + return "HTTP Version Not Supported" + case StatusVariantAlsoNegotiates: + return "Variant Also Negotiates" + case StatusInsufficientStorage: + return "Insufficient Storage" + case StatusLoopDetected: + return "Loop Detected" + case StatusNotExtended: + return "Not Extended" + case StatusNetworkAuthenticationRequired: + return "Network Authentication Required" + default: + return "" + } +} + +const () + +type status int + +// HTTP status codes as registered with IANA. +// See: https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml +const ( + // RFC 9110, 15.2.1 + StatusContinue = 100 // Continue + // RFC 9110, 15.2.2 + StatusSwitchingProtocols = 101 // Switching Protocols + // RFC 2518, 10.1 + StatusProcessing = 102 // Processing + // RFC 8297 + StatusEarlyHints = 103 // Early Hints + + // RFC 9110, 15.3.1 + StatusOK = 200 // OK + // RFC 9110, 15.3.2 + StatusCreated = 201 // Created + // RFC 9110, 15.3.3 + StatusAccepted = 202 // Accepted + // RFC 9110, 15.3.4 + StatusNonAuthoritativeInfo = 203 // Non-Authoritative Information + // RFC 9110, 15.3.5 + StatusNoContent = 204 // No Content + // RFC 9110, 15.3.6 + StatusResetContent = 205 // Reset Content + // RFC 9110, 15.3.7 + StatusPartialContent = 206 // Partial Content + // RFC 4918, 11.1 + StatusMultiStatus = 207 // Multi-Status + // RFC 5842, 7.1 + StatusAlreadyReported = 208 // Already Reported + // RFC 3229, 10.4.1 + StatusIMUsed = 226 // IM Used + + // RFC 9110, 15.4.1 + StatusMultipleChoices = 300 // Multiple Choices + // RFC 9110, 15.4.2 + StatusMovedPermanently = 301 // Moved Permanently + // RFC 9110, 15.4.3 + StatusFound = 302 // Found + // RFC 9110, 15.4.4 + StatusSeeOther = 303 // See Other + // RFC 9110, 15.4.5 + StatusNotModified = 304 // Not Modified + // RFC 9110, 15.4.6 + StatusUseProxy = 305 // Use Proxy + // RFC 9110, 15.4.7 (Unused) + _ = 306 + // RFC 9110, 15.4.8 + StatusTemporaryRedirect = 307 // Temporary Redirect + // RFC 9110, 15.4.9 + StatusPermanentRedirect = 308 // Permanent Redirect + + // RFC 9110, 15.5.1 + StatusBadRequest = 400 // Bad Request + // RFC 9110, 15.5.2 + StatusUnauthorized = 401 // Unauthorized + // RFC 9110, 15.5.3 + StatusPaymentRequired = 402 // Payment Required + // RFC 9110, 15.5.4 + StatusForbidden = 403 // Forbidden + // RFC 9110, 15.5.5 + StatusNotFound = 404 // Not Found + // RFC 9110, 15.5.6 + StatusMethodNotAllowed = 405 // Method Not Allowed + // RFC 9110, 15.5.7 + StatusNotAcceptable = 406 // Not Acceptable + // RFC 9110, 15.5.8 + StatusProxyAuthRequired = 407 // Proxy Authentication Required + // RFC 9110, 15.5.9 + StatusRequestTimeout = 408 // Request Timeout + // RFC 9110, 15.5.10 + StatusConflict = 409 // Conflict + // RFC 9110, 15.5.11 + StatusGone = 410 // Gone + // RFC 9110, 15.5.12 + StatusLengthRequired = 411 // Length Required + // RFC 9110, 15.5.13 + StatusPreconditionFailed = 412 // Precondition Failed + // RFC 9110, 15.5.14 + StatusRequestEntityTooLarge = 413 // Request Entity Too Large + // RFC 9110, 15.5.15 + StatusRequestURITooLong = 414 // Request URI Too Long + // RFC 9110, 15.5.16 + StatusUnsupportedMediaType = 415 // Unsupported Media Type + // RFC 9110, 15.5.17 + StatusRequestedRangeNotSatisfiable = 416 // Requested Range Not Satisfiable + // RFC 9110, 15.5.18 + StatusExpectationFailed = 417 // Expectation Failed + // RFC 9110, 15.5.19 (Unused) + StatusTeapot = 418 // I'm a teapot + // RFC 9110, 15.5.20 + StatusMisdirectedRequest = 421 // Misdirected Request + // RFC 9110, 15.5.21 + StatusUnprocessableEntity = 422 // Unprocessable Entity + // RFC 4918, 11.3 + StatusLocked = 423 // Locked + // RFC 4918, 11.4 + StatusFailedDependency = 424 // Failed Dependency + // RFC 8470, 5.2. + StatusTooEarly = 425 // Too Early + // RFC 9110, 15.5.22 + StatusUpgradeRequired = 426 // Upgrade Required + // RFC 6585, 3 + StatusPreconditionRequired = 428 // Precondition Required + // RFC 6585, 4 + StatusTooManyRequests = 429 // Too Many Requests + // RFC 6585, 5 + StatusRequestHeaderFieldsTooLarge = 431 // Request Header Fields Too Large + // RFC 7725, 3 + StatusUnavailableForLegalReasons = 451 // Unavailable For Legal Reasons + + // RFC 9110, 15.6.1 + StatusInternalServerError = 500 // Internal Server Error + // RFC 9110, 15.6.2 + StatusNotImplemented = 501 // Not Implemented + // RFC 9110, 15.6.3 + StatusBadGateway = 502 // Bad Gateway + // RFC 9110, 15.6.4 + StatusServiceUnavailable = 503 // Service Unavailable + // RFC 9110, 15.6.5 + StatusGatewayTimeout = 504 // Gateway Timeout + // RFC 9110, 15.6.6 + StatusHTTPVersionNotSupported = 505 // HTTP Version Not Supported + // RFC 2295, 8.1 + StatusVariantAlsoNegotiates = 506 // Variant Also Negotiates + // RFC 4918, 11.5 + StatusInsufficientStorage = 507 // Insufficient Storage + // RFC 5842, 7.2 + StatusLoopDetected = 508 // Loop Detected + // RFC 2774, 7 + StatusNotExtended = 510 // Not Extended + // RFC 6585, 6 + StatusNetworkAuthenticationRequired = 511 // Network Authentication Required +) diff --git a/http/httphi/stringers.go b/http/httphi/stringers.go new file mode 100644 index 0000000..71dc256 --- /dev/null +++ b/http/httphi/stringers.go @@ -0,0 +1,34 @@ +// Code generated by "stringer -type Method -linecomment -output stringers.go"; DO NOT EDIT. + +package httphi + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[MethUndefined-0] + _ = x[MethGet-1] + _ = x[MethHead-2] + _ = x[MethPost-3] + _ = x[MethPut-4] + _ = x[MethPatch-5] + _ = x[MethDelete-6] + _ = x[MethConnect-7] + _ = x[MethOptions-8] + _ = x[MethTrace-9] + _ = x[MethUnknown-10] +} + +const _Method_name = "undefinedGETHEADPOSTPUTPATCHDELETECONNECTOPTIONSTRACEunknown" + +var _Method_index = [...]uint8{0, 9, 12, 16, 20, 23, 28, 34, 41, 48, 53, 60} + +func (i Method) String() string { + idx := int(i) - 0 + if i < 0 || idx >= len(_Method_index)-1 { + return "Method(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Method_name[_Method_index[idx]:_Method_index[idx+1]] +} diff --git a/http/httpraw/cookie.go b/http/httpraw/cookie.go index ea8b530..00c3717 100644 --- a/http/httpraw/cookie.go +++ b/http/httpraw/cookie.go @@ -7,105 +7,89 @@ import ( // Cookie implements cookie key-value parsing. Methods function similarly to eponymous [Header] methods. // Cookie represents a single-line Cookie header value in a HTTP header, much like the standard library Cookie. type Cookie struct { - buf []byte - kvs []argsKV // first key-value pair is the data Key/Value pair. + kv kvBuffer +} + +// EnableBufferGrowth allows the cookie's buffer to grow past what [Cookie.Reset] was +// handed. See [kvBuffer.EnableBufferGrowth]. +func (c *Cookie) EnableBufferGrowth(enableBufferGrowth bool) { + c.kv.EnableBufferGrowth(enableBufferGrowth) } // Reset functions very similarly to [Header.Reset]. Can be used for in-place cookie parsing. -func (c *Cookie) Reset(buf []byte) { - if buf == nil { - buf = c.buf[:0] - } - *c = Cookie{ - buf: buf, - kvs: c.kvs[:0], - } +func (c *Cookie) Reset(buf []byte, capKV int) { c.kv.Reset(buf, capKV) } + +func (c *Cookie) valid() bool { + return len(c.kv.kvs) > 0 && c.kv.kvs[0].key.len > 0 } // Name returns the first cookie key which is commonly referred to as the cookie's name. Returns nil if not found. func (c *Cookie) Name() []byte { - if len(c.kvs) == 0 || c.kvs[0].key.len == 0 { + if !c.valid() { return nil } - return tok2bytes(c.buf, c.kvs[0].key) + return c.kv.AtKey(0) } // Value returns the first cookie value associated with the name. Returns nil if not found. func (c *Cookie) Value() []byte { - if len(c.kvs) == 0 || c.kvs[0].value.len == 0 { + if !c.valid() { return nil } - return tok2bytes(c.buf, c.kvs[0].value) + return c.kv.AtValue(0) } // ParseBytes copies the argument bytes to the Cookie's underlying buffer and parses the cookie. func (c *Cookie) ParseBytes(cookie []byte) error { - c.Reset(nil) - c.buf = append(c.buf[:0], cookie...) + c.Reset(nil, 0) + c.kv.buf = append(c.kv.buf[:0], cookie...) return c.Parse() } // CopyFrom makes a copy of the argument cookie to the receiver dst argument. No memory is shared between cookies. -func (dst *Cookie) CopyFrom(c Cookie) { - dst.buf = append(dst.buf[:0], c.buf...) - dst.kvs = append(dst.kvs[:0], c.kvs...) -} +func (dst *Cookie) CopyFrom(c Cookie) { dst.kv.CopyFrom(&c.kv) } // Parse parses the cookie's buffer in place. func (c *Cookie) Parse() error { - if len(c.kvs) > 0 { + if c.kv.Len() > 0 { return errCookiesParsed } off := 0 for { - k, v, n := parseCookie(c.buf[off:]) + k, v, n := parseCookie(c.kv.buf[off:]) if n == 0 { break } - c.kvs = append(c.kvs, argsKV{ - key: bytes2tok(c.buf, k), - value: bytes2tok(c.buf, v), - }) + if !c.kv.setInternal(k, v) { + return ErrBufferExhausted + } + off += n } - if len(c.kvs) == 0 { + if c.kv.Len() == 0 { return errNoCookies } return nil } -func (c *Cookie) ForEach(cb func(key, value []byte) error) error { - nc := len(c.kvs) - for i := range nc { - kv := c.kvs[i] - key := tok2bytes(c.buf, kv.key) - value := tok2bytes(c.buf, kv.value) - err := cb(key, value) - if err != nil { - return err - } - } - return nil +// ForEach iterates over the cookie's key-value pairs, stopping on the first +// error returned by cb and returning it. +func (c *Cookie) ForEach(cb func(key, value []byte) bool) { + c.kv.ForEach(cb) } // Get gets a cookie's value from its key. Use HasValueOrKey to check if a key or single-valued cookie is present in the cookie. -func (c *Cookie) Get(key string) []byte { - nc := len(c.kvs) - for i := range nc { - kv := c.kvs[i] - if b2s(tok2bytes(c.buf, kv.key)) == key { - return tok2bytes(c.buf, kv.value) - } - } - return nil -} +func (c *Cookie) Get(key string) []byte { return c.kv.Get(key) } +// HasKeyOrSingleValue returns true if the cookie contains a pair with the given +// key or a valueless attribute with the given text, i.e: "Secure" or "HttpOnly". +// It cannot defer to [KVBuffer.Present]: parseCookie stores a valueless +// attribute with an empty key and the text as the value, so a key-only lookup +// would never match one. func (c *Cookie) HasKeyOrSingleValue(keyOrSingleValue string) bool { - nc := len(c.kvs) - for i := range nc { - kv := c.kvs[i] - if kv.key.len == 0 && b2s(tok2bytes(c.buf, kv.value)) == keyOrSingleValue || - b2s(tok2bytes(c.buf, kv.key)) == keyOrSingleValue { + for i, nc := 0, c.kv.Len(); i < nc; i++ { + k, v := c.kv.At(i) + if (len(k) == 0 && b2s(v) == keyOrSingleValue) || b2s(k) == keyOrSingleValue { return true } } @@ -158,16 +142,14 @@ func (c *Cookie) String() string { // AppendKeyValues appends the HTTP header value of the cookie expected after the "Cookie:" string. Does not include trailing \r\n's. func (c *Cookie) AppendKeyValues(dst []byte) []byte { - nc := len(c.kvs) + nc := c.kv.Len() for i := range nc { - kv := c.kvs[i] - key := tok2bytes(c.buf, kv.key) - value := tok2bytes(c.buf, kv.value) - if len(key) != 0 { - dst = append(dst, key...) + k, v := c.kv.At(i) + if len(k) != 0 { + dst = append(dst, k...) dst = append(dst, '=') } - dst = append(dst, value...) + dst = append(dst, v...) if i+1 < nc { dst = append(dst, ';', ' ') } diff --git a/http/httpraw/form.go b/http/httpraw/form.go new file mode 100644 index 0000000..314b539 --- /dev/null +++ b/http/httpraw/form.go @@ -0,0 +1,145 @@ +package httpraw + +import "bytes" + +// Form holds "application/x-www-form-urlencoded" key-value pairs, the encoding +// HTML forms use for POST bodies and query strings alike. Methods function +// similarly to eponymous [Cookie] methods. +// +// Pairs are stored as they appear on the wire, percent-encoded and with '+' +// undecoded, until [Form.Decode] rewrites them in place. The caller bounds the +// data: Form parses the buffer it is handed and reads nothing more. +type Form struct { + kv kvBuffer +} + +// EnableBufferGrowth allows the form's buffer to grow past what [Form.Reset] was +// handed. See [kvBuffer.EnableBufferGrowth]. +func (f *Form) EnableBufferGrowth(enableGrowth bool) { f.kv.EnableBufferGrowth(enableGrowth) } + +// Reset discards parsed pairs and sets the buffer to parse in place. +// If buf is nil the current buffer is reused. +func (f *Form) Reset(buf []byte, capKV int) { + f.kv.Reset(buf, capKV) +} + +// ParseBytes copies the argument bytes to the Form's underlying buffer and parses them. +func (f *Form) ParseBytes(b []byte) error { + f.Reset(nil, 0) + if len(b) == 0 { + return nil // An empty body is an empty form, not a failure to read one. + } + err := f.kv.ReadFromBytes(b) + if err != nil { + return err + } + return f.Parse() +} + +// Parse parses the form's buffer in place. +func (f *Form) Parse() error { + f.kv.discardKVs() + key, value, rest := NextQueryPair(f.kv.buf) + for key != nil { + if !f.kv.setInternal(key, value) { + return ErrBufferExhausted + } + key, value, rest = NextQueryPair(rest) + } + return nil +} + +// Decode rewrites every key and value in place, replacing percent escapes and +// '+' with the bytes they encode. Decoding only shrinks, so no memory is added. +func (f *Form) Decode() error { + const plusAsSpace = true // Form encoded data, unlike a path. + nkvs := f.kv.Len() + for i := range nkvs { + k, v := f.kv.At(i) + nk, err := CopyDecodedPercentURL(k, k, plusAsSpace) + if err != nil { + return err + } else if len(v) == 0 { + if nk != len(k) { + f.kv.setAt(i, k[:nk], v) // k[:nk]: the decoded key is shorter. + } + continue + } + nv, err := CopyDecodedPercentURL(v, v, plusAsSpace) + if err != nil { + return err + } + if nk != len(k) || nv != len(v) { + f.kv.setAt(i, k[:nk], v[:nv]) + } + } + return nil +} + +// Len returns the amount of key-value pairs parsed. +func (f *Form) Len() int { return f.kv.Len() } + +// Pair returns the i'th key-value pair in wire order. The value is nil for a +// pair with no '=', i.e: "ok" in "ok&q=go", which distinguishes it from "ok=" +// where the value is present and empty. +func (f *Form) Pair(i int) (key, value []byte) { + return f.kv.At(i) +} + +// Get returns the value of the first pair matching key, nil if absent or if the +// pair has no value. Bytes are compared as stored, so call [Form.Decode] first +// when keys may be encoded. +func (f *Form) Get(key string) []byte { return f.kv.Get(key) } + +// Has returns true if key is present, with or without a value. +func (f *Form) Has(key string) bool { return f.kv.Present(key) } + +// AppendKeyValues appends the form's wire representation to dst and returns it. +func (f *Form) AppendKeyValues(dst []byte) []byte { + nkv := f.kv.Len() + for i := range nkv { + key, value := f.Pair(i) + if i > 0 { + dst = append(dst, '&') + } + dst = append(dst, key...) + if value != nil { + dst = append(dst, '=') + dst = append(dst, value...) + } + } + return dst +} + +// NextQueryPair splits the leading key-value pair off a query string and returns +// what remains of it. Loop until rawkey is nil: +// +// rawkey, rawval, rest := httpraw.NextQueryPair(h.RequestQuery()) +// for rawkey != nil { +// // use rawkey, rawval. +// rawkey, rawval, rest = httpraw.NextQueryPair(rest) +// } +// +// A pair with no '=' yields a nil rawval, i.e: "debug" in "?debug&q=go", which +// distinguishes it from "?debug=" where the value is present and empty. Empty +// sequences are skipped, so "?&&q=go&" yields a single pair. Only '&' separates +// pairs and only the first '=' splits a pair. +func NextQueryPair(query []byte) (rawkey, rawval, rest []byte) { + for len(query) > 0 { + pair := query + amp := bytes.IndexByte(query, '&') + if amp >= 0 { + pair, query = query[:amp], query[amp+1:] + } else { + query = nil + } + if len(pair) == 0 { + continue // Empty sequence, see WHATWG URL urlencoded parsing. + } + if before, after, ok := bytes.Cut(pair, []byte{'='}); ok { + return before, after, query + } + return pair, nil, query + } + return nil, nil, nil +} diff --git a/http/httpraw/form_test.go b/http/httpraw/form_test.go new file mode 100644 index 0000000..7545118 --- /dev/null +++ b/http/httpraw/form_test.go @@ -0,0 +1,157 @@ +package httpraw + +import ( + "strings" + "testing" +) + +// render joins a form's pairs as "key=value", a valueless key as "key". +func render(f *Form) string { + var sb strings.Builder + for i := range f.Len() { + key, value := f.Pair(i) + if i > 0 { + sb.WriteByte('|') + } + sb.Write(key) + if value != nil { + sb.WriteByte('=') + sb.Write(value) + } + } + return sb.String() +} + +func TestFormParse(t *testing.T) { + for _, test := range []struct { + body string + want string + }{ + {body: "", want: ""}, + {body: "q=go", want: "q=go"}, + {body: "name=Jos%C3%A9+P%C3%A9rez&msg=hi+there&ok=on", want: "name=Jos%C3%A9+P%C3%A9rez|msg=hi+there|ok=on"}, + {body: "ok", want: "ok"}, // Flag: no '=' at all. + {body: "ok=", want: "ok="}, // Present but empty. + {body: "&&q=go&", want: "q=go"}, // Empty sequences skipped. + {body: "tag=a&tag=b", want: "tag=a|tag=b"}, // Duplicates kept in order. + {body: "=v", want: "=v"}, // Empty name kept. + {body: "a=b=c", want: "a=b=c"}, // Only the first '=' splits. + } { + var f Form + if err := f.ParseBytes([]byte(test.body)); err != nil { + t.Fatalf("%q: %s", test.body, err) + } + if got := render(&f); got != test.want { + t.Errorf("%q: want %q, got %q", test.body, test.want, got) + } + } +} + +// Decode rewrites keys and values in place: percent escapes and '+' as space. +func TestFormDecode(t *testing.T) { + var f Form + const body = "name=Jos%C3%A9+P%C3%A9rez&a%20b=c%2Bd&ok" + if err := f.ParseBytes([]byte(body)); err != nil { + t.Fatal(err) + } + if err := f.Decode(); err != nil { + t.Fatal(err) + } + const want = "name=José Pérez|a b=c+d|ok" + if got := render(&f); got != want { + t.Errorf("want %q, got %q", want, got) + } + if got := string(f.Get("a b")); got != "c+d" { + t.Errorf("want decoded key lookup %q, got %q", "c+d", got) + } +} + +// Decoding a valueless key that shrinks must rewrite the key without inventing +// a value: the pair has no '=' before Decode and must have none after. +func TestFormDecodeValuelessKeyShrinks(t *testing.T) { + var f Form + const body = "a=1&o%6Bay" + if err := f.ParseBytes([]byte(body)); err != nil { + t.Fatal(err) + } + if err := f.Decode(); err != nil { + t.Fatal(err) + } + const want = "a=1|okay" + if got := render(&f); got != want { + t.Errorf("want %q, got %q", want, got) + } + if _, value := f.Pair(1); value != nil { + t.Errorf("want valueless pair to stay valueless, got value %q", value) + } + if !f.Has("okay") { + t.Error("want decoded valueless key present") + } +} + +// A malformed escape must be reported, never silently passed through. +func TestFormDecodeMalformed(t *testing.T) { + for _, body := range []string{"q=%zz", "%zz=v", "q=%4"} { + var f Form + if err := f.ParseBytes([]byte(body)); err != nil { + t.Fatal(err) + } + if err := f.Decode(); err == nil { + t.Errorf("%q: want decode error, got nil", body) + } + } +} + +func TestFormGetHas(t *testing.T) { + var f Form + if err := f.ParseBytes([]byte("tag=a&tag=b&ok&empty=")); err != nil { + t.Fatal(err) + } + if got := string(f.Get("tag")); got != "a" { + t.Errorf("want first value %q, got %q", "a", got) + } + if got := f.Get("ok"); got != nil { + t.Errorf("want nil value for valueless key, got %q", got) + } + if got := f.Get("nope"); got != nil { + t.Errorf("want nil for absent key, got %q", got) + } + if v := f.Get("empty"); v == nil || len(v) != 0 { + t.Errorf("want present empty value, got %v", v) + } + for _, key := range []string{"tag", "ok", "empty"} { + if !f.Has(key) { + t.Errorf("want Has(%q) true", key) + } + } + if f.Has("nope") { + t.Error("want Has(nope) false") + } +} + +func TestFormAppendKeyValues(t *testing.T) { + const body = "name=go&ok&empty=&tag=a&tag=b" + var f Form + if err := f.ParseBytes([]byte(body)); err != nil { + t.Fatal(err) + } + if got := string(f.AppendKeyValues(nil)); got != body { + t.Errorf("want round trip %q, got %q", body, got) + } +} + +// Parsing into a reused Form must not allocate: the pair storage is reused. +func TestFormParseReuseNoAlloc(t *testing.T) { + body := []byte("name=go&tag=a&tag=b&ok") + var f Form + if err := f.ParseBytes(body); err != nil { // Warm up the pair storage. + t.Fatal(err) + } + allocs := testing.AllocsPerRun(100, func() { + f.Reset(body, 0) + f.Parse() + }) + if allocs != 0 { + t.Errorf("reused Form allocated %v times, want 0", allocs) + } +} diff --git a/http/httpraw/header.go b/http/httpraw/header.go index d92ba68..8c40377 100644 --- a/http/httpraw/header.go +++ b/http/httpraw/header.go @@ -3,31 +3,40 @@ package httpraw import ( "bytes" "io" - "slices" + "strconv" ) const ( - methodGet = "GET" - strHTTP11 = "HTTP/1.1" - strCRLF = "\r\n" - headerCookie = "Cookie" - headerConnection = "Connection" - strClose = "close" + methodGet = "GET" + strHTTP11 = "HTTP/1.1" + strCRLF = "\r\n" + headerCookie = "Cookie" + headerConnection = "Connection" + headerContentLength = "Content-Length" + strClose = "close" + strKeepAlive = "keep-alive" ) -type flags uint16 +// Flags is a bitset of signals gathered while parsing or building a header, +// such as a status code having been set or the peer requesting connection +// close. See [Header.Flags]. +type Flags uint16 const ( - flagNoBufferGrow flags = 1 << iota + flagNoBufferGrow Flags = 1 << iota flagDoneParsingHeader flagOOMReached flagConnClose flagNoHTTP11 flagMangledBuffer // set when header fields appended to buffer via Add,Set calls + flagKVAppended // set after KV appended to buffer outside Read methods. flagReaderEOF + // set if [Header.SetStatus] or [Header.SetStatusInt] has been called. + FlagStatusSet ) -func (f flags) hasAny(checkThese flags) bool { +// HasAny returns true if any of the argument flags are set. +func (f Flags) HasAny(checkThese Flags) bool { return f&checkThese != 0 } @@ -42,32 +51,34 @@ type Header struct { hbuf headerBuf // Request fields. - method headerSlice - requestURI headerSlice - proto headerSlice + method view + requestTarget view + proto view // Response fields. - statusCode headerSlice - statusText headerSlice - - flags flags - _ noCopy + statusCode view + statusText view + _ noCopy } -// EnableBufferGrowth disables buffer growth during parsing if b is false. Is enabled by default. -// Disabling buffer growth prevents allocations but methods may throw errors on insufficient memory. -func (h *Header) EnableBufferGrowth(b bool) { - if !b { - h.flags |= flagNoBufferGrow - } else { - h.flags &^= flagNoBufferGrow - } +// Flags returns [Flags] to signal status code has been set, Connection:Close or other useful signals provided by flags. +func (h *Header) Flags() Flags { return h.hbuf.kv.flags } + +// ConfigBufferGrowth configures the memory the header may use. Setting +// outlives [Header.Reset]. Call before parsing/reading. +// +// enableBufferGrowth enables growing both the header buffer and the header key/value pair slice. +func (h *Header) ConfigBufferGrowth(enableBufferGrowth bool) { + h.hbuf.kv.EnableBufferGrowth(enableBufferGrowth) } // ParseBytes copies the bytes into buffer and parses the HTTP header. It fails if HTTP header data is incomplete. func (h *Header) ParseBytes(asResponse bool, b []byte) error { - h.Reset(nil) - h.hbuf.readFromBytes(b) + h.Reset(nil, 0) + err := h.hbuf.kv.ReadFromBytes(b) + if err != nil { + return err + } return h.parse(asResponse) } @@ -75,7 +86,7 @@ func (h *Header) ParseBytes(asResponse bool, b []byte) error { // It fails if HTTP data is incomplete. func (h *Header) Parse(asResponse bool) error { debuglog("http:parse:reset") - h.Reset(h.hbuf.buf) + h.Reset(h.hbuf.kv.buf, 0) debuglog("http:parse:start") return h.parse(asResponse) } @@ -97,96 +108,68 @@ func (h *Header) Parse(asResponse bool) error { // return err // } func (h *Header) TryParse(asResponse bool) (needMoreData bool, err error) { - if h.flags.hasAny(flagDoneParsingHeader) { + flags := h.Flags() + if flags.HasAny(flagDoneParsingHeader) { return false, errAlreadyParsed - } else if h.flags.hasAny(flagMangledBuffer) { + } else if flags.HasAny(flagMangledBuffer) { return false, errMangledBuffer } - if asResponse && h.statusCode.len == 0 || !asResponse && h.requestURI.start == 0 { + if asResponse && h.statusCode.len == 0 || !asResponse && h.requestTarget.start == 0 { err = h.parseFirstLine(asResponse) if err != nil { - return err == errNeedMore, err + return err == ErrNeedMoreData, err } } - err = h.parseNextHeaders() - return err == errNeedMore, err + err = h.parseNextHeaders(flags) + return err == ErrNeedMoreData, err } // ParsingSuccess returns true if TryParse was successful, that is to say it returned needMoreData==false and err==nil. func (h *Header) ParsingSuccess() bool { - return h.flags.hasAny(flagDoneParsingHeader) + return h.Flags().HasAny(flagDoneParsingHeader) } // ReadFromLimited reads at most maxBytesToRead from reader and appends them to underlying buffer. // Used to accumulate HTTP header for later parsing with [Header.TryParse]. // If read is successful (read length>0) and reader returns [io.EOF] then ReadFromLimited will return a nil error. func (h *Header) ReadFromLimited(r io.Reader, maxBytesToRead int) (int, error) { - if maxBytesToRead <= 0 { - return 0, errSmallBuffer - } else if h.flags.hasAny(flagMangledBuffer) { - return 0, errMangledBuffer - } - free := h.BufferFree() - if free < maxBytesToRead { - if h.flags.hasAny(flagNoBufferGrow) { - return 0, errSmallBuffer - } - h.hbuf.buf = slices.Grow(h.hbuf.buf, maxBytesToRead) - } - blen := len(h.hbuf.buf) - b := h.hbuf.buf[blen:min(blen+maxBytesToRead, cap(h.hbuf.buf))] - n, err := r.Read(b) - if err != nil && err == io.EOF { - h.flags |= flagReaderEOF - if n > 0 { - err = nil // Nil-out error if read was succesful so as to not spook readers. - } - } - h.hbuf.buf = h.hbuf.buf[:blen+n] - return n, err + return h.hbuf.kv.ReadLimited(r, maxBytesToRead) } // ReadFromBytes appends argument buffer to underlying buffer. // Used to accumulate HTTP header for later parsing with [Header.TryParse]. -func (h *Header) ReadFromBytes(b []byte) (int, error) { - if len(b) == 0 { - return 0, errSmallBuffer - } - free := h.BufferFree() - if free < len(b) { - if h.flags.hasAny(flagNoBufferGrow) { - return 0, errSmallBuffer - } - h.hbuf.buf = slices.Grow(h.hbuf.buf, len(b)) - } - h.hbuf.readFromBytes(b) - return len(b), nil +func (h *Header) ReadFromBytes(b []byte) error { + return h.hbuf.kv.ReadFromBytes(b) } // BufferReceived returns the amoung of bytes read during calls to Read* methods. // Returns 0 if buffer is invalid/mangled. func (h *Header) BufferReceived() int { - if h.flags.hasAny(flagMangledBuffer | flagOOMReached) { + if h.Flags().HasAny(flagMangledBuffer | flagOOMReached) { return 0 } - return len(h.hbuf.buf) + return len(h.hbuf.kv.BufferRaw()) } // BufferParsed returns the amount of bytes parsed during a call to Parse* methods. // If the Parse* method completed without error then BufferParsed returns the header's length including the final "\r\n\r\n" text. // BufferParsed returns 0 if the buffer is invalid/mangled or if no header data has been parsed succesfully. func (h *Header) BufferParsed() int { - if h.flags.hasAny(flagMangledBuffer | flagOOMReached) { + if h.Flags().HasAny(flagMangledBuffer | flagOOMReached) { return 0 } return h.hbuf.off } +// BufferRaw returns the undeerlying buffer as stored currently in memory. +// The length of the returned buffer is the used portion. Capacity of returned slice is [Header.BufferCapacity]. +func (h *Header) BufferRaw() []byte { return h.hbuf.kv.BufferRaw() } + // BufferUsed returns the raw memory used. // // BufferUsed + BufferFree == BufferCapacity func (h *Header) BufferUsed() int { - return len(h.hbuf.buf) + return len(h.hbuf.kv.BufferRaw()) } // BufferFree returns amount of bytes free in underlying buffer. @@ -200,29 +183,12 @@ func (h *Header) BufferFree() int { // // BufferUsed + BufferFree == BufferCapacity func (h *Header) BufferCapacity() int { - return cap(h.hbuf.buf) + return cap(h.hbuf.kv.BufferRaw()) } // ForEach iterates over header key-value field tuples. -func (h *Header) ForEach(cb func(key, value []byte) error) error { - return h.hbuf.forEach(cb) -} - -func (hb *headerBuf) forEach(cb func(key, value []byte) error) error { - nh := len(hb.headers) - for i := range nh { - kv := hb.headers[i] - if !kv.isValid() { - continue - } - key := hb.musttoken(kv.key) - value := hb.musttoken(kv.value) - err := cb(key, value) - if err != nil { - return err - } - } - return nil +func (h *Header) ForEach(cb func(key, value []byte) bool) { + h.hbuf.kv.ForEach(cb) } // Reset discards all parsed data and sets the buffer data to buf. This method @@ -230,30 +196,28 @@ func (hb *headerBuf) forEach(cb func(key, value []byte) error) error { // data with Reset to parse data in-place. // If buf is nil then the current buffer is reused. There are 3 ways to use Reset: // -// h.Reset(prealloc[:0]); h.ParseBytes(httpHeader) // Tell header to use a pre-allocated buffer capacity. -// h.Reset(httpHeader); h.Parse() // Parse bytes in place with no copying. +// h.Reset(prealloc[:0], 16); h.ParseBytes(httpHeader) // Tell header to use a pre-allocated buffer capacity. +// h.Reset(httpHeader, 16); h.Parse() // Parse bytes in place with no copying. // h.Reset(nil) // Reuse buffer previously set in a call to Reset. -func (h *Header) Reset(buf []byte) { - if h.flags.hasAny(flagNoBufferGrow) && cap(buf) < 32 { - panic("small buffer and flagNoBufferGrow set") - } +func (h *Header) Reset(buf []byte, numHeaderCapacity int) { const persistentFlags = flagNoBufferGrow debuglog("http:reset:hbuf") - h.hbuf.reset(buf) - *h = Header{ - hbuf: h.hbuf, - flags: h.flags & persistentFlags, + h.hbuf.reset(buf, numHeaderCapacity) + if h.Flags().HasAny(flagNoBufferGrow) && h.BufferCapacity() < 32 { + panic("small buffer and flagNoBufferGrow set") } + *h = Header{hbuf: h.hbuf} debuglog("http:reset:done") } // Body returns the surplus data following headers. It is only valid as long as Parse* or Reset methods are not called. func (h *Header) Body() ([]byte, error) { debuglog("http:body") - if h.flags.hasAny(flagMangledBuffer) { + flags := h.Flags() + if flags.HasAny(flagMangledBuffer) { return nil, errMangledBuffer - } else if h.flags.hasAny(flagDoneParsingHeader) { - return h.hbuf.buf[h.hbuf.off:], nil + } else if flags.HasAny(flagDoneParsingHeader) { + return h.BufferRaw()[h.hbuf.off:], nil } return nil, errUnparsed } @@ -271,63 +235,56 @@ func (h *Header) SetInt(key string, value int64, base int) { if base < 2 || base > 36 { return // strconv.AppendInt only supports base 2..36. } - useKv := h.takeReusableSlot(key) - if useKv == nil { - h.appendHeaderInt(key, value, base) - } else { - useKv.value = h.reuseOrAppendInt(useKv.value, value, base) - } + h.hbuf.kv.SetInt(key, value, base) } // Set sets a key-value pair in the HTTP header. // Calling Set mangles the buffer. -func (h *Header) Set(key, value string) { - useKv := h.takeReusableSlot(key) - if useKv == nil { - h.appendHeader(key, value) - } else { - useKv.value = h.reuseOrAppend(useKv.value, value) - } +func (h *Header) Set(key, value string) (enoughSpace bool) { + return h.hbuf.kv.Set(key, value) } -// takeReusableSlot returns the valid key-value entry for key with the largest -// value buffer (best candidate for in-place reuse) and invalidates any other -// entries sharing the key. Returns nil if the key is not present. -func (h *Header) takeReusableSlot(key string) *argsKV { - hb := &h.hbuf - var useKv *argsKV - for i := 0; i < len(hb.headers); i++ { - // Search for key-value with largest buffer for value to store value reusing buffer. - gotkv := &hb.headers[i] - if gotkv.isValid() && b2s(hb.musttoken(gotkv.key)) == key { - if useKv == nil { - useKv = gotkv - } else if gotkv.value.len > useKv.value.len { - useKv.invalidate() - useKv = gotkv - } else { - gotkv.invalidate() - } +// Get gets the first exact-match value of a key found in the headers. Use [Header.ForEach] to find multiple values corresponding to same key. +func (h *Header) Get(key string) []byte { + return h.hbuf.kv.Get(key) +} + +// GetFold gets the first value whose key matches key under ASCII case-insensitive +// comparison, i.e: "content-length" matches "Content-Length". +// Use [Header.Get] for exact match and [Header.ForEach] to find multiple values +// corresponding to same key. +func (h *Header) GetFold(key string) []byte { + return h.hbuf.kv.GetFold(key) +} + +// NormalizeKeys normalizes all header keys. i.e: CONTENT-type -> Content-Type +func (h *Header) NormalizeKeys() { + for i, kv := range h.hbuf.kv.kvs { + if kv.isValidHeader() { + NormalizeHeaderKey(h.hbuf.kv.AtKey(i)) } } - return useKv } -// Get gets the first value of a key found in the headers. Use [Header.ForEach] to find multiple values corresponding to same key. -func (h *Header) Get(key string) []byte { - debuglog("http:get:start") - kv := h.peekHeader(key) - if kv.isValid() { - debuglog("http:get:found") - return h.hbuf.musttoken(kv.value) +// ContentLength returns the body length declared by the Content-Length field. +// If the field is not present then the returned bool is false. Will return error for invalid or non-integer value. +func (h *Header) ContentLength() (_ int64, present bool, _ error) { + value := h.GetFold(headerContentLength) + if value == nil { + return 0, false, nil } - debuglog("http:get:notfound") - return nil + value = trimOWS(value) + // Unsigned parse of 63 bits rejects a sign and anything past int64's range. + n, err := strconv.ParseInt(b2s(value), 10, 64) + if err != nil || n < 0 { + return n, true, errBadContentLength // strconv's error allocates and is not comparable. + } + return n, true, nil } // Add adds a new key-value pair to the HTTP header. Calling Add mangles the buffer. func (h *Header) Add(key, value string) { - h.appendHeader(key, value) + h.hbuf.kv.appendPair(key, value) } // Method returns HTTP request method. @@ -337,17 +294,41 @@ func (h *Header) Method() []byte { // SetMethod sets the request header's method. func (h *Header) SetMethod(method string) { - h.method = h.reuseOrAppend(h.method, method) + h.method = h.hbuf.kv.reuseOrAppend(h.method, method) } -// SetRequestURI sets RequestURI for the first HTTP request line. -func (h *Header) SetRequestURI(requestURI string) { - h.requestURI = h.reuseOrAppend(h.requestURI, requestURI) +// SetRequestTarget sets request-target (URI) for the first HTTP request line. +func (h *Header) SetRequestTarget(requestTarget string) { + h.requestTarget = h.hbuf.kv.reuseOrAppend(h.requestTarget, requestTarget) } -// RequestURI returns RequestURI from the first HTTP request line. -func (h *Header) RequestURI() []byte { - return h.getNonEmptyValue(h.requestURI) +// RequestTarget returns a view of the request-target (URI) of the first HTTP request line. +// Called Request-URI in the obsolete RFC 2616, renamed request-target by RFC 9112. +func (h *Header) RequestTarget() []byte { + return h.getNonEmptyValue(h.requestTarget) +} + +// RequestPath returns the request-target (URI) up to the query string, i.e: "/search" +// for "/search?q=go". Returns the whole target if it contains no query string. +func (h *Header) RequestPath() []byte { + target := h.RequestTarget() + before, _, ok := bytes.Cut(target, []byte{'?'}) + if !ok { + return target + } + return before +} + +// RequestQuery returns the request-target (URI) query string as it appears on the +// wire, percent-encoded and with '+' undecoded, i.e: "q=go" for "/search?q=go". +// Returns nil if the target has no query string. Iterate it with [NextQueryPair]. +func (h *Header) RequestQuery() []byte { + target := h.RequestTarget() + _, after, ok := bytes.Cut(target, []byte{'?'}) + if !ok { + return nil + } + return after } // Protocol returns the request header's HTTP protocol. Usually "HTTP/1.1". @@ -357,7 +338,7 @@ func (h *Header) Protocol() []byte { // SetProtocol sets the request header's protocol. Usually "HTTP/1.1". func (h *Header) SetProtocol(protocol string) { - h.proto = h.reuseOrAppend(h.proto, protocol) + h.proto = h.hbuf.kv.reuseOrAppend(h.proto, protocol) } // Status returns the response header's status code and status text. i.e: "200" "OK". @@ -365,28 +346,36 @@ func (h *Header) Status() (code, statusText []byte) { if h.statusCode.len == 0 { return nil, nil } - return h.hbuf.musttoken(h.statusCode), h.hbuf.musttoken(h.statusText) + return h.hbuf.kv.musttoken(h.statusCode), h.hbuf.kv.musttoken(h.statusText) } -// Status sets the response header's status code and status text. i.e: "200" "OK". +// SetStatus sets the response header's status code and status text. i.e: "200" "OK". func (h *Header) SetStatus(code, statusText string) { - h.statusCode = h.reuseOrAppend(h.statusCode, code) - h.statusText = h.reuseOrAppend(h.statusText, statusText) + h.hbuf.kv.flags |= FlagStatusSet + h.statusCode = h.hbuf.kv.reuseOrAppend(h.statusCode, code) + h.statusText = h.hbuf.kv.reuseOrAppend(h.statusText, statusText) } -func (h *Header) getNonEmptyValue(s headerSlice) []byte { +// SetStatusInt is identical to [Header.SetStatus] but performs integer to text conversion for status code. +func (h *Header) SetStatusInt(code int64, statusText string) { + h.hbuf.kv.flags |= FlagStatusSet + h.statusCode = h.hbuf.kv.reuseOrAppendInt(h.statusCode, code, 10) + h.statusText = h.hbuf.kv.reuseOrAppend(h.statusText, statusText) +} + +func (h *Header) getNonEmptyValue(s view) []byte { if s.len == 0 { return nil // If empty then value is invalid, return nil. } - return h.hbuf.musttoken(s) + return h.hbuf.kv.musttoken(s) } // AppendRequest appends the request header representation to the buffer and returns the result. func (h *Header) AppendRequest(dst []byte) ([]byte, error) { proto := h.Protocol() - if h.flags.hasAny(flagOOMReached) { - return dst, errOOM - } else if h.requestURI.len == 0 || h.method.len == 0 { + if h.hbuf.kv.flags.HasAny(flagOOMReached) { + return dst, ErrBufferExhausted + } else if h.requestTarget.len == 0 || h.method.len == 0 { return dst, errNeedMethodURI } else if len(proto) == 0 { return dst, errNoProto @@ -398,7 +387,7 @@ func (h *Header) AppendRequest(dst []byte) ([]byte, error) { } else { dst = append(dst, method...) } - uri := h.RequestURI() + uri := h.RequestTarget() dst = append(dst, ' ') dst = append(dst, uri...) @@ -413,9 +402,19 @@ func (h *Header) AppendRequest(dst []byte) ([]byte, error) { // AppendResponse appends the response header representation to the buffer and returns the result. func (h *Header) AppendResponse(dst []byte) ([]byte, error) { + dst, err := h.AppendResponseNoHeaders(dst) + if err != nil { + return dst, err + } + dst = h.AppendHeaders(dst) + return append(dst, strCRLF...), nil +} + +// AppendResponseNoHeaders appends the first line of the response containing protocol and status code/text: i.e: "HTTP/1.1 200 OK\r\n" +func (h *Header) AppendResponseNoHeaders(dst []byte) ([]byte, error) { proto := h.Protocol() - if h.flags.hasAny(flagOOMReached) { - return dst, errOOM + if h.hbuf.kv.flags.HasAny(flagOOMReached) { + return dst, ErrBufferExhausted } else if h.statusCode.len == 0 || h.statusText.len == 0 { return dst, errBadStatusCodeTxt } else if len(proto) == 0 { @@ -429,26 +428,24 @@ func (h *Header) AppendResponse(dst []byte) ([]byte, error) { dst = append(dst, ' ') dst = append(dst, text...) dst = append(dst, strCRLF...) - - dst = h.AppendHeaders(dst) - - return append(dst, strCRLF...), nil + return dst, nil } // AppendHeaders appends headers to buffer. Use AppendRequest and AppendResponse over this. // Does not append extra \r\n to end. Appends nothing if contains no headers. func (h *Header) AppendHeaders(dst []byte) []byte { - for i, n := 0, len(h.hbuf.headers); i < n; i++ { - kv := &h.hbuf.headers[i] - if kv.isValid() { - key := h.hbuf.musttoken(kv.key) - value := h.hbuf.musttoken(kv.value) - dst = appendHeaderLine(dst, b2s(key), b2s(value)) + for i, kv := range h.hbuf.kv.kvs { + if kv.isValidHeader() { + k, v := h.hbuf.kv.At(i) + dst = appendHeaderLine(dst, b2s(k), b2s(v)) } } return dst } +// String returns the header's wire representation, as a request if it has a +// request line and as a response otherwise. Returns the error text if neither +// can be built. Allocates, so it is meant for debugging and logging only. func (h *Header) String() string { buf, err := h.AppendRequest(nil) if err != nil { @@ -540,3 +537,99 @@ func CopyNormalizedHeaderValue(dst []byte, value []byte) (n int, modified bool) } return write, modified } + +// CopyDecodedPercentURL decodes percent-escapes in value into dst and returns bytes written. +// n < len(value) implies percent-escapes were decoded; the converse does not hold since +// '+' substitution preserves length. If plusAsSpace is set '+' decodes to ' ', +// which is correct for query and form-encoded data but NOT for path segments. +// On malformed escape returns n bytes written before the fault and a non-nil error. +// dst and value may only alias if &dst[0] == &value[0]. +func CopyDecodedPercentURL(dst, value []byte, plusAsSpace bool) (n int, err error) { + if len(dst) < len(value) { + panic("httpraw.CopyDecodedPercentURL: dst buffer shorter than value") + } + read := 0 + for { + escape := bytes.IndexByte(value[read:], '%') + if escape < 0 { + n += copyPlusDecoded(dst[n:], value[read:], plusAsSpace) + return n, nil + } + escape += read + n += copyPlusDecoded(dst[n:], value[read:escape], plusAsSpace) + if escape+2 >= len(value) { + return n, errBadPercentEncode // Truncated escape at end of value. + } + hi, okhi := unhexdigit(value[escape+1]) + lo, oklo := unhexdigit(value[escape+2]) + if !okhi || !oklo { + return n, errBadPercentEncode + } + // Write index n is always <= escape since decoding shrinks 3 bytes to 1, + // so writing here never clobbers an unread byte when dst aliases value. + dst[n] = hi<<4 | lo + n++ + read = escape + 3 + } +} + +// EqualDecodedPercentURL reports whether value, once decoded, equals want. It +// decodes as it compares so it needs no scratch buffer, and reports false on a +// malformed escape just as [CopyDecodedPercentURL] errors on one. +// plusAsSpace decodes '+' to ' ', correct for query and form-encoded data but +// NOT for path segments. +func EqualDecodedPercentURL(value []byte, want string, plusAsSpace bool) bool { + w := 0 + for i := 0; i < len(value); { + var c byte + switch { + case value[i] == '%': + if i+2 >= len(value) { + return false // Truncated escape at end of value. + } + hi, okhi := unhexdigit(value[i+1]) + lo, oklo := unhexdigit(value[i+2]) + if !okhi || !oklo { + return false + } + c = hi<<4 | lo + i += 3 + case plusAsSpace && value[i] == '+': + c = ' ' + i++ + default: + c = value[i] + i++ + } + if w >= len(want) || want[w] != c { + return false + } + w++ + } + return w == len(want) +} + +// copyPlusDecoded copies src to dst replacing '+' with ' ' if plusAsSpace set. +func copyPlusDecoded(dst, src []byte, plusAsSpace bool) int { + n := copy(dst, src) + if plusAsSpace { + for i := range n { + if dst[i] == '+' { + dst[i] = ' ' + } + } + } + return n +} + +func unhexdigit(c byte) (byte, bool) { + switch { + case c >= '0' && c <= '9': + return c - '0', true + case c >= 'a' && c <= 'f': + return c - 'a' + 10, true + case c >= 'A' && c <= 'F': + return c - 'A' + 10, true + } + return 0, false +} diff --git a/http/httpraw/header_test.go b/http/httpraw/header_test.go index 2d94955..449f494 100644 --- a/http/httpraw/header_test.go +++ b/http/httpraw/header_test.go @@ -2,6 +2,7 @@ package httpraw import ( "bytes" + "errors" "fmt" "net/http" "strconv" @@ -10,6 +11,8 @@ import ( "time" ) +const numHeaderCapacity = 16 + func TestHeaderParseRequest(t *testing.T) { const ( wantMethod = "GET" @@ -50,8 +53,8 @@ func TestHeaderParseRequest(t *testing.T) { if string(hdr.Method()) != wantMethod { t.Errorf("want method %s, got %q", wantMethod, hdr.Method()) } - if !bytes.Equal(hdr.RequestURI(), []byte(wantURI)) { - t.Errorf("want request URI %q, got %q", wantURI, hdr.RequestURI()) + if !bytes.Equal(hdr.RequestTarget(), []byte(wantURI)) { + t.Errorf("want request URI %q, got %q", wantURI, hdr.RequestTarget()) } contentLength, _ := strconv.Atoi(string(hdr.Get("Content-Length"))) if contentLength != len(wantMessage) { @@ -59,7 +62,7 @@ func TestHeaderParseRequest(t *testing.T) { } var c Cookie cookie := hdr.Get("Cookie") - c.Reset(cookie) + c.Reset(cookie, 0) err = c.Parse() if err != nil { t.Error(err) @@ -151,6 +154,125 @@ func strSameSite(mode http.SameSite) string { } } +func TestHeaderRequestPath(t *testing.T) { + for _, test := range []struct { + uri string + want string + }{ + {uri: "/", want: "/"}, + {uri: "/search?q=go", want: "/search"}, + {uri: "/search?", want: "/search"}, + {uri: "/a/b/c?x=1&y=2", want: "/a/b/c"}, + {uri: "/?q=go", want: "/"}, + } { + var h Header + err := h.ParseBytes(false, []byte("GET "+test.uri+" HTTP/1.1\r\nHost: h\r\n\r\n")) + if err != nil { + t.Fatal(err) + } + if got := string(h.RequestPath()); got != test.want { + t.Errorf("uri %q: want path %q, got %q", test.uri, test.want, got) + } + } +} + +func TestHeaderContentLength(t *testing.T) { + for _, test := range []struct { + field string // Extra header lines, empty for an absent field. + want int64 + wantErr error + }{ + {field: "Content-Length: 0", want: 0}, + {field: "Content-Length: 12", want: 12}, + {field: "Content-Length: 12 ", want: 12}, // OWS around the value, RFC 9110 5.6.3. + {field: "Content-Length: 9223372036854775807", want: 9223372036854775807}, + {field: ""}, // Absent field is no body and no error, RFC 9112 6.3. + {field: "Content-Length:", wantErr: errBadContentLength}, // Present but empty. + {field: "Content-Length: -1", wantErr: errBadContentLength}, // Digits only, RFC 9112 6.2. + {field: "Content-Length: 1 2", wantErr: errBadContentLength}, // Not a list. + {field: "Content-Length: 9223372036854775808", wantErr: errBadContentLength}, + } { + var h Header + raw := "POST / HTTP/1.1\r\nHost: h\r\n" + if test.field != "" { + raw += test.field + "\r\n" + } + if err := h.ParseBytes(false, []byte(raw+"\r\n")); err != nil { + t.Fatal(err) + } + got, present, err := h.ContentLength() + if err != test.wantErr { + t.Errorf("%q: want error %v, got %v", test.field, test.wantErr, err) + } else if err == nil && got != test.want { + t.Errorf("%q: want %d, got %d", test.field, test.want, got) + } + key, _, _ := strings.Cut(test.field, ":") + if strings.EqualFold(key, headerContentLength) != present { + t.Error("unexpected 'present'", test.field) + } + } +} + +func TestNextQueryPair(t *testing.T) { + for _, test := range []struct { + uri string + want string // "key=value" pairs joined by '|'; nil value shown as "key". + }{ + {uri: "/", want: ""}, + {uri: "/x?", want: ""}, + {uri: "/x?q=go", want: "q=go"}, + {uri: "/x?q=go&n=1", want: "q=go|n=1"}, + {uri: "/x?debug&q=go", want: "debug|q=go"}, // No '=' yields a nil value. + {uri: "/x?q=", want: "q="}, // Empty but present value. + {uri: "/x?&&q=go&", want: "q=go"}, // Empty sequences skipped. + {uri: "/x?=v", want: "=v"}, // Empty name is kept. + {uri: "/x?a=1&a=2", want: "a=1|a=2"}, // Duplicates all yielded. + {uri: "/x?a%20b=c%20d", want: "a%20b=c%20d"}, // Raw, undecoded. + {uri: "/x?a=b=c", want: "a=b=c"}, // Only first '=' splits. + } { + var h Header + err := h.ParseBytes(false, []byte("GET "+test.uri+" HTTP/1.1\r\nHost: h\r\n\r\n")) + if err != nil { + t.Fatal(err) + } + var got []byte + rawkey, rawval, rest := NextQueryPair(h.RequestQuery()) + for rawkey != nil { + if len(got) > 0 { + got = append(got, '|') + } + got = append(got, rawkey...) + if rawval != nil { + got = append(got, '=') + got = append(got, rawval...) + } + rawkey, rawval, rest = NextQueryPair(rest) + } + if string(got) != test.want { + t.Errorf("uri %q: want %q, got %q", test.uri, test.want, got) + } + } +} + +// A nil key ends iteration, and stopping early is just not looping again. +func TestNextQueryPairEnd(t *testing.T) { + rawkey, rawval, rest := NextQueryPair([]byte("a=1&b=2")) + if string(rawkey) != "a" || string(rawval) != "1" || string(rest) != "b=2" { + t.Fatalf("want a=1 with rest b=2, got %q=%q rest %q", rawkey, rawval, rest) + } + rawkey, _, rest = NextQueryPair(rest) + if string(rawkey) != "b" || len(rest) != 0 { + t.Fatalf("want b with empty rest, got %q rest %q", rawkey, rest) + } + if rawkey, _, _ = NextQueryPair(rest); rawkey != nil { + t.Fatalf("want nil key at end of query, got %q", rawkey) + } + // Trailing separators yield no pair rather than an empty one. + if rawkey, _, _ = NextQueryPair([]byte("&&")); rawkey != nil { + t.Fatalf("want nil key for empty sequences, got %q", rawkey) + } +} + func TestHeaderNormalizeKey(t *testing.T) { var tests = []struct { key string @@ -209,11 +331,72 @@ func TestCopyNormalizedHeaderValue(t *testing.T) { } } +func TestCopyDecodedPercentURL(t *testing.T) { + var tests = []struct { + value string + plusAsSpace bool + want string + wantErr bool + }{ + {value: "", want: ""}, + {value: "/plain/path", want: "/plain/path"}, + {value: "/a%20b", want: "/a b"}, + {value: "%41%42%43", want: "ABC"}, + {value: "%2f%2F", want: "//"}, // lower and upper case hex digits. + {value: "%25", want: "%"}, // escaped percent must not re-trigger decoding. + {value: "%2525", want: "%25"}, // decoded output is not re-scanned. + {value: "a+b", want: "a+b"}, // plus is literal in path segments. + {value: "a+b", plusAsSpace: true, want: "a b"}, + {value: "%20+%20", plusAsSpace: true, want: " "}, + {value: "/x?q=%E2%82%AC", want: "/x?q=\xe2\x82\xac"}, // multi-byte UTF-8 sequence. + // Malformed escapes must error, never pass through silently. + {value: "%zz", want: "", wantErr: true}, + {value: "ok%4", want: "ok", wantErr: true}, // truncated escape at end. + {value: "ok%", want: "ok", wantErr: true}, // bare percent at end. + {value: "a%2gb", want: "a", wantErr: true}, // second digit not hex. + {value: "a%g2b", want: "a", wantErr: true}, // first digit not hex. + } + dst := make([]byte, 256) + for _, test := range tests { + value := []byte(test.value) + n, err := CopyDecodedPercentURL(dst[:len(value)], value, test.plusAsSpace) + if test.wantErr && err == nil { + t.Errorf("%q: want error, got nil", test.value) + } else if !test.wantErr && err != nil { + t.Errorf("%q: unexpected error %s", test.value, err) + } + if got := string(dst[:n]); got != test.want { + t.Errorf("%q: want %q got %q", test.value, test.want, got) + } + // n' ' substitution preserves length. + if !test.wantErr && n < len(test.value) && test.want == test.value { + t.Errorf("%q: n=%d signals decoding but value unchanged", test.value, n) + } + if !test.wantErr && !test.plusAsSpace && test.want != test.value && n >= len(test.value) { + t.Errorf("%q: n=%d does not signal decoding of %q", test.value, n, test.want) + } + } +} + +// In-place decoding (dst aliasing value at offset 0) must yield the same result. +func TestCopyDecodedPercentURLInPlace(t *testing.T) { + const value, want = "/a%20b%2Fc%25", "/a b/c%" + buf := []byte(value) + n, err := CopyDecodedPercentURL(buf, buf, false) + if err != nil { + t.Fatal(err) + } + if got := string(buf[:n]); got != want { + t.Fatalf("want %q got %q", want, got) + } +} + func TestHeaderSetOverwrite(t *testing.T) { var h Header - h.Reset(nil) + h.Reset(nil, numHeaderCapacity) h.SetMethod("GET") - h.SetRequestURI("/") + h.SetRequestTarget("/") h.SetProtocol("HTTP/1.1") h.Set("Host", "first.example.com") @@ -234,7 +417,7 @@ func TestHeaderSetOverwrite(t *testing.T) { func TestHeaderSetBytesEmptyValue(t *testing.T) { var h Header - h.Reset(nil) + h.Reset(nil, numHeaderCapacity) h.SetBytes("X-Empty", nil) if got := h.Get("X-Empty"); len(got) != 0 { t.Errorf("want empty value, got %q", got) @@ -268,7 +451,7 @@ func TestHeader_LargeBufferOverflow(t *testing.T) { } // a complete but malformed header line with no colon must be a hard error, -// not errNeedMore (which makes a streaming parser wait forever). +// not ErrNeedMoreData (which makes a streaming parser wait forever). func TestHeader_ColonlessLineIsHardError(t *testing.T) { raw := "GET / HTTP/1.1\r\nBadHeaderNoColon\r\n\r\n" var h Header @@ -276,8 +459,8 @@ func TestHeader_ColonlessLineIsHardError(t *testing.T) { if err == nil { t.Fatal("want error on colonless header line, got nil") } - if err == errNeedMore { - t.Fatalf("colonless line reported as errNeedMore (parser would hang); want a hard error like errInvalidName") + if err == ErrNeedMoreData { + t.Fatalf("colonless line reported as ErrNeedMoreData (parser would hang); want a hard error like errInvalidName") } } @@ -289,19 +472,19 @@ func TestHeader_SplitBeforeColonStillParses(t *testing.T) { const part2 = ": example.com\r\n\r\n" var h Header - h.Reset(nil) - if _, err := h.ReadFromBytes([]byte(part1)); err != nil { + h.Reset(nil, numHeaderCapacity) + if err := h.ReadFromBytes([]byte(part1)); err != nil { t.Fatal(err) } needMore, err := h.TryParse(false) - if err != nil && err != errNeedMore { - t.Fatalf("split before colon: want errNeedMore/nil, got %v", err) + if err != nil && err != ErrNeedMoreData { + t.Fatalf("split before colon: want ErrNeedMoreData/nil, got %v", err) } if !needMore { t.Fatal("want needMoreData=true after partial input") } - if _, err := h.ReadFromBytes([]byte(part2)); err != nil { + if err := h.ReadFromBytes([]byte(part2)); err != nil { t.Fatal(err) } needMore, err = h.TryParse(false) @@ -322,7 +505,7 @@ func TestHeader_AppendHeaderExactCapNoPanic(t *testing.T) { const key, value = "K", "V" buf := make([]byte, 0, len(key)+len(value)) // exact cap, no slack. var h Header - h.Reset(buf) + h.Reset(buf, numHeaderCapacity) defer func() { if r := recover(); r != nil { t.Fatalf("appendHeader panicked on exact-cap buffer: %v", r) @@ -339,10 +522,10 @@ func TestHeader_AppendHeaderExactCapNoPanic(t *testing.T) { func TestHeader_AddFullBufferNoPanic(t *testing.T) { buf := make([]byte, 0, 40) // Small cap; enough for Reset (len 0) but not the field below. var h Header - h.Reset(buf) - h.EnableBufferGrowth(false) + h.Reset(buf, numHeaderCapacity) + h.ConfigBufferGrowth(false) h.SetMethod("GET") - h.SetRequestURI("/") + h.SetRequestTarget("/") h.SetProtocol("HTTP/1.1") defer func() { @@ -374,7 +557,7 @@ func TestHeader_SetInt(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { var h Header - h.Reset(nil) + h.Reset(nil, numHeaderCapacity) h.SetInt("Content-Length", tc.value, tc.base) if got := string(h.Get("Content-Length")); got != tc.want { t.Fatalf("want %q, got %q", tc.want, got) @@ -386,9 +569,9 @@ func TestHeader_SetInt(t *testing.T) { // SetInt on an existing key must reuse the slot in place (single field, latest value). func TestHeader_SetIntOverwrite(t *testing.T) { var h Header - h.Reset(nil) + h.Reset(nil, numHeaderCapacity) h.SetMethod("GET") - h.SetRequestURI("/") + h.SetRequestTarget("/") h.SetProtocol("HTTP/1.1") h.SetInt("Content-Length", 100, 10) @@ -410,8 +593,8 @@ func TestHeader_SetIntOverwrite(t *testing.T) { func TestHeader_SetIntNoAlloc(t *testing.T) { buf := make([]byte, 0, 256) var h Header - h.Reset(buf) - h.EnableBufferGrowth(false) + h.Reset(buf, numHeaderCapacity) + h.ConfigBufferGrowth(false) h.Add("Content-Length", "0000000000000000000000") // pre-size a reusable slot. allocs := testing.AllocsPerRun(100, func() { h.SetInt("Content-Length", 1234567890, 10) @@ -423,3 +606,98 @@ func TestHeader_SetIntNoAlloc(t *testing.T) { t.Fatalf("want %q, got %q", "1234567890", got) } } + +// A browser sends upwards of twenty header fields and an API client with a few +// custom fields is not far behind. The field table must be sized from the +// buffer the caller handed over, not fixed at a count that a real request +// exceeds. +func TestHeader_FieldTableSizedFromBuffer(t *testing.T) { + const wantVal = "the-canary-value" + var raw strings.Builder + raw.WriteString("GET / HTTP/1.1\r\nHost: lneto.test\r\n") + for i := range 40 { + raw.WriteString("X-Field-") + raw.WriteString(strconv.Itoa(i)) + raw.WriteString(": value-of-a-realistic-length-here\r\n") + } + raw.WriteString("X-Canary: " + wantVal + "\r\n\r\n") + + var h Header + h.Reset(make([]byte, 0, 8192), numHeaderCapacity) // Room for the block with plenty to spare. + err := h.ParseBytes(false, []byte(raw.String())) + if err != nil { + t.Fatalf("parsing a 42 field request into an 8kB buffer: %s", err) + } + if got := string(h.Get("X-Canary")); got != wantVal { + t.Fatalf("want X-Canary %q, got %q", wantVal, got) + } +} + +// A buffer too small for the fields it is handed must be refused with an error +// the caller can act on, so a server answers 431 instead of dropping the peer. +func TestHeader_FieldTableFullIsReported(t *testing.T) { + var raw strings.Builder + raw.WriteString("GET / HTTP/1.1\r\n") + for i := range 64 { + // As short as a field gets. + raw.WriteString("H") + raw.WriteString(strconv.Itoa(i)) + raw.WriteString(":v\r\n") + } + raw.WriteString("\r\n") + var h Header + h.Reset(make([]byte, 0, 512), numHeaderCapacity) + h.ConfigBufferGrowth(false) + err := h.ParseBytes(false, []byte(raw.String())) + if !errors.Is(err, ErrHeaderTooMany) { + t.Fatalf("want ErrHeaderFieldsTooLarge, got %v", err) + } +} + +// EqualDecodedPercentURL must agree with CopyDecodedPercentURL on every input: +// same decoded bytes, and false wherever the copying decoder reports an error. +func TestEqualDecodedPercentURL(t *testing.T) { + for _, value := range []string{ + "", "plain", "a+b", "a%20b", "%41%42", "100%25", "a%2Fb", "+", "%2b", + "trailing%", "trailing%4", "%zz", "a%2", "%%", "a+b%20c", "%00", + } { + for _, plusAsSpace := range []bool{false, true} { + dst := make([]byte, len(value)) + n, err := CopyDecodedPercentURL(dst, []byte(value), plusAsSpace) + // The copying decoder is the reference: whatever it produces is what + // an equal comparison must accept, and only that. + want := "" + if err == nil { + want = string(dst[:n]) + } + got := EqualDecodedPercentURL([]byte(value), want, plusAsSpace) + if err != nil { + if got { + t.Errorf("%q plus=%v: malformed escape must not compare equal", value, plusAsSpace) + } + continue + } + if !got { + t.Errorf("%q plus=%v: want equal to its own decoding %q", value, plusAsSpace, want) + } + if EqualDecodedPercentURL([]byte(value), want+"x", plusAsSpace) { + t.Errorf("%q plus=%v: must not equal a longer want", value, plusAsSpace) + } + if want != "" && EqualDecodedPercentURL([]byte(value), want[:len(want)-1], plusAsSpace) { + t.Errorf("%q plus=%v: must not equal a shorter want", value, plusAsSpace) + } + } + } +} + +// The comparison must not allocate: it is the reason a query lookup can return +// a view without scratch space. +func TestEqualDecodedPercentURLNoAlloc(t *testing.T) { + value := []byte("a%20long%2Dish+key") + allocs := testing.AllocsPerRun(100, func() { + EqualDecodedPercentURL(value, "a long-ish key", true) + }) + if allocs != 0 { + t.Fatalf("EqualDecodedPercentURL allocated %v times, want 0", allocs) + } +} diff --git a/http/httpraw/kvbuffer.go b/http/httpraw/kvbuffer.go new file mode 100644 index 0000000..b709c77 --- /dev/null +++ b/http/httpraw/kvbuffer.go @@ -0,0 +1,499 @@ +package httpraw + +import ( + "io" + "slices" + "strconv" + "unsafe" + + "github.com/soypat/lneto/internal" +) + +// kvBuffer is a common key-value store engine for Cookie, Form, Header and other HTTP abstractions that need +// a key-value store with underlying buffer memory. +type kvBuffer struct { + buf []byte + kvs []pairKV + flags Flags +} + +func (kvb *kvBuffer) free() int { return cap(kvb.buf) - len(kvb.buf) } + +// BufferRaw returns the underlying buffer, its length being the portion in use. +// Stored pairs alias it, so writing to it mangles them. +func (kvb *kvBuffer) BufferRaw() []byte { return kvb.buf } + +// EnableBufferGrowth allows the buffer to grow past the memory [kvBuffer.Reset] +// was handed. The setting outlives Reset; with growth off callers get [ErrBufferExhausted]. +func (kvb *kvBuffer) EnableBufferGrowth(enableGrowth bool) { + if enableGrowth { + kvb.flags &^= flagNoBufferGrow + } else { + kvb.flags |= flagNoBufferGrow + } +} + +func (kvb *kvBuffer) discardKVs() { kvb.kvs = kvb.kvs[:0] } + +// BufferGrowthEnabled reports whether the buffer may grow, see [kvBuffer.EnableBufferGrowth]. +func (kvb *kvBuffer) BufferGrowthEnabled() bool { return !kvb.flags.HasAny(flagNoBufferGrow) } + +// ReadFromBytes appends buf to the underlying buffer, accumulating data to parse. +// Returns [ErrBufferExhausted] when buf does not fit and growth is disabled. +func (kvb *kvBuffer) ReadFromBytes(buf []byte) error { + if len(buf) == 0 { + return io.ErrNoProgress // Nothing handed over, not a buffer problem. + } else if kvb.flags.HasAny(flagMangledBuffer) { + return errMangledBuffer + } else if len(buf)+cap(kvb.buf) > maxBufLen { + return ErrBufferExhausted + } + free := kvb.free() + if len(buf) > free && !kvb.BufferGrowthEnabled() { + return ErrBufferExhausted + } + kvb.buf = append(kvb.buf, buf...) + return nil +} + +// ReadLimited appends at most limit bytes read from r to the underlying buffer. +// A read returning data alongside [io.EOF] reports a nil error, later ones io.EOF. +func (kvb *kvBuffer) ReadLimited(r io.Reader, limit int) (int, error) { + free := kvb.free() + growthEnabled := kvb.BufferGrowthEnabled() + if !growthEnabled && (free == 0 || free < limit) || len(kvb.buf) >= maxBufLen { + return 0, ErrBufferExhausted + } else if kvb.flags.HasAny(flagMangledBuffer) { + return 0, errMangledBuffer + } else if kvb.flags.HasAny(flagReaderEOF) { + return 0, io.EOF + } else if limit <= 0 { + return 0, io.ErrNoProgress + } + kvb.buf = slices.Grow(kvb.buf, limit) + n, err := r.Read(kvb.buf[len(kvb.buf):min(len(kvb.buf)+limit, maxBufLen)]) + kvb.buf = kvb.buf[:len(kvb.buf)+n] + if err != nil { + if n > 0 && err == io.EOF { + kvb.flags |= flagReaderEOF + err = nil // Nil out EOF to not scare off readers. + } + } + return n, err +} + +// Reset discards all pairs and takes buf as the buffer to parse in place, nil +// reusing the current one. kvCap sizes the pair table. Only the growth setting survives. +func (kvb *kvBuffer) Reset(buf []byte, kvCap int) { + if buf == nil { + kvb.buf = kvb.buf[:0] + } else { + kvb.buf = buf + } + internal.SliceReuse(&kvb.kvs, kvCap) + kvb.flags = kvb.flags & flagNoBufferGrow // Only flag persisted is buffer grow config. +} + +// CopyFrom replaces the receiver's contents with a copy of src, sharing no +// memory with it afterwards. +func (kvb *kvBuffer) CopyFrom(src *kvBuffer) { + kvb.buf = append(kvb.buf[:0], src.buf...) + kvb.kvs = append(kvb.kvs[:0], src.kvs...) +} + +// Get returns the value of the first pair matching key. +// Bytes are compared as stored, so if using a Form call [Form.Decode] first when keys may be encoded. +// Returns nil for an absent key and for a valueless pair alike, so use +// [kvBuffer.Present] to tell the two apart. +func (kvb *kvBuffer) Get(key string) []byte { + i := kvb.getIdx(key) + if i < 0 { + return nil + } + return kvb.AtValue(i) +} + +// GetFold returns the value of the first key that matches ascii-case-insensitive. +func (kvb *kvBuffer) GetFold(key string) []byte { + i := kvb.getFoldIdx(key) + if i < 0 { + return nil + } + return kvb.AtValue(i) +} + +// ForEach iterates over the cookie's key-value pairs as stored until cb returns false. +func (kvb *kvBuffer) ForEach(cb func(key, value []byte) bool) { + nc := len(kvb.kvs) + for i := range nc { + if !kvb.kvs[i].isValid() { + continue + } else if !cb(kvb.At(i)) { + break + } + } +} + +// Has returns true if key is present, with or without a value. +func (kvb *kvBuffer) Present(key string) bool { // TODO: rename to Has. + return kvb.getIdx(key) >= 0 +} + +// Has returns true if key is present, with or without a value. +func (kvb *kvBuffer) HasKeyValue(key, value string) bool { + idx := kvb.getIdx(key) + if idx >= 0 { + return b2s(kvb.musttoken(kvb.kvs[idx].value)) == value + } + return false +} + +// Add appends a pair, keeping any already sharing the key: use [kvBuffer.Set] +// to replace instead. Reports false if the buffer could not hold it. +func (kvb *kvBuffer) Add(key, value string) (enoughSpace bool) { + kvb.appendPair(key, value) + return kvb.getIdx(key) >= 0 +} + +// Set replaces key's value and invalidates every other pair sharing the key, so +// a following [kvBuffer.Get] sees exactly one value. +// +// It rewrites in place when it can: of the pairs it would invalidate it keeps +// the smallest whose key and value regions both still hold the new pair, +// leaving the roomier regions for a later Set. When none fits the pair is +// appended with [kvBuffer.Add] and the invalidated regions are stranded, since +// nothing here compacts the buffer. +func (kvb *kvBuffer) Set(key, value string) (enoughSpace bool) { + reuse := kvb.takeReusableSlot(key, len(key), len(value)) + if reuse < 0 { + return kvb.Add(key, value) + } + kvb.overwriteAt(reuse, key, value) + return true +} + +// SetInt is [kvBuffer.Set]'s integer counterpart. It formats value straight into +// the slot it reuses, so overwriting a pair never allocates. +func (kvb *kvBuffer) SetInt(key string, value int64, base int) (enoughSpace bool) { + reuse := kvb.takeReusableSlot(key, len(key), internal.IntLen(value, base)) + if reuse < 0 { + return kvb.appendPairInt(key, value, base) + } + kvb.flags |= flagMangledBuffer + kv := &kvb.kvs[reuse] + copy(kvb.buf[kv.key.start:], key) + kv.key.len = tokint(len(key)) + // The slot was picked to hold keyLen/valueLen, so AppendInt writes inside + // buf and never grows a new backing array. + v := strconv.AppendInt(kvb.buf[kv.value.start:kv.value.start], value, base) + kv.value.len = tokint(len(v)) + return true +} + +// takeReusableSlot invalidates every pair matching key except the smallest one +// whose key and value regions hold keyLen and valueLen bytes, whose index it +// returns. It returns -1 when no surviving slot fits, meaning the caller must +// append instead. +func (kvb *kvBuffer) takeReusableSlot(key string, keyLen, valueLen int) int { + reuse := -1 + for i := range kvb.kvs { + kv := &kvb.kvs[i] + if !kv.isValid() || b2s(kvb.musttoken(kv.key)) != key { + continue + } + // A valueless pair holds no value region, so reusing one would write the + // value over byte 0. Let it fall through to the caller's append, which + // gives the pair a real region and keeps "ok" distinct from "ok=". + fits := kv.HasValue() && int(kv.key.len) >= keyLen && int(kv.value.len) >= valueLen + if fits && (reuse < 0 || kv.size() < kvb.kvs[reuse].size()) { + if reuse >= 0 { + kvb.kvs[reuse].invalidate() // Superseded by a tighter fit. + } + reuse = i + continue + } + kv.invalidate() + } + return reuse +} + +// overwriteAt writes key and value over the regions pair i already owns. The +// caller must have checked both fit; the bytes freed by a shorter pair are +// stranded, not reclaimed. +func (kvb *kvBuffer) overwriteAt(i int, key, value string) { + kvb.flags |= flagMangledBuffer + kv := &kvb.kvs[i] + copy(kvb.buf[kv.key.start:], key) + kv.key.len = tokint(len(key)) + copy(kvb.buf[kv.value.start:], value) + kv.value.len = tokint(len(value)) +} + +func (kvb *kvBuffer) setInternal(key, value []byte) (enoughSpace bool) { + if !kvb.canAddOneKV() { + return false + } + kvb.flags |= flagKVAppended + kvb.kvs = append(kvb.kvs, pairKV{ + key: kvb.view(key), + value: kvb.view(value), + }) + return true +} + +// Len returns the number of slots stored, counting those [kvBuffer.Set] invalidated. +func (kvb *kvBuffer) Len() int { return len(kvb.kvs) } + +// At returns the i'th pair in wire order. value is nil for a pair holding none, +// which is what tells a form's "ok" from "ok=". +func (kvb *kvBuffer) At(i int) (key, value []byte) { + kv := kvb.kvs[i] + if !kv.HasValue() { + return kvb.musttoken(kv.key), nil + } + return kvb.musttoken(kv.key), kvb.musttoken(kv.value) +} +func (kvb *kvBuffer) setAt(i int, k, v []byte) { + kvb.flags |= flagMangledBuffer + // Route through slice, not bytes2tok: a nil v is a pair with no '=' and must + // stay absent rather than trip the alias check on a nil pointer. + kvb.kvs[i] = pairKV{ + key: kvb.view(k), + value: kvb.view(v), + } +} + +// AtKey is [kvBuffer.At] limited to the i'th key. +func (kvb *kvBuffer) AtKey(i int) (key []byte) { return kvb.musttoken(kvb.kvs[i].key) } + +// AtValue is [kvBuffer.At] limited to the i'th value, nil when the pair holds none. +func (kvb *kvBuffer) AtValue(i int) (key []byte) { + if !kvb.kvs[i].HasValue() { + return nil + } + return kvb.musttoken(kvb.kvs[i].value) +} + +func (kvb *kvBuffer) getIdx(key string) int { + for i, pair := range kvb.kvs { + if pair.isValid() && b2s(kvb.musttoken(pair.key)) == key { + return i + } + } + return -1 +} + +func (kvb *kvBuffer) getFoldIdx(key string) int { + for i, pair := range kvb.kvs { + if pair.isValid() && asciiEqualFold(key, b2s(kvb.AtKey(i))) { + return i + } + } + return -1 +} + +// asciiEqualFold reports whether a and b are equal under ASCII case folding. +// Unlike strings.EqualFold it does not fold non-ASCII runes, so no multi-byte +// rune such as U+212A KELVIN SIGN can alias a header key. +func asciiEqualFold(a, b string) bool { + if len(a) != len(b) { + return false + } + const asciiCapDiff = 'a' - 'A' + for i := 0; i < len(a); i++ { + ca, cb := a[i], b[i] + if ca >= 'A' && ca <= 'Z' { + ca += asciiCapDiff + } + if cb >= 'A' && cb <= 'Z' { + cb += asciiCapDiff + } + if ca != cb { + return false + } + } + return true +} + +// reserve ensures need free bytes are available in the buffer, growing it when +// permitted. It accounts for the byte-0 reservation on an empty buffer (see +// mustAppendSlice). It returns false and sets flagOOMReached when the space +// cannot be guaranteed: a tokint offset overflow, or a full buffer with +// flagNoBufferGrow set. +func (kvb *kvBuffer) reserve(need int) (enoughSpace bool) { + if len(kvb.buf) == 0 { + need++ // mustAppend* reserves byte 0 on an empty buffer. + } + if len(kvb.buf)+need > maxBufLen { + kvb.flags |= flagOOMReached // Offsets would overflow uint16 tokint. + return false + } + if need > kvb.free() { + if kvb.flags.HasAny(flagNoBufferGrow) { + kvb.flags |= flagOOMReached + return false + } + kvb.buf = slices.Grow(kvb.buf, need) + } + return true +} + +func (kvb *kvBuffer) appendPair(key, value string) bool { + if !kvb.canAddOneKV() || !kvb.reserve(len(key)+len(value)) { + return false + } + kvb.flags |= flagKVAppended + kvb.kvs = append(kvb.kvs, pairKV{ + key: kvb.mustAppendSlice(key), + value: kvb.mustAppendSlice(value), + }) + return true +} + +func (kvb *kvBuffer) appendPairInt(key string, value int64, base int) bool { + vlen := internal.IntLen(value, base) + if !kvb.canAddOneKV() || !kvb.reserve(len(key)+vlen) { + return false + } + kvb.flags |= flagKVAppended + kvb.kvs = append(kvb.kvs, pairKV{ + key: kvb.mustAppendSlice(key), + value: kvb.mustAppendInt(value, base), + }) + return true +} + +func (kvb *kvBuffer) canAddOneKV() (enoughSpace bool) { + return len(kvb.kvs) < cap(kvb.kvs) || kvb.flags&flagNoBufferGrow == 0 +} + +func (kvb *kvBuffer) mustAppendSlice(value string) view { + L := len(kvb.buf) + if L == 0 { + L++ // Valid key-values start after 0. + } + copy(kvb.buf[L:L+len(value)], value) + kvb.buf = kvb.buf[:L+len(value)] + return kvb.view(kvb.buf[L : L+len(value)]) +} + +func (kvb *kvBuffer) mustAppendInt(value int64, base int) view { + L := len(kvb.buf) + if L == 0 { + L++ // Valid key-values start after byte 0. + } + v := strconv.AppendInt(kvb.buf[L:L], value, base) + kvb.buf = kvb.buf[:L+len(v)] + return kvb.view(kvb.buf[L : L+len(v)]) +} + +// reuseOrAppend writes value over tok's slot when it fits there, avoiding any +// buffer growth; otherwise it appends a fresh slot. +func (kvb *kvBuffer) reuseOrAppend(tok view, value string) view { + if tok.len > tokint(len(value)) { + copy(kvb.musttoken(tok), value) + tok.len = tokint(len(value)) + return tok + } + return kvb.appendSlice(value) +} + +// appendSlice reserves space (growing or flagging OOM) and appends value as a +// new slot. +func (kvb *kvBuffer) appendSlice(value string) view { + debuglog("http:appendslice:start") + if !kvb.reserve(len(value)) { + return view{} // Drop and flag OOM; never panic. + } + kvb.flags |= flagMangledBuffer + return kvb.mustAppendSlice(value) +} + +// reuseOrAppendInt is [kvBuffer.reuseOrAppend]'s integer counterpart. +func (kvb *kvBuffer) reuseOrAppendInt(tok view, value int64, base int) view { + n := internal.IntLen(value, base) + if int(tok.len) >= n { + // Reuse: format directly over the existing slot. No free space needed + // since n <= tok.len and the slot already lives inside buf. + v := strconv.AppendInt(kvb.buf[tok.start:tok.start], value, base) + tok.len = tokint(len(v)) + kvb.flags |= flagMangledBuffer + return tok + } + return kvb.appendInt(value, base, n) +} + +// appendInt reserves space (growing or flagging OOM) and appends value as a new slot. +func (kvb *kvBuffer) appendInt(value int64, base, n int) view { + if !kvb.reserve(n) { + return view{} // Drop and flag OOM; never panic. + } + kvb.flags |= flagMangledBuffer + return kvb.mustAppendInt(value, base) +} + +func (kvb *kvBuffer) view(value []byte) view { + if value == nil { + return view{} + } + return bytes2tok(kvb.buf, value) +} + +func (kvb kvBuffer) musttoken(slice view) []byte { + return tok2bytes(kvb.buf, slice) +} +func (kvb *kvBuffer) noKV() pairKV { return pairKV{} } + +type tokint = uint16 + +// view is a smaller `string`-like representation of a section in [kvBuffer]'s buffer. +type view struct { + start tokint + len tokint +} + +type pairKV struct { + key view + value view // value start >0 means value is present. +} + +// isValid is for stores parsed in place, where offset 0 is the first key so +// only length can signal presence. Empty keys are valid: see valueless cookies. +func (pair pairKV) isValid() bool { + return pair.key.len > 0 || pair.value.len > 0 +} + +// isValidHeader is for the append-built [Header] store, where mustAppendSlice +// burns byte 0 so a zero offset means absent. Drops offset-0 pairs otherwise. +func (pair pairKV) isValidHeader() bool { return pair.key.start > 0 } + +func (pair *pairKV) invalidate() { + *pair = pairKV{} +} + +// size is the buffer a pair occupies, used to pick the tightest slot to reuse. +func (pair pairKV) size() int { return int(pair.key.len) + int(pair.value.len) } + +func (pair pairKV) HasValue() bool { return pair.value.start > 0 } + +// b2s converts byte slice to a string without memory allocation. +// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . +func b2s(b []byte) string { + return unsafe.String(unsafe.SliceData(b), len(b)) +} + +func tok2bytes(buf []byte, slice view) []byte { + return buf[slice.start : slice.start+slice.len] +} + +func bytes2tok(buf, value []byte) view { + base := uintptr(unsafe.Pointer(unsafe.SliceData(buf))) + off := uintptr(unsafe.Pointer(unsafe.SliceData(value))) + if off < base || off > base+uintptr(len(buf)) { + panic("httpx: argument buffer does not alias header buffer") + } + return view{ + start: tokint(off - base), + len: tokint(len(value)), + } +} diff --git a/http/httpraw/multipart.go b/http/httpraw/multipart.go new file mode 100644 index 0000000..e24672f --- /dev/null +++ b/http/httpraw/multipart.go @@ -0,0 +1,272 @@ +package httpraw + +import ( + "bytes" + "io" +) + +// Multipart splits a "multipart/form-data" body into its parts. Such bodies +// frame their fields with a delimiter instead of escaping them, so a part's +// value has no length: it ends where the next delimiter begins. +// +// Multipart stores none of the body, leaving the caller to decide what to keep, +// what to skip and when a part has grown too large. Both methods report how much +// of buf they consumed, which the caller compacts away before reading more in: +// +// var m httpraw.Multipart +// m.SetContentType(contentType) +// var hdr httpraw.MultipartHeader +// for { +// parsed, err := m.NextHeader(&hdr, buf[:buflen]) +// if err != nil { +// break // io.EOF at the closing delimiter, body done. +// } else if parsed == 0 { +// // Header block incomplete: read more into buf[buflen:] and retry. +// continue +// } +// buflen = copy(buf, buf[parsed:buflen]) +// for { +// bodyLen, restOff, done := m.NextBody(buf[:buflen]) +// // Consume buf[:bodyLen] for hdr.Name, then compact and read more. +// buflen = copy(buf, buf[restOff:buflen]) +// if done { +// break +// } +// } +// } +type Multipart struct { + // Boundary is the delimiter parameter of the body's Content-Type field, + // without the leading "--" the delimiter carries on the wire. + Boundary []byte +} + +// MultipartHeader is a part's header block and the Content-Disposition +// parameters that identify it. +type MultipartHeader struct { + // PartView is the part's raw header block, ending in its final CRLF. It + // aliases the buffer it was parsed from, so it is only valid until that + // buffer is compacted or read into again. + PartView []byte + // Name is the name parameter of a part's Content-Disposition field, + // i.e: "photo" for `form-data; name="photo"; filename="beach.png"`. + // Copied out of the buffer, so it outlives it, and reused between parts. + Name []byte + // Filename is the filename parameter of a part's Content-Disposition + // field, empty when the part is not a file upload. Copied like Name. + Filename []byte +} + +// Reset clears the header for the next part, keeping the buffers Name and +// Filename were copied into so a reused header stops allocating. +func (hdr *MultipartHeader) Reset() { + hdr.PartView = nil + hdr.Name = hdr.Name[:0] + hdr.Filename = hdr.Filename[:0] +} + +// SetContentType sets [Multipart.Boundary] from the boundary parameter of a +// Content-Type field value, i.e: "abc123" for +// "multipart/form-data; boundary=abc123". The leading "--" the delimiter carries +// on the wire is not included. Fails when the parameter is absent or is not +// 1 to 70 characters long, RFC 2046 5.1.1; a zero length boundary would match +// every "--" in the body. +func (m *Multipart) SetContentType(contentType []byte) error { + m.Boundary = ContentParam(contentType, "boundary") + if len(m.Boundary) == 0 || len(m.Boundary) > 70 { + return errNoBoundary // RFC 2046 5.1.1: 1..70 characters, required. + } + return nil +} + +// NextHeader parses the leading part's header block off a multipart body into +// dst, returning how many bytes of data it consumed: the part's content begins +// at data[parsedLen]. A zero parsedLen and no error means data holds no complete +// delimiter and header block yet, so the caller reads more in and retries. +// Returns [io.EOF] once the closing delimiter is reached. dst is reset on error. +func (m *Multipart) NextHeader(dst *MultipartHeader, data []byte) (parsedLen int, err error) { + dst.Reset() + if len(m.Boundary) == 0 { + return 0, errNoBoundary + } + idx := m.indexDelimiter(data) + if idx < 0 { + return 0, nil + } + after := idx + len("--") + len(m.Boundary) + if after+2 > len(data) { + return 0, nil // Cannot tell a closing delimiter yet. + } else if data[after] == '-' && data[after+1] == '-' { + return 0, io.EOF + } + // Delimiter is followed by CRLF, then the part's header block. + if data[after] == '\r' { + after++ + } + if after >= len(data) { + return 0, nil + } else if data[after] != '\n' { + return 0, errBadDelimiter + } + after++ + end := bytes.Index(data[after:], []byte("\r\n\r\n")) + if end < 0 { + return 0, nil + } + dst.PartView = data[after : after+end+2] + disposition := partField(dst.PartView) + dst.Name = append(dst.Name[:0], ContentParam(disposition, "name")...) + dst.Filename = append(dst.Filename[:0], ContentParam(disposition, "filename")...) + return after + end + 4, nil +} + +// NextBody reports how much of data is part content, data[:bodyLen], and where +// what is left begins, data[restOff:], which the caller compacts to the front of +// its buffer before reading more in. done reports the part ended, in which case +// data[restOff:] begins the next part's delimiter; otherwise the bytes past +// bodyLen are a tail held back because it could still turn into a delimiter. +func (m *Multipart) NextBody(data []byte) (bodyLen, restOff int, done bool) { + idx := m.indexPartEnd(data) + if idx >= 0 { + return idx, idx + len("\r\n"), true + } + // Longest prefix of "\r\n--"+boundary that could still be completed. + hold := min(len("\r\n--")+len(m.Boundary)-1, len(data)) + return len(data) - hold, len(data) - hold, false +} + +// indexDelimiter returns the offset of the leading "--"+Boundary in data. +func (m *Multipart) indexDelimiter(data []byte) int { + for i := 0; i+len("--")+len(m.Boundary) <= len(data); i++ { + dash := bytes.IndexByte(data[i:], '-') + if dash < 0 { + return -1 + } + i += dash + if i+len("--")+len(m.Boundary) > len(data) { + return -1 + } + if data[i+1] == '-' && b2s(data[i+2:i+2+len(m.Boundary)]) == b2s(m.Boundary) { + return i + } + } + return -1 +} + +// indexPartEnd returns the offset of the CRLF that closes a part, that is the +// CRLF preceding the next delimiter. +func (m *Multipart) indexPartEnd(data []byte) int { + for i := 0; i+len("\r\n--")+len(m.Boundary) <= len(data); i++ { + cr := bytes.IndexByte(data[i:], '\r') + if cr < 0 { + return -1 + } + i += cr + if i+len("\r\n--")+len(m.Boundary) > len(data) { + return -1 + } + if data[i+1] == '\n' && data[i+2] == '-' && data[i+3] == '-' && + b2s(data[i+4:i+4+len(m.Boundary)]) == b2s(m.Boundary) { + return i + } + } + return -1 +} + +// MediaTypeIs reports whether a Content-Type field value carries the given +// media type, ignoring case and any parameters that follow it, i.e: true for +// "text/plain; charset=utf-8" and media type "text/plain". mediaType must be +// ASCII lowercase. RFC 9110 8.3.1. +func MediaTypeIs(value []byte, mediaType string) bool { + if semi := bytes.IndexByte(value, ';'); semi >= 0 { + value = value[:semi] + } + return equalFold(trimOWS(value), mediaType) +} + +// ContentParam returns the value of a parameter of a header field value, i.e: +// "utf-8" for key "charset" of "text/plain; charset=utf-8". Quoted values are +// returned without their quotes and with escapes left as they appear on the +// wire. Key matching is case insensitive, RFC 9110 5.6.6. +func ContentParam(value []byte, key string) []byte { + for len(value) > 0 { + semi := bytes.IndexByte(value, ';') + if semi < 0 { + return nil // No parameters left. + } + value = trimOWS(value[semi+1:]) + eq := bytes.IndexByte(value, '=') + if eq < 0 { + return nil + } + gotKey := trimOWS(value[:eq]) + value = value[eq+1:] + param := value + if len(param) > 0 && param[0] == '"' { + end := bytes.IndexByte(param[1:], '"') + if end < 0 { + return nil // Unterminated quoted string. + } + param, value = param[1:end+1], param[end+2:] + } else { + end := bytes.IndexByte(param, ';') + if end >= 0 { + param, value = param[:end], param[end:] + } else { + value = nil + } + param = trimOWS(param) + } + if equalFold(gotKey, key) { + return param + } + } + return nil +} + +// partField returns the Content-Disposition field value of a part header block. +func partField(partHdr []byte) []byte { + const key = "content-disposition" + for len(partHdr) > 0 { + eol := bytes.IndexByte(partHdr, '\n') + line := partHdr + if eol >= 0 { + line, partHdr = partHdr[:eol], partHdr[eol+1:] + } else { + partHdr = nil + } + colon := bytes.IndexByte(line, ':') + if colon > 0 && equalFold(line[:colon], key) { + return line[colon+1:] + } + } + return nil +} + +// trimOWS trims optional whitespace off both ends of b, RFC 9110 5.6.3. +func trimOWS(b []byte) []byte { + for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') { + b = b[1:] + } + for len(b) > 0 && (b[len(b)-1] == ' ' || b[len(b)-1] == '\t') { + b = b[:len(b)-1] + } + return b +} + +// equalFold compares b to the ASCII lowercase key, case insensitively. +func equalFold(b []byte, key string) bool { + if len(b) != len(key) { + return false + } + const asciiCapDiff = 'a' - 'A' + for i := range b { + c := b[i] + if c >= 'A' && c <= 'Z' { + c += asciiCapDiff + } + if c != key[i] { + return false + } + } + return true +} diff --git a/http/httpraw/multipart_test.go b/http/httpraw/multipart_test.go new file mode 100644 index 0000000..971e522 --- /dev/null +++ b/http/httpraw/multipart_test.go @@ -0,0 +1,334 @@ +package httpraw + +import ( + "io" + "strconv" + "strings" + "testing" +) + +// A two part body: a text field and a PNG upload whose bytes contain CRLFs and +// even the boundary text, which must not desync the parser. +const ( + multiBoundary = "----abc123" + multiBody = "------abc123\r\n" + + "Content-Disposition: form-data; name=\"caption\"\r\n" + + "\r\n" + + "hi there\r\n" + + "------abc123\r\n" + + "Content-Disposition: form-data; name=\"photo\"; filename=\"beach.png\"\r\n" + + "Content-Type: image/png\r\n" + + "\r\n" + + "\x89PNG\r\n--not-the-boundary\r\n\x00\xff\r\n" + + "------abc123--\r\n" +) + +func TestMultipartBoundary(t *testing.T) { + var mp Multipart + for _, test := range []struct { + contentType string + want string + wantErr bool + }{ + {contentType: "multipart/form-data; boundary=abc123", want: "abc123"}, + {contentType: "multipart/form-data; boundary=\"a b\"", want: "a b"}, + {contentType: "multipart/form-data; charset=utf-8; boundary=xyz", want: "xyz"}, + {contentType: "multipart/form-data; BOUNDARY=xyz", want: "xyz"}, // Keys are case insensitive. + {contentType: "multipart/form-data", wantErr: true}, // Absent, RFC 2046 5.1.1 requires it. + {contentType: "application/x-www-form-urlencoded", wantErr: true}, // Not multipart at all. + {contentType: "multipart/form-data; boundary=", wantErr: true}, // Empty matches every "--". + } { + err := mp.SetContentType([]byte(test.contentType)) + if test.wantErr { + if err == nil { + t.Errorf("%q: want error, got boundary %q", test.contentType, mp.Boundary) + } + continue + } else if err != nil { + t.Errorf("%q: %s", test.contentType, err) + continue + } + got := string(mp.Boundary) + if got != test.want { + t.Errorf("%q: want %q, got %q", test.contentType, test.want, got) + } + } +} + +func TestMediaTypeIs(t *testing.T) { + for _, test := range []struct { + value string + media string + want bool + }{ + {value: "text/plain", media: "text/plain", want: true}, + {value: "text/plain; charset=utf-8", media: "text/plain", want: true}, + {value: "text/plain;charset=utf-8", media: "text/plain", want: true}, + {value: "Text/Plain", media: "text/plain", want: true}, // Case insensitive, RFC 9110 8.3.1. + {value: " text/plain ; x=1", media: "text/plain", want: true}, + {value: "text/plain", media: "text/html"}, + {value: "text/plainish", media: "text/plain"}, // Prefix must not match. + {value: "", media: "text/plain"}, + {value: "multipart/form-data; boundary=abc", media: "multipart/form-data", want: true}, + } { + if got := MediaTypeIs([]byte(test.value), test.media); got != test.want { + t.Errorf("%q is %q: want %v, got %v", test.value, test.media, test.want, got) + } + } +} + +func TestContentParam(t *testing.T) { + for _, test := range []struct { + value string + key string + want string + }{ + {value: "text/plain; charset=utf-8", key: "charset", want: "utf-8"}, + {value: "text/plain;charset=utf-8", key: "charset", want: "utf-8"}, // No space. + {value: "text/plain; charset=\"utf-8\"", key: "charset", want: "utf-8"}, + {value: "form-data; name=\"photo\"; filename=\"a;b.png\"", key: "filename", want: "a;b.png"}, + {value: "form-data; name=\"photo\"", key: "nope", want: ""}, + {value: "form-data; names=x; name=y", key: "name", want: "y"}, // Prefix must not match. + {value: "text/plain", key: "charset", want: ""}, + } { + got := ContentParam([]byte(test.value), test.key) + if string(got) != test.want { + t.Errorf("%q key %q: want %q, got %q", test.value, test.key, test.want, got) + } + } +} + +func TestNextPartHeader(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + var hdr MultipartHeader + parsed, err := m.NextHeader(&hdr, []byte(multiBody)) + if err != nil { + t.Fatal(err) + } + const wantHdr = "Content-Disposition: form-data; name=\"caption\"\r\n" + if string(hdr.PartView) != wantHdr { + t.Errorf("want header %q, got %q", wantHdr, hdr.PartView) + } + if string(hdr.Name) != "caption" { + t.Errorf("want name %q, got %q", "caption", hdr.Name) + } + if len(hdr.Filename) != 0 { + t.Errorf("want no filename for a non file part, got %q", hdr.Filename) + } + if !strings.HasPrefix(multiBody[parsed:], "hi there\r\n") { + t.Errorf("want rest at part body, got %q", multiBody[parsed:]) + } +} + +// Names and filenames must outlive the buffer they were parsed from, so a +// caller may compact it and read more without losing the part it is reading. +func TestNextPartHeaderOutlivesBuffer(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + data := []byte(multiBody) + var hdr MultipartHeader + if _, err := m.NextHeader(&hdr, data); err != nil { + t.Fatal(err) + } + for i := range data { + data[i] = 'x' // Buffer reused for the next read. + } + if string(hdr.Name) != "caption" { + t.Errorf("want name %q to survive the buffer, got %q", "caption", hdr.Name) + } +} + +// Incomplete data must ask for more, never guess. +func TestNextPartHeaderNeedMore(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + for _, data := range []string{ + "", + "------abc", // Delimiter cut short. + "------abc123\r\n", // No header block yet. + "------abc123\r\nContent-Disposition: form-", // Header block unterminated. + } { + var hdr MultipartHeader + parsed, err := m.NextHeader(&hdr, []byte(data)) + if parsed != 0 || err != nil { + t.Errorf("%q: want (0, nil) asking for more data, got (%d, %v)", data, parsed, err) + } + } +} + +// Junk between the delimiter and the part header is a multipart framing error, +// not a header field name error. +func TestNextPartHeaderJunk(t *testing.T) { + m := Multipart{Boundary: []byte("abc")} + var hdr MultipartHeader + if _, err := m.NextHeader(&hdr, []byte("--abcX\r\nA: b\r\n\r\n")); err != errBadDelimiter { + t.Errorf("want errBadDelimiter, got %v", err) + } +} + +// The closing delimiter ends iteration. +func TestNextPartHeaderEnd(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + var hdr MultipartHeader + if _, err := m.NextHeader(&hdr, []byte("------abc123--\r\n")); err != io.EOF { + t.Errorf("want io.EOF, got %v", err) + } +} + +func TestNextPartBody(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + var hdr MultipartHeader + parsed, err := m.NextHeader(&hdr, []byte(multiBody)) + if err != nil { + t.Fatal(err) + } + rest := multiBody[parsed:] + bodyLen, restOff, done := m.NextBody([]byte(rest)) + if !done { + t.Fatal("want the part to end within the buffer") + } + if rest[:bodyLen] != "hi there" { + t.Errorf("want body %q, got %q", "hi there", rest[:bodyLen]) + } + if !strings.HasPrefix(rest[restOff:], "------abc123\r\n") { + t.Errorf("want rest at next delimiter, got %q", rest[restOff:]) + } +} + +// A part whose bytes contain CRLFs and boundary-like text must survive intact. +func TestNextPartBodyBinary(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + rest := []byte(multiBody) + var hdr MultipartHeader + parsed, err := m.NextHeader(&hdr, rest) // caption part. + if err != nil { + t.Fatal(err) + } + _, restOff, _ := m.NextBody(rest[parsed:]) + rest = rest[parsed+restOff:] + parsed, err = m.NextHeader(&hdr, rest) // photo part. + if err != nil { + t.Fatal(err) + } + rest = rest[parsed:] + bodyLen, restOff, done := m.NextBody(rest) + if !done { + t.Fatal("want the part to end within the buffer") + } + const want = "\x89PNG\r\n--not-the-boundary\r\n\x00\xff" + if string(rest[:bodyLen]) != want { + t.Errorf("want body %q, got %q", want, rest[:bodyLen]) + } + if _, err = m.NextHeader(&hdr, rest[restOff:]); err != io.EOF { + t.Errorf("want io.EOF after last part, got %v", err) + } +} + +// A delimiter split across two reads must not be mistaken for part data: the +// tail is held back until proven not to be a delimiter. +func TestNextPartBodySplitDelimiter(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + const part = "hi there" + full := part + "\r\n------abc123\r\n" + for split := 1; split < len(full); split++ { + data := full[:split] + bodyLen, restOff, done := m.NextBody([]byte(data)) + if done { + continue // Whole delimiter already present, nothing to prove. + } + if bodyLen > len(part) { + t.Fatalf("split %d: emitted %q, past the end of the part", split, data[:bodyLen]) + } + if data[:bodyLen]+data[restOff:] != data { + t.Fatalf("split %d: body+rest %q%q does not reconstruct input", split, data[:bodyLen], data[restOff:]) + } + } +} + +// A file part carries both parameters, and the raw block stays available. +func TestNextHeaderFilePart(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + var hdr MultipartHeader + parsed, err := m.NextHeader(&hdr, []byte(multiBody)) // caption part. + if err != nil { + t.Fatal(err) + } + rest := []byte(multiBody[parsed:]) + _, restOff, _ := m.NextBody(rest) + if _, err = m.NextHeader(&hdr, rest[restOff:]); err != nil { // photo part. + t.Fatal(err) + } + if got := string(hdr.Name); got != "photo" { + t.Errorf("want name %q, got %q", "photo", got) + } + if got := string(hdr.Filename); got != "beach.png" { + t.Errorf("want filename %q, got %q", "beach.png", got) + } + if !strings.Contains(string(hdr.PartView), "Content-Type: image/png") { + t.Errorf("want the raw block to hold every field, got %q", hdr.PartView) + } +} + +// A failed call must not leave the previous part's fields behind. +func TestNextHeaderResetsOnError(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + var hdr MultipartHeader + if _, err := m.NextHeader(&hdr, []byte(multiBody)); err != nil { + t.Fatal(err) + } + if _, err := m.NextHeader(&hdr, []byte("------abc123--\r\n")); err != io.EOF { + t.Fatalf("want io.EOF, got %v", err) + } + if hdr.PartView != nil || len(hdr.Name) != 0 || len(hdr.Filename) != 0 { + t.Errorf("want cleared header on error, got %+v", hdr) + } +} + +// A header reused across parts must stop allocating once its name and filename +// buffers are big enough. +func TestNextHeaderReuseNoAlloc(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + data := []byte(multiBody) + var hdr MultipartHeader + allocs := testing.AllocsPerRun(10, func() { + if _, err := m.NextHeader(&hdr, data); err != nil { + t.Fatal(err) + } + }) + if allocs != 0 { + t.Errorf("want a reused header to allocate 0 times, got %v", allocs) + } +} + +// The whole loop, as a caller writes it over a buffer it compacts. +func TestMultipartLoop(t *testing.T) { + m := Multipart{Boundary: []byte(multiBoundary)} + rest := []byte(multiBody) + var got []string + var hdr MultipartHeader + for { + parsed, err := m.NextHeader(&hdr, rest) + if err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } else if parsed == 0 { + t.Fatal("header must complete within the buffer") + } + name := string(hdr.Name) + total := 0 + rest = rest[parsed:] + for { + bodyLen, restOff, done := m.NextBody(rest) + total += bodyLen + rest = rest[restOff:] + if done { + break + } + t.Fatal("part must complete within the buffer") + } + got = append(got, name+":"+strconv.Itoa(total)) + } + want := "caption:8|photo:28" + if strings.Join(got, "|") != want { + t.Errorf("want %q, got %q", want, strings.Join(got, "|")) + } +} diff --git a/http/httpraw/parse.go b/http/httpraw/parse.go index 82f4027..4016fba 100644 --- a/http/httpraw/parse.go +++ b/http/httpraw/parse.go @@ -3,20 +3,28 @@ package httpraw import ( "bytes" "errors" - "slices" - "strconv" - "unsafe" "github.com/soypat/lneto/internal" ) var ( - errNoProto = errors.New("missing protocol, HTTP/0.9 unsupported") - errNeedMore = errors.New("need more data: cannot find trailing lf") - errUnparsed = errors.New("need to finish parsing") - errInvalidName = errors.New("invalid header name") - errSmallBuffer = errors.New("small read buffer. Increase ReadBufferSize") - errOOM = errors.New("httpraw: buffer out of memory") + errNoProto = errors.New("missing protocol, HTTP/0.9 unsupported") + // ErrNeedMoreData signals a parser was handed an incomplete buffer: append + // more data to it and call again. + ErrNeedMoreData = errors.New("need more data: cannot find trailing lf/delimiter") + errNoBoundary = errors.New("httpraw: multipart boundary not set") + errUnparsed = errors.New("need to finish parsing") + errInvalidName = errors.New("invalid header name") + // ErrBufferExhausted signals a buffer with no room left for the data being + // written and no permission to grow, see [KVBuffer.EnableBufferGrowth]. + // Enlarging the buffer handed to Reset is the only fix; a server answers it + // on a request header with 431, RFC 6585 5. + ErrBufferExhausted = errors.New("httpraw: buffer exhausted, increase size") + // ErrHeaderTooMany signals a header block carrying more fields than + // the buffer it is parsed into has room for, see [Header.Reset]. A server + // answers it with 431, RFC 6585 5: no larger buffer is coming, so reading + // the rest of the block would only spend memory on a request already lost. + ErrHeaderTooMany = errors.New("httpraw: more header fields than buffer holds") // Header.Set and Header.Add mangles the buffer. // Call them after retrieving the Body. Do not call them before parsing the header (why would you even do that?). errMangledBuffer = errors.New("httpraw: mangled buffer") @@ -29,6 +37,10 @@ var ( errBadStatusCodeTxt = errors.New("invalid status code or text") errCookiesParsed = errors.New("cookies already parsed, reset before parsing again") errBufferTooLarge = errors.New("httpraw: buffer exceeds max size (offsets are uint16)") + errBadPercentEncode = errors.New("httpraw: invalid percent-encoding in URL") + errBadDelimiter = errors.New("httpraw: junk between multipart delimiter and part") + errNoContentLength = errors.New("httpraw: no Content-Length field") + errBadContentLength = errors.New("httpraw: invalid Content-Length value") ) // maxBufLen bounds the header buffer. Offsets/lengths are stored as uint16 @@ -37,38 +49,21 @@ var ( const maxBufLen = 0xffff type headerBuf struct { + kv kvBuffer // buf[:len] holds entire HTTP header data, which may be normalized by [flags]. buf[off:len] holds data not yet processed during parsing. - buf []byte + // buf []byte // offset into buf for parsing. off int // args contains key-value store. - headers []argsKV + // headers []argsKV } -// reset sets the buffer data and discards all parsed data. -func (h *headerBuf) reset(buf []byte) { - if buf == nil { - buf = h.buf[:0] // Reuse buffer but discard raw data on nil input. - } - if cap(h.headers) == 0 { - h.headers = make([]argsKV, 16) - } - *h = headerBuf{ - buf: buf, - headers: h.headers[:0], - } -} - -type tokint = uint16 - -type headerSlice struct { - start tokint - len tokint -} - -type argsKV struct { - key headerSlice - value headerSlice // value start >0 means value is present. +// reset sets the buffer data and discards all parsed data. The field table is +// grown to match the new buffer's capacity and never shrinks, so a header +// reused across requests settles on its largest buffer and stops allocating. +func (h *headerBuf) reset(buf []byte, numHeaderCapacity int) { + h.kv.Reset(buf, numHeaderCapacity) + h.off = 0 } type scannerState struct { @@ -93,60 +88,56 @@ func (h *Header) parse(asResponse bool) (err error) { return err } debuglog("http:firstline:done") - err = h.parseNextHeaders() + err = h.parseNextHeaders(h.Flags()) debuglog("http:headers:done") return err } func (h *Header) parseFirstLine(asResponse bool) (err error) { - if len(h.hbuf.buf) > maxBufLen { + if len(h.hbuf.kv.buf) > maxBufLen { return errBufferTooLarge // Offsets would overflow uint16 tokint. } + flags := h.Flags() if asResponse { - h.statusCode, h.statusText, h.flags, err = h.hbuf.parseFirstLineResponse(h.flags) + h.statusCode, h.statusText, flags, err = h.hbuf.parseFirstLineResponse(flags) } else { - h.method, h.requestURI, h.proto, h.flags, err = h.hbuf.parseFirstLineRequest(h.flags) + h.method, h.requestTarget, h.proto, flags, err = h.hbuf.parseFirstLineRequest(flags) } + h.hbuf.kv.flags = flags return err } -func (h *Header) parseNextHeaders() error { +func (h *Header) parseNextHeaders(flags Flags) error { var ss scannerState - h.hbuf.parseNextHeaders(&ss) + h.hbuf.parseNextHeaders(&ss, flags) if ss.err != nil { - h.flags |= flagConnClose + h.hbuf.kv.flags |= flagConnClose return ss.err } - h.flags |= flagDoneParsingHeader + h.hbuf.kv.flags |= flagDoneParsingHeader return nil } -func (hb *headerBuf) readFromBytes(b []byte) { - hb.buf = append(hb.buf, b...) -} +func (hb *headerBuf) free() int { return hb.kv.free() } -func (hb *headerBuf) free() int { return cap(hb.buf) - len(hb.buf) } - -func (hb *headerBuf) parseNextHeaders(ss *scannerState) { +func (hb *headerBuf) parseNextHeaders(ss *scannerState, flags Flags) { debuglog("http:nexthdr:loop") - for kv := hb.next(ss); kv.isValid(); kv = hb.next(ss) { - if len(hb.headers) == cap(hb.headers) { - // Refuse to grow the headers slice: caller must pre-allocate - // sufficient capacity via reset or use a larger initial size. - ss.err = errOOM + for kv := hb.next(ss); kv.isValidHeader(); kv = hb.next(ss) { + if !hb.kv.canAddOneKV() { + ss.err = ErrHeaderTooMany return } - hb.headers = append(hb.headers, kv) + hb.kv.kvs = append(hb.kv.kvs, kv) } debuglog("http:nexthdr:done") } func (hb *headerBuf) offBuf() []byte { - return hb.buf[hb.off:] + return hb.kv.buf[hb.off:] } func (hb *headerBuf) skipLeadingCRLF() { - for hb.off < len(hb.buf) && (hb.buf[hb.off] == '\n' || hb.buf[hb.off] == '\r') { + for hb.off < len(hb.kv.buf) && (hb.kv.buf[hb.off] == '\n' || hb.kv.buf[hb.off] == '\r') { hb.off++ } } @@ -156,7 +147,7 @@ func (hb *headerBuf) scanLine() []byte { if len(buf) > 0 && buf[len(buf)-1] == '\r' { buf = buf[:len(buf)-1] // exclude carriage return. } - if hb.off < len(hb.buf) { + if hb.off < len(hb.kv.buf) { hb.off++ // consume newline. } return buf @@ -172,17 +163,17 @@ func (hb *headerBuf) scanUntilByte(c byte) []byte { return buf } -func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto headerSlice, flags flags, err error) { +func (hb *headerBuf) parseFirstLineRequest(initFlags Flags) (method, uri, proto view, flags Flags, err error) { debuglog("http:req:scan") hb.off = 0 // Parsing first line resets offset. hb.skipLeadingCRLF() flags = initFlags if bytes.IndexByte(hb.offBuf(), '\n') < 0 { - return method, uri, proto, flags, errNeedMore // Incomplete line. + return method, uri, proto, flags, ErrNeedMoreData // Incomplete line. } b := hb.scanLine() if len(b) < 5 { - return method, uri, proto, flags, errNeedMore + return method, uri, proto, flags, ErrNeedMoreData } debuglog("http:req:parse") @@ -190,8 +181,8 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto reqURIEnd := bytes.IndexByte(b[methodEnd+1:], ' ') if reqURIEnd > 0 { reqURIEnd += methodEnd + 1 - uri = hb.slice(b[methodEnd+1 : reqURIEnd]) - proto = hb.slice(b[reqURIEnd+1:]) // Skip space before protocol. + uri = hb.kv.view(b[methodEnd+1 : reqURIEnd]) + proto = hb.kv.view(b[reqURIEnd+1:]) // Skip space before protocol. if b2s(b[reqURIEnd+1:]) != strHTTP11 { flags |= flagNoHTTP11 } @@ -200,30 +191,30 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto } else { // No version provided. flags |= flagNoHTTP11 - uri = hb.slice(b[methodEnd+1:]) + uri = hb.kv.view(b[methodEnd+1:]) } - method = hb.slice(b[:methodEnd]) + method = hb.kv.view(b[:methodEnd]) return method, uri, proto, flags, nil } -func (hb *headerBuf) parseFirstLineResponse(initFlags flags) (statusCode, statusText headerSlice, flags flags, err error) { +func (hb *headerBuf) parseFirstLineResponse(initFlags Flags) (statusCode, statusText view, flags Flags, err error) { debuglog("http:resp:scan") hb.off = 0 // Parsing first line resets offset. hb.skipLeadingCRLF() flags = initFlags if bytes.IndexByte(hb.offBuf(), '\n') < 0 { - return statusCode, statusText, flags, errNeedMore // Incomplete line. + return statusCode, statusText, flags, ErrNeedMoreData // Incomplete line. } b := hb.scanLine() if len(b) < 5 { - return statusCode, statusText, flags, errNeedMore + return statusCode, statusText, flags, ErrNeedMoreData } debuglog("http:resp:parse") // Parse protocol (e.g. "HTTP/1.1"), then status code, then status text. protoEnd := bytes.IndexByte(b, ' ') if protoEnd < 0 { - return statusCode, statusText, flags, errNeedMore + return statusCode, statusText, flags, ErrNeedMoreData } if b2s(b[:protoEnd]) != strHTTP11 { flags |= flagNoHTTP11 @@ -244,208 +235,27 @@ func (hb *headerBuf) parseFirstLineResponse(initFlags flags) (statusCode, status return statusCode, statusText, flags, errBadStatusCode } } - statusCode = hb.slice(code) + statusCode = hb.kv.view(code) if codeEnd < len(b) { - statusText = hb.slice(b[codeEnd+1:]) // Skip space before text. + statusText = hb.kv.view(b[codeEnd+1:]) // Skip space before text. } debuglog("http:resp:done") return statusCode, statusText, flags, nil } -func (kv argsKV) isValid() bool { - return kv.key.start > 0 -} - -func (kv *argsKV) invalidate() { - *kv = argsKV{} -} - -func (tb headerBuf) musttoken(slice headerSlice) []byte { - return tok2bytes(tb.buf, slice) - -} - -func (tb headerBuf) slice(b []byte) headerSlice { - return bytes2tok(tb.buf, b) -} - -func (kv argsKV) HasValue() bool { return kv.value.start > 0 } - -func (h *Header) hasHeaderValue(key, value string) bool { - kv := h.peekHeader(key) - return kv.isValid() && b2s(h.hbuf.musttoken(kv.value)) == value -} - -// peekHeader returns header key-value for the given key. -// -// The returned value is valid until the request is released, -// either though ReleaseRequest or your request handler returning. -// Do not store references to returned value. Make copies instead. -func (h *Header) peekHeader(key string) argsKV { - hb := &h.hbuf - for i := 0; i < len(h.hbuf.headers); i++ { - if b2s(hb.musttoken(h.hbuf.headers[i].key)) == key { - return h.hbuf.headers[i] - } - } - return hb.noKV() -} - -func (hb *headerBuf) mustAppendSlice(value string) headerSlice { - L := len(hb.buf) - if L == 0 { - L++ // Valid key-values start after 0. - } - copy(hb.buf[L:L+len(value)], value) - hb.buf = hb.buf[:L+len(value)] - return hb.slice(hb.buf[L : L+len(value)]) -} - -func (h *Header) reuseOrAppend(tok headerSlice, value string) headerSlice { - if tok.len > tokint(len(value)) { - copy(h.hbuf.musttoken(tok), value) - tok.len = tokint(len(value)) - return tok - } - return h.appendSlice(value) -} - -func (h *Header) appendSlice(value string) headerSlice { - debuglog("http:appendslice:start") - if !h.reserve(len(value)) { - return headerSlice{} - } - h.flags |= flagMangledBuffer - return h.hbuf.mustAppendSlice(value) -} - -func (h *Header) appendHeader(key, value string) { - // reserve accounts for the byte-0 reservation mustAppendSlice makes on an - // empty buffer, and drops (flagging OOM) rather than panicking when growth - // is disabled and space runs out. - if !h.reserve(len(key) + len(value)) { - return - } - h.flags |= flagMangledBuffer - hb := &h.hbuf - k := hb.mustAppendSlice(key) - v := hb.mustAppendSlice(value) - debuglog("http:appendhdr:grow-hdrs") - hb.headers = append(hb.headers, argsKV{ - key: k, - value: v, - }) -} - -// appendHeaderInt is appendHeader's integer counterpart: it appends key and the -// formatted integer value as a new header field. -func (h *Header) appendHeaderInt(key string, value int64, base int) { - n := intLen(value, base) - if !h.reserve(len(key) + n) { - return // Drop and flag OOM; never panic. - } - h.flags |= flagMangledBuffer - hb := &h.hbuf - k := hb.mustAppendSlice(key) - v := hb.mustAppendInt(value, base) - hb.headers = append(hb.headers, argsKV{ - key: k, - value: v, - }) -} - -// reserve ensures need free bytes are available in the buffer, growing it when -// permitted. It accounts for the byte-0 reservation on an empty buffer (see -// mustAppendSlice). It returns false and sets flagOOMReached when the space -// cannot be guaranteed: a tokint offset overflow, or a full buffer with -// flagNoBufferGrow set. -func (h *Header) reserve(need int) bool { - hb := &h.hbuf - if len(hb.buf) == 0 { - need++ // mustAppend* reserves byte 0 on an empty buffer. - } - if len(hb.buf)+need > maxBufLen { - h.flags |= flagOOMReached // Offsets would overflow uint16 tokint. - return false - } - if need > hb.free() { - if h.flags.hasAny(flagNoBufferGrow) { - h.flags |= flagOOMReached - return false - } - hb.buf = slices.Grow(hb.buf, need) - } - return true -} - -// reuseOrAppendInt writes value into tok's slot in place when it fits, avoiding -// any buffer growth; otherwise it appends a fresh slot. -func (h *Header) reuseOrAppendInt(tok headerSlice, value int64, base int) headerSlice { - n := intLen(value, base) - if int(tok.len) >= n { - // Reuse: format directly over the existing slot. No free space needed - // since n <= tok.len and the slot already lives inside buf. - v := strconv.AppendInt(h.hbuf.buf[tok.start:tok.start], value, base) - tok.len = tokint(len(v)) - h.flags |= flagMangledBuffer - return tok - } - return h.appendInt(value, base, n) -} - -// appendInt reserves space (growing or flagging OOM) and appends value as a new slot. -func (h *Header) appendInt(value int64, base, n int) headerSlice { - if !h.reserve(n) { - return headerSlice{} // Drop and flag OOM; never panic. - } - h.flags |= flagMangledBuffer - return h.hbuf.mustAppendInt(value, base) -} - -// mustAppendInt formats value into the buffer's free region and commits it. -// The caller must have reserved at least intLen(value, base) free bytes. -func (hb *headerBuf) mustAppendInt(value int64, base int) headerSlice { - L := len(hb.buf) - if L == 0 { - L++ // Valid key-values start after byte 0. - } - v := strconv.AppendInt(hb.buf[L:L], value, base) - hb.buf = hb.buf[:L+len(v)] - return hb.slice(hb.buf[L : L+len(v)]) -} - -// intLen returns the number of bytes strconv.AppendInt would emit for value in -// the given base (including a leading minus sign for negatives). Used to size -// the buffer and to test whether a value fits an existing slot without writing. -func intLen(value int64, base int) int { - n := 1 - u := uint64(value) - if value < 0 { - n++ // Leading minus sign. - u = -u // Two's-complement magnitude; correct even for math.MinInt64. - } - for u >= uint64(base) { - u /= uint64(base) - n++ - } - return n -} - -func (hb *headerBuf) noKV() argsKV { return argsKV{} } - -func (hb *headerBuf) next(ss *scannerState) argsKV { +func (hb *headerBuf) next(ss *scannerState) pairKV { if !ss.initialized { ss.nextColon = -1 ss.nextNewLine = -1 } - buf := hb.buf[hb.off:] + buf := hb.kv.buf[hb.off:] blen := len(buf) if blen >= 2 && buf[0] == '\r' && buf[1] == '\n' { hb.off += 2 - return hb.noKV() // \r\n\r\n Ends header. + return hb.kv.noKV() // \r\n\r\n Ends header. } else if blen >= 1 && buf[0] == '\n' { hb.off += 1 - return hb.noKV() // \n\n Ends header. + return hb.kv.noKV() // \n\n Ends header. } // n is parsing offset. Will start by storing colon index. @@ -460,19 +270,19 @@ func (hb *headerBuf) next(ss *scannerState) argsKV { if x < 0 { // A header name should always at some point be followed by a \n // even if it's the one that terminates the header block. - ss.err = errNeedMore - return hb.noKV() + ss.err = ErrNeedMoreData + return hb.kv.noKV() } else if x < n { // There was a \n before the colon! This is invalid. ss.err = errInvalidName - return hb.noKV() + return hb.kv.noKV() } else if n < 0 { // A newline is present (x>=0 reached here) but the line has no // colon: malformed, not incomplete. A split arriving before the // colon has no newline yet and is caught by the x<0 branch above, - // so it still returns errNeedMore. + // so it still returns ErrNeedMoreData. ss.err = errInvalidName - return hb.noKV() + return hb.kv.noKV() } } // n stores colon position by now. @@ -480,12 +290,12 @@ func (hb *headerBuf) next(ss *scannerState) argsKV { // Spaces between the header key and colon are not allowed. // See RFC 7230, Section 3.2.4. ss.err = errInvalidName - return hb.noKV() + return hb.kv.noKV() } // Ready to store key.. - var resultKV argsKV - resultKV.key = hb.slice(buf[:n]) + var resultKV pairKV + resultKV.key = hb.kv.view(buf[:n]) n++ // consume colon. for len(buf) > n && buf[n] == ' ' { n++ // Trim leading spaces. @@ -498,8 +308,8 @@ func (hb *headerBuf) next(ss *scannerState) argsKV { nl := bytes.IndexByte(buf[n:], '\n') if nl < 0 || nl+n+1 == len(buf) { // No newline or newline is last character and can't know if is multiline. - ss.err = errNeedMore - return hb.noKV() + ss.err = ErrNeedMoreData + return hb.kv.noKV() } n += nl + 1 // Index of the newly found newline. nextChar := buf[n] @@ -512,42 +322,40 @@ func (hb *headerBuf) next(ss *scannerState) argsKV { if valueEnd > valueStart && buf[valueEnd-1] == '\r' { valueEnd-- // Trim \r character if present before value. } - resultKV.value = hb.slice(buf[valueStart:valueEnd]) + resultKV.value = hb.kv.view(buf[valueStart:valueEnd]) hb.off += n return resultKV } // ConnectionClose returns true if 'Connection: close' header is set or if a invalid header was found. func (h *Header) ConnectionClose() bool { - closed := h.flags.hasAny(flagConnClose) || - h.hasHeaderValue(headerConnection, strClose) || - (h.flags.hasAny(flagNoHTTP11) && !h.hasHeaderValue(headerConnection, "keep-alive")) + flags := h.Flags() + closed := flags.HasAny(flagConnClose) || + h.hasConnectionToken(strClose) || + (flags.HasAny(flagNoHTTP11) && !h.hasConnectionToken(strKeepAlive)) if closed { - h.flags |= flagConnClose + h.hbuf.kv.flags |= flagConnClose } return closed } -// b2s converts byte slice to a string without memory allocation. -// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ . -func b2s(b []byte) string { - return unsafe.String(unsafe.SliceData(b), len(b)) -} - -func tok2bytes(buf []byte, slice headerSlice) []byte { - return buf[slice.start : slice.start+slice.len] -} - -func bytes2tok(buf, value []byte) headerSlice { - base := uintptr(unsafe.Pointer(unsafe.SliceData(buf))) - off := uintptr(unsafe.Pointer(unsafe.SliceData(value))) - if off < base || off > base+uintptr(len(buf)) { - panic("httpx: argument buffer does not alias header buffer") - } - return headerSlice{ - start: tokint(off - base), - len: tokint(len(value)), +// hasConnectionToken reports whether the Connection field lists token, which +// must be lowercase. The field name, its comma list and each token all compare +// case insensitively, RFC 9110 5.1 and 7.6.1. +func (h *Header) hasConnectionToken(token string) bool { + value := h.GetFold(headerConnection) + for len(value) > 0 { + item := value + if comma := bytes.IndexByte(value, ','); comma >= 0 { + item, value = value[:comma], value[comma+1:] + } else { + value = nil + } + if equalFold(trimOWS(item), token) { + return true + } } + return false } const enableDebug = internal.HeapAllocDebugging diff --git a/http/httpraw/parse_test.go b/http/httpraw/parse_test.go index c94616e..0a3160e 100644 --- a/http/httpraw/parse_test.go +++ b/http/httpraw/parse_test.go @@ -10,20 +10,17 @@ func TestTryParse_IncrementalRequest(t *testing.T) { // Full HTTP request split across multiple ReadFromBytes calls. full := "GET /index.html HTTP/1.1\r\nHost: example.com\r\nContent-Type: text/html\r\n\r\nbody here" var hdr Header - hdr.Reset(make([]byte, 0, 256)) + hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) // Feed data in small chunks to exercise incremental parsing. chunks := splitInto(full, 10) var done bool var doneIdx int for i, chunk := range chunks { - n, err := hdr.ReadFromBytes([]byte(chunk)) + err := hdr.ReadFromBytes([]byte(chunk)) if err != nil { t.Fatalf("ReadFromBytes: %v", err) } - if n != len(chunk) { - t.Fatalf("expected %d bytes read, got %d", len(chunk), n) - } var needMore bool needMore, err = hdr.TryParse(false) @@ -52,19 +49,16 @@ func TestTryParse_IncrementalRequest(t *testing.T) { if string(hdr.Method()) != "GET" { t.Errorf("method = %q; want GET", hdr.Method()) } - if string(hdr.RequestURI()) != "/index.html" { - t.Errorf("URI = %q; want /index.html", hdr.RequestURI()) + if string(hdr.RequestTarget()) != "/index.html" { + t.Errorf("URI = %q; want /index.html", hdr.RequestTarget()) } // Verify headers via ForEach. headers := make(map[string]string) - err := hdr.ForEach(func(key, value []byte) error { + hdr.ForEach(func(key, value []byte) bool { headers[string(key)] = string(value) - return nil + return true }) - if err != nil { - t.Fatal(err) - } if headers["Host"] != "example.com" { t.Errorf("Host = %q; want example.com", headers["Host"]) } @@ -85,7 +79,7 @@ func TestTryParse_IncrementalRequest(t *testing.T) { func TestTryParse_IncrementalResponse(t *testing.T) { full := "HTTP/1.1 200 OK\r\nContent-Length: 5\r\nServer: lneto\r\n\r\nhello" var hdr Header - hdr.Reset(make([]byte, 0, 256)) + hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) chunks := splitInto(full, 8) var done bool @@ -137,7 +131,7 @@ func TestReadFromLimited(t *testing.T) { r := strings.NewReader(data) var hdr Header - hdr.Reset(make([]byte, 0, 256)) + hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) // Read in one shot. n, err := hdr.ReadFromLimited(r, 256) @@ -163,7 +157,7 @@ func TestReadFromLimited(t *testing.T) { func TestReadFromLimited_MaxBytes(t *testing.T) { var hdr Header - hdr.Reset(make([]byte, 0, 256)) + hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) // Zero maxBytesToRead should error. _, err := hdr.ReadFromLimited(strings.NewReader("data"), 0) @@ -174,9 +168,9 @@ func TestReadFromLimited_MaxBytes(t *testing.T) { func TestReadFromBytes_Empty(t *testing.T) { var hdr Header - hdr.Reset(make([]byte, 0, 256)) + hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) - _, err := hdr.ReadFromBytes(nil) + err := hdr.ReadFromBytes(nil) if err == nil { t.Fatal("expected error for empty bytes") } @@ -184,7 +178,7 @@ func TestReadFromBytes_Empty(t *testing.T) { func TestBufferFreeAndCapacity(t *testing.T) { var hdr Header - hdr.Reset(make([]byte, 0, 100)) + hdr.Reset(make([]byte, 0, 100), numHeaderCapacity) if hdr.BufferCapacity() != 100 { t.Errorf("capacity = %d; want 100", hdr.BufferCapacity()) @@ -202,15 +196,14 @@ func TestBufferFreeAndCapacity(t *testing.T) { func TestEnableBufferGrowth(t *testing.T) { var hdr Header buf := make([]byte, 0, 64) - hdr.Reset(buf) - hdr.EnableBufferGrowth(false) - + hdr.Reset(buf, numHeaderCapacity) + hdr.ConfigBufferGrowth(false) // With growth disabled, reading more than capacity should fail. big := make([]byte, 128) for i := range big { big[i] = 'A' } - _, err := hdr.ReadFromBytes(big) + err := hdr.ReadFromBytes(big) if err == nil { t.Fatal("expected error when buffer growth disabled and data exceeds capacity") } @@ -229,11 +222,11 @@ func TestHeader_Add(t *testing.T) { // ForEach should find both. var values []string - hdr.ForEach(func(key, value []byte) error { + hdr.ForEach(func(key, value []byte) bool { if string(key) == "X-Custom" { values = append(values, string(value)) } - return nil + return true }) if len(values) != 2 { t.Fatalf("expected 2 X-Custom headers, got %d", len(values)) @@ -332,6 +325,14 @@ func TestCookie_ParseBytes(t *testing.T) { if string(c.Get("Path")) != "/" { t.Errorf("Path = %q; want /", c.Get("Path")) } + // The first pair sits at buffer offset 0, which a presence check keyed on + // the offset rather than the length reads as absent. + if string(c.Get("session")) != "abc123" { + t.Errorf("Get(session) = %q; want abc123", c.Get("session")) + } + if !c.HasKeyOrSingleValue("session") { + t.Error("expected first pair to be present by key") + } if !c.HasKeyOrSingleValue("Secure") { t.Error("expected Secure flag") } @@ -357,13 +358,10 @@ func TestCookie_ForEach(t *testing.T) { c.ParseBytes([]byte("a=1; b=2; c=3")) var keys []string - err := c.ForEach(func(key, value []byte) error { + c.ForEach(func(key, value []byte) bool { keys = append(keys, string(key)) - return nil + return true }) - if err != nil { - t.Fatal(err) - } if len(keys) != 3 { t.Fatalf("expected 3 cookie entries, got %d", len(keys)) } @@ -389,7 +387,7 @@ func TestHeader_MultilineValue(t *testing.T) { func TestHeader_ResponseRoundTrip(t *testing.T) { var hdr Header - hdr.Reset(make([]byte, 0, 256)) + hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) hdr.SetProtocol("HTTP/1.1") hdr.SetStatus("404", "Not Found") hdr.Add("Content-Type", "text/plain") @@ -429,10 +427,10 @@ func TestHeader_ResponseRoundTrip(t *testing.T) { func TestHeader_RequestRoundTrip(t *testing.T) { var hdr Header - hdr.Reset(make([]byte, 0, 256)) + hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) hdr.SetProtocol("HTTP/1.1") hdr.SetMethod("POST") - hdr.SetRequestURI("/api/data") + hdr.SetRequestTarget("/api/data") hdr.Add("Host", "example.com") hdr.Add("Content-Type", "application/json") @@ -454,8 +452,8 @@ func TestHeader_RequestRoundTrip(t *testing.T) { if string(hdr2.Method()) != "POST" { t.Errorf("re-parsed method = %q; want POST", hdr2.Method()) } - if string(hdr2.RequestURI()) != "/api/data" { - t.Errorf("re-parsed URI = %q; want /api/data", hdr2.RequestURI()) + if string(hdr2.RequestTarget()) != "/api/data" { + t.Errorf("re-parsed URI = %q; want /api/data", hdr2.RequestTarget()) } if string(hdr2.Get("Host")) != "example.com" { t.Errorf("re-parsed Host = %q; want example.com", hdr2.Get("Host")) @@ -504,8 +502,8 @@ func TestParseRequest_NoProtocol(t *testing.T) { if string(hdr.Method()) != "GET" { t.Errorf("method = %q; want GET", hdr.Method()) } - if string(hdr.RequestURI()) != "/simple" { - t.Errorf("URI = %q; want /simple", hdr.RequestURI()) + if string(hdr.RequestTarget()) != "/simple" { + t.Errorf("URI = %q; want /simple", hdr.RequestTarget()) } if hdr.Protocol() != nil { t.Errorf("protocol should be nil for version-less request, got %q", hdr.Protocol()) @@ -539,3 +537,44 @@ func splitInto(s string, n int) []string { } return chunks } + +// Connection is a case-insensitive list of case-insensitive tokens, RFC 9110 +// 7.6.1, and its field name folds like any other, RFC 9110 5.1. Missing a close +// token keeps serving a peer that asked to hang up; missing keep-alive hangs up +// on an HTTP/1.0 peer that asked to stay. +func TestConnectionCloseFolded(t *testing.T) { + for _, test := range []struct { + proto string + field string + wantClose bool + }{ + {proto: "HTTP/1.1", field: "Connection: close", wantClose: true}, + {proto: "HTTP/1.1", field: "connection: close", wantClose: true}, + {proto: "HTTP/1.1", field: "CONNECTION: close", wantClose: true}, + {proto: "HTTP/1.1", field: "Connection: Close", wantClose: true}, + {proto: "HTTP/1.1", field: "Connection: CLOSE", wantClose: true}, + {proto: "HTTP/1.1", field: "Connection: keep-alive, close", wantClose: true}, + {proto: "HTTP/1.1", field: "Connection: close, keep-alive", wantClose: true}, + {proto: "HTTP/1.1", field: "Connection: TE, Close", wantClose: true}, + // A token that merely contains "close" is not the close token. + {proto: "HTTP/1.1", field: "Connection: closed", wantClose: false}, + {proto: "HTTP/1.1", field: "Connection: keep-alive", wantClose: false}, + // HTTP/1.0 closes unless the peer asks to keep the connection. + {proto: "HTTP/1.0", field: "Connection: keep-alive", wantClose: false}, + {proto: "HTTP/1.0", field: "connection: keep-alive", wantClose: false}, + {proto: "HTTP/1.0", field: "Connection: Keep-Alive", wantClose: false}, + {proto: "HTTP/1.0", field: "Connection: TE, keep-alive", wantClose: false}, + {proto: "HTTP/1.0", field: "Host: h", wantClose: true}, + } { + t.Run(test.proto+" "+test.field, func(t *testing.T) { + var hdr Header + full := "GET / " + test.proto + "\r\nHost: h\r\n" + test.field + "\r\n\r\n" + if err := hdr.ParseBytes(false, []byte(full)); err != nil { + t.Fatal(err) + } + if got := hdr.ConnectionClose(); got != test.wantClose { + t.Errorf("want ConnectionClose=%v, got %v", test.wantClose, got) + } + }) + } +} diff --git a/internal/strconv.go b/internal/strconv.go new file mode 100644 index 0000000..d70e0fb --- /dev/null +++ b/internal/strconv.go @@ -0,0 +1,19 @@ +package internal + +// IntLen returns the number of bytes [strconv.AppendInt] emits for value in the +// given base, including a leading minus sign for negatives. Lets callers size a +// buffer, or test whether a value fits an existing slot, before writing a byte. +// base must be in the range 2..36, as accepted by [strconv.AppendInt]. +func IntLen(value int64, base int) int { + n := 1 + u := uint64(value) + if value < 0 { + n++ // Leading minus sign. + u = -u // Two's-complement magnitude; correct even for math.MinInt64. + } + for u >= uint64(base) { + u /= uint64(base) + n++ + } + return n +} diff --git a/internet/pcap/capture.go b/internet/pcap/capture.go index 74dd8e8..f55abdb 100644 --- a/internet/pcap/capture.go +++ b/internet/pcap/capture.go @@ -768,6 +768,11 @@ func httpBodyClass(contentType, body []byte) FieldClass { return FieldClassText } +// maxCapturedHeaderFields bounds the field table a captured HTTP header is +// parsed into. Captures are read once and discarded, so the table is sized for +// a realistic request rather than for whatever the capture happens to hold. +const maxCapturedHeaderFields = 64 + func (pc *PacketBreakdown) CaptureHTTP(dst []Frame, pkt []byte, bitOffset int) ([]Frame, error) { debuglog("pcap:http:start") const httpProtocol = "HTTP" @@ -777,10 +782,10 @@ func (pc *PacketBreakdown) CaptureHTTP(dst []Frame, pkt []byte, bitOffset int) ( const asResponse = true const asRequest = false httpData := pkt[bitOffset/8:] - pc.hdr.Reset(httpData) + pc.hdr.Reset(httpData, maxCapturedHeaderFields) err := pc.hdr.Parse(asResponse) if err != nil { - pc.hdr.Reset(httpData) + pc.hdr.Reset(httpData, 0) // Field table already sized, reuse it. err = pc.hdr.Parse(asRequest) // try as request. } if err != nil { diff --git a/internet/pcap/stringers.go b/internet/pcap/stringers.go index ef2dae0..0b4da7b 100644 --- a/internet/pcap/stringers.go +++ b/internet/pcap/stringers.go @@ -33,8 +33,9 @@ const _FieldClass_name = "undefinedsourcedestinationprotocolversiontypesizeflags var _FieldClass_index = [...]uint8{0, 9, 15, 26, 34, 41, 45, 49, 54, 68, 76, 83, 90, 94, 101, 112, 114, 123, 131} func (i FieldClass) String() string { - if i >= FieldClass(len(_FieldClass_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_FieldClass_index)-1 { return "FieldClass(" + strconv.FormatInt(int64(i), 10) + ")" } - return _FieldClass_name[_FieldClass_index[i]:_FieldClass_index[i+1]] + return _FieldClass_name[_FieldClass_index[idx]:_FieldClass_index[idx+1]] } diff --git a/ipv4/icmpv4/stringers.go b/ipv4/icmpv4/stringers.go index 5481f09..defc5bd 100644 --- a/ipv4/icmpv4/stringers.go +++ b/ipv4/icmpv4/stringers.go @@ -66,10 +66,11 @@ const _CodeDestinationUnreachable_name = "net unreachablehost unreachableprotoco var _CodeDestinationUnreachable_index = [...]uint8{0, 15, 31, 51, 67, 98, 117} func (i CodeDestinationUnreachable) String() string { - if i >= CodeDestinationUnreachable(len(_CodeDestinationUnreachable_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_CodeDestinationUnreachable_index)-1 { return "CodeDestinationUnreachable(" + strconv.FormatInt(int64(i), 10) + ")" } - return _CodeDestinationUnreachable_name[_CodeDestinationUnreachable_index[i]:_CodeDestinationUnreachable_index[i+1]] + return _CodeDestinationUnreachable_name[_CodeDestinationUnreachable_index[idx]:_CodeDestinationUnreachable_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. @@ -86,8 +87,9 @@ const _CodeRedirect_name = "redirect for networkredirect for hostredirect for To var _CodeRedirect_index = [...]uint8{0, 20, 37, 61, 82} func (i CodeRedirect) String() string { - if i >= CodeRedirect(len(_CodeRedirect_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_CodeRedirect_index)-1 { return "CodeRedirect(" + strconv.FormatInt(int64(i), 10) + ")" } - return _CodeRedirect_name[_CodeRedirect_index[i]:_CodeRedirect_index[i+1]] + return _CodeRedirect_name[_CodeRedirect_index[idx]:_CodeRedirect_index[idx+1]] } diff --git a/ipv6/icmpv6/stringers.go b/ipv6/icmpv6/stringers.go index a184f29..8b3cf93 100644 --- a/ipv6/icmpv6/stringers.go +++ b/ipv6/icmpv6/stringers.go @@ -66,10 +66,11 @@ const _CodeDestinationUnreachable_name = "no route to destinationcommunication a var _CodeDestinationUnreachable_index = [...]uint8{0, 23, 64, 94, 113, 129, 172, 199} func (i CodeDestinationUnreachable) String() string { - if i >= CodeDestinationUnreachable(len(_CodeDestinationUnreachable_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_CodeDestinationUnreachable_index)-1 { return "CodeDestinationUnreachable(" + strconv.FormatInt(int64(i), 10) + ")" } - return _CodeDestinationUnreachable_name[_CodeDestinationUnreachable_index[i]:_CodeDestinationUnreachable_index[i+1]] + return _CodeDestinationUnreachable_name[_CodeDestinationUnreachable_index[idx]:_CodeDestinationUnreachable_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. @@ -85,8 +86,9 @@ const _CodeParameterProblem_name = "erroneous header field encounteredunrecogniz var _CodeParameterProblem_index = [...]uint8{0, 34, 75, 111} func (i CodeParameterProblem) String() string { - if i >= CodeParameterProblem(len(_CodeParameterProblem_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_CodeParameterProblem_index)-1 { return "CodeParameterProblem(" + strconv.FormatInt(int64(i), 10) + ")" } - return _CodeParameterProblem_name[_CodeParameterProblem_index[i]:_CodeParameterProblem_index[i+1]] + return _CodeParameterProblem_name[_CodeParameterProblem_index[idx]:_CodeParameterProblem_index[idx+1]] } diff --git a/phy/phy_stringers.go b/phy/phy_stringers.go index 772c694..8b18474 100644 --- a/phy/phy_stringers.go +++ b/phy/phy_stringers.go @@ -29,8 +29,9 @@ const _LinkMode_name = "down10M-H10M-F100M-H100M-F100M-T41000M-H1000M-F2.5G-F5G- var _LinkMode_index = [...]uint8{0, 4, 9, 14, 20, 26, 33, 40, 47, 53, 57, 62, 67, 72, 78} func (i LinkMode) String() string { - if i >= LinkMode(len(_LinkMode_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_LinkMode_index)-1 { return "LinkMode(" + strconv.FormatInt(int64(i), 10) + ")" } - return _LinkMode_name[_LinkMode_index[i]:_LinkMode_index[i+1]] + return _LinkMode_name[_LinkMode_index[idx]:_LinkMode_index[idx+1]] } diff --git a/stringers.go b/stringers.go index 58e5b74..2b117d8 100644 --- a/stringers.go +++ b/stringers.go @@ -212,16 +212,17 @@ func _() { _ = x[ErrAlreadyRegistered-17] _ = x[ErrTruncatedFrame-18] _ = x[ErrMissingHALConfig-19] + _ = x[ErrBadState-20] } const ( _errGeneric_name_0 = "lneto-bug(use build tag \"debugheaplog\")packet droppedincorrect checksumzero source(port/addr)zero destination(port/addr)short bufferbuffer fullinvalid addressunsupportedmismatchmismatched lengthinvalid configuration" - _errGeneric_name_1 = "invalid fieldinvalid length fieldresource exhaustedprotocol already registeredtruncated framemissing HAL configuration" + _errGeneric_name_1 = "invalid fieldinvalid length fieldresource exhaustedprotocol already registeredtruncated framemissing HAL configurationoperation invalid in current state" ) var ( _errGeneric_index_0 = [...]uint8{0, 39, 53, 71, 93, 120, 132, 143, 158, 169, 177, 194, 215} - _errGeneric_index_1 = [...]uint8{0, 13, 33, 51, 78, 93, 118} + _errGeneric_index_1 = [...]uint8{0, 13, 33, 51, 78, 93, 118, 152} ) func (i errGeneric) String() string { @@ -229,7 +230,7 @@ func (i errGeneric) String() string { case 1 <= i && i <= 12: i -= 1 return _errGeneric_name_0[_errGeneric_index_0[i]:_errGeneric_index_0[i+1]] - case 14 <= i && i <= 19: + case 14 <= i && i <= 20: i -= 14 return _errGeneric_name_1[_errGeneric_index_1[i]:_errGeneric_index_1[i+1]] default: diff --git a/tcp/stringers.go b/tcp/stringers.go index 52c90c7..abc7bb6 100644 --- a/tcp/stringers.go +++ b/tcp/stringers.go @@ -26,10 +26,11 @@ const _State_name = "CLOSEDLISTENSYN-RECEIVEDSYN-SENTESTABLISHEDFIN-WAIT-1FIN-WA var _State_index = [...]uint8{0, 6, 12, 24, 32, 43, 53, 63, 70, 79, 89, 97} func (i State) String() string { - if i >= State(len(_State_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_State_index)-1 { return "State(" + strconv.FormatInt(int64(i), 10) + ")" } - return _State_name[_State_index[i]:_State_index[i+1]] + return _State_name[_State_index[idx]:_State_index[idx+1]] } func _() { // An "invalid array index" compiler error signifies that the constant values have changed. diff --git a/x/rawsock/doc.go b/x/rawsock/doc.go new file mode 100644 index 0000000..6109c83 --- /dev/null +++ b/x/rawsock/doc.go @@ -0,0 +1,11 @@ +// Package rawsock listens and reads over Linux sockets through syscalls +// directly, without the net package. It exists so a server can be measured +// without net.TCPConn, its netFD and its poller on the path. +// +// [Conn] and [Listener] implement [net.Conn] and [net.Listener] the way +// TinyGo's net package does: a connection is its file descriptor plus its +// addresses, deadlines live on the struct and are handed to the socket, and +// nothing sits between a Read and the syscall. A server written against these +// interfaces runs unchanged over the net package, over this package and over +// TinyGo's netdev. +package rawsock diff --git a/x/rawsock/rawsock_darwin.go b/x/rawsock/rawsock_darwin.go new file mode 100644 index 0000000..6846d59 --- /dev/null +++ b/x/rawsock/rawsock_darwin.go @@ -0,0 +1,7 @@ +//go:build !tinygo && darwin + +package rawsock + +import "syscall" + +const sysaccept = syscall.SYS_ACCEPT diff --git a/x/rawsock/rawsock_linux.go b/x/rawsock/rawsock_linux.go new file mode 100644 index 0000000..b141fd7 --- /dev/null +++ b/x/rawsock/rawsock_linux.go @@ -0,0 +1,7 @@ +//go:build !tinygo && linux + +package rawsock + +import "syscall" + +const sysaccept = syscall.SYS_ACCEPT4 diff --git a/x/rawsock/rawsock_other.go b/x/rawsock/rawsock_other.go new file mode 100644 index 0000000..cb3f271 --- /dev/null +++ b/x/rawsock/rawsock_other.go @@ -0,0 +1,49 @@ +//go:build tinygo || (!linux && !darwin) + +package rawsock + +import ( + "errors" + "net" + "net/netip" + "time" +) + +var ( + _ net.Conn = (*Conn)(nil) + _ net.Listener = (*Listener)(nil) +) + +// Addr is a socket address implementing [net.Addr]. +type Addr netip.AddrPort + +func (a Addr) Network() string { return "tcp" } +func (a Addr) String() string { return netip.AddrPort(a).String() } +func (a Addr) AddrPort() netip.AddrPort { return netip.AddrPort(a) } + +// Conn is an accepted connection. Unsupported on this platform. +type Conn struct{} + +func (c *Conn) Read(b []byte) (int, error) { return 0, errors.ErrUnsupported } +func (c *Conn) Write(b []byte) (int, error) { return 0, errors.ErrUnsupported } +func (c *Conn) Close() error { return errors.ErrUnsupported } +func (c *Conn) LocalAddr() net.Addr { return Addr{} } +func (c *Conn) RemoteAddr() net.Addr { return Addr{} } + +func (c *Conn) SetDeadline(t time.Time) error { return errors.ErrUnsupported } +func (c *Conn) SetReadDeadline(t time.Time) error { return errors.ErrUnsupported } +func (c *Conn) SetWriteDeadline(t time.Time) error { return errors.ErrUnsupported } +func (c *Conn) SetReadTimeout(timeout time.Duration) error { return errors.ErrUnsupported } +func (c *Conn) SetWriteTimeout(timeout time.Duration) error { return errors.ErrUnsupported } + +// Listener is a listening socket. Unsupported on this platform. +type Listener struct{} + +// Listen is unsupported on this platform. +func (l *Listener) Listen(port uint16) error { return errors.ErrUnsupported } + +func (l *Listener) Accept() (net.Conn, error) { return nil, errors.ErrUnsupported } +func (l *Listener) AcceptConn(conn *Conn) error { return errors.ErrUnsupported } +func (l *Listener) Addr() net.Addr { return Addr{} } +func (l *Listener) Port() uint16 { return 0 } +func (l *Listener) Close() error { return errors.ErrUnsupported } diff --git a/x/rawsock/rawsock_unix.go b/x/rawsock/rawsock_unix.go new file mode 100644 index 0000000..7bfbdde --- /dev/null +++ b/x/rawsock/rawsock_unix.go @@ -0,0 +1,244 @@ +//go:build !tinygo && (linux || darwin) + +package rawsock + +import ( + "net" + "net/netip" + "syscall" + "time" + "unsafe" +) + +// The interfaces this package exists to satisfy: an http.Server and a heapless +// router must both accept what Listen hands back. +var ( + _ net.Conn = (*Conn)(nil) + _ net.Listener = (*Listener)(nil) +) + +// Addr is a socket address. It implements [net.Addr] over a +// [netip.AddrPort], which costs no allocation to carry around, unlike +// [net.TCPAddr] and its IP slice. +type Addr netip.AddrPort + +// Network returns "tcp". +func (a Addr) Network() string { return "tcp" } + +// String returns the address in host:port form. +func (a Addr) String() string { return netip.AddrPort(a).String() } + +// AddrPort returns the address as a [netip.AddrPort]. +func (a Addr) AddrPort() netip.AddrPort { return netip.AddrPort(a) } + +// Conn is an accepted TCP connection over a raw socket file descriptor. It +// implements [net.Conn], so both an [net/http.Server] and a heapless router can +// be handed the same connection. +// +// Deadlines are enforced by the socket itself through SO_RCVTIMEO and +// SO_SNDTIMEO: a read that runs out of time fails with a [net.Error] that +// reports Timeout, as callers of net.Conn expect. +type Conn struct { + fd int + local, remote Addr + readDeadline time.Time + writeDeadline time.Time +} + +// Read reads bytes from the connection into b. +func (c *Conn) Read(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + n, err := syscall.Read(c.fd, b) + if err != nil { + return 0, c.opError("read", err) + } + if n == 0 { + return 0, syscall.ECONNRESET // Peer closed. + } + return n, nil +} + +// Write writes b to the connection, looping until all bytes are sent. +func (c *Conn) Write(b []byte) (int, error) { + total := 0 + for total < len(b) { + n, err := syscall.Write(c.fd, b[total:]) + if err != nil { + return total, c.opError("write", err) + } + total += n + } + return total, nil +} + +// Close closes the underlying file descriptor. +func (c *Conn) Close() error { + return syscall.Close(c.fd) +} + +// LocalAddr returns the address the connection was accepted on. +func (c *Conn) LocalAddr() net.Addr { return c.local } + +// RemoteAddr returns the peer address of the connection. +func (c *Conn) RemoteAddr() net.Addr { return c.remote } + +// SetDeadline sets both the read and write deadline. A zero time removes them. +func (c *Conn) SetDeadline(t time.Time) error { + err := c.SetReadDeadline(t) + if err != nil { + return err + } + return c.SetWriteDeadline(t) +} + +// SetReadDeadline makes reads after t fail with a timeout error. A zero time +// lets reads block indefinitely. +func (c *Conn) SetReadDeadline(t time.Time) error { + c.readDeadline = t + return c.SetReadTimeout(untilDeadline(t)) +} + +// SetWriteDeadline makes writes after t fail with a timeout error. A zero time +// lets writes block indefinitely. +func (c *Conn) SetWriteDeadline(t time.Time) error { + c.writeDeadline = t + return c.SetWriteTimeout(untilDeadline(t)) +} + +// SetReadTimeout limits how long a single Read waits for data before failing +// with a timeout error. Zero blocks indefinitely. This is what stops a peer +// that opens a connection and then stalls from holding a server worker: the +// connection, not the HTTP handler, owns the idle policy. +func (c *Conn) SetReadTimeout(timeout time.Duration) error { + return setSockTimeout(c.fd, syscall.SO_RCVTIMEO, timeout) +} + +// SetWriteTimeout limits how long a single Write waits for the send buffer to +// drain before failing with a timeout error. Zero blocks indefinitely. +func (c *Conn) SetWriteTimeout(timeout time.Duration) error { + return setSockTimeout(c.fd, syscall.SO_SNDTIMEO, timeout) +} + +// opError wraps a socket error the way the net package does, so that callers +// which test for a timeout with [net.Error] see one. A deadline that has passed +// surfaces from the kernel as EAGAIN, which on a blocking socket only ever +// means the timeout fired. +func (c *Conn) opError(op string, err error) error { + if err == syscall.EAGAIN || err == syscall.EWOULDBLOCK { + err = errTimeout{} + } + return &net.OpError{Op: op, Net: "tcp", Source: c.local, Addr: c.remote, Err: err} +} + +// errTimeout is the error a read or write past its deadline fails with. +type errTimeout struct{} + +func (errTimeout) Error() string { return "i/o timeout" } +func (errTimeout) Timeout() bool { return true } +func (errTimeout) Temporary() bool { return true } + +func untilDeadline(t time.Time) time.Duration { + if t.IsZero() { + return 0 // No deadline: block indefinitely. + } + remaining := time.Until(t) + if remaining <= 0 { + return time.Nanosecond // Already past: fail the next call, do not block. + } + return remaining +} + +func setSockTimeout(fd, option int, timeout time.Duration) error { + tv := syscall.NsecToTimeval(int64(timeout)) + return syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, option, &tv) +} + +// Listener is a listening TCP socket bound to a local port. It implements +// [net.Listener]. +type Listener struct { + fd int + local Addr +} + +// Listen creates a listening TCP socket bound to port on all interfaces. A +// zero port lets the kernel choose one, see [Listener.Addr]. +func (l *Listener) Listen(port uint16) error { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != nil { + return err + } + // Allow quick rebind after restart. + if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil { + syscall.Close(fd) + return err + } + addr := &syscall.SockaddrInet4{Port: int(port)} + if err = syscall.Bind(fd, addr); err != nil { + syscall.Close(fd) + return err + } + if err = syscall.Listen(fd, syscall.SOMAXCONN); err != nil { + syscall.Close(fd) + return err + } + l.fd = fd + l.local = Addr{} + bound, err := syscall.Getsockname(fd) + if err != nil { + syscall.Close(fd) + return err + } + if sa4, ok := bound.(*syscall.SockaddrInet4); ok { + l.local = Addr(netip.AddrPortFrom(netip.AddrFrom4(sa4.Addr), uint16(sa4.Port))) + } + return nil +} + +// Accept blocks until an incoming connection arrives and returns it. It +// allocates the connection, as [net.Listener] requires. A server that keeps its +// own connection storage should call [Listener.AcceptConn] instead. +func (l *Listener) Accept() (net.Conn, error) { + conn := new(Conn) + err := l.AcceptConn(conn) + if err != nil { + return nil, err + } + return conn, nil +} + +// AcceptConn blocks until an incoming connection arrives and stores it in conn, +// reusing whatever conn already held. Nothing is allocated. +// +// The syscall is made by hand because [syscall.Accept] allocates the +// [syscall.Sockaddr] it returns, one per accepted connection: the kernel is +// given address storage this call owns instead, and the address is read out of +// it into conn. +func (l *Listener) AcceptConn(conn *Conn) error { + var rsa syscall.RawSockaddrAny + salen := uint32(unsafe.Sizeof(rsa)) + nfd, _, errno := syscall.Syscall6(sysaccept, uintptr(l.fd), + uintptr(unsafe.Pointer(&rsa)), uintptr(unsafe.Pointer(&salen)), 0, 0, 0) + if errno != 0 { + return errno + } + *conn = Conn{fd: int(nfd), local: l.local} + if rsa.Addr.Family == syscall.AF_INET { + sa4 := (*syscall.RawSockaddrInet4)(unsafe.Pointer(&rsa)) + // Port is in network byte order in the sockaddr the kernel filled. + port := uint16(sa4.Port<<8) | uint16(sa4.Port>>8) + conn.remote = Addr(netip.AddrPortFrom(netip.AddrFrom4(sa4.Addr), port)) + } + return nil +} + +// Addr returns the address the socket is bound to, which carries the port the +// kernel picked when the listener was created with port zero. +func (l *Listener) Addr() net.Addr { return l.local } + +// Port returns the port the socket is bound to. +func (l *Listener) Port() uint16 { return l.local.AddrPort().Port() } + +// Close closes the listening socket. +func (l *Listener) Close() error { return syscall.Close(l.fd) } diff --git a/x/xnet/xnet_fuzz_test.go b/x/xnet/xnet_fuzz_test.go index 827f169..9db5503 100644 --- a/x/xnet/xnet_fuzz_test.go +++ b/x/xnet/xnet_fuzz_test.go @@ -32,7 +32,7 @@ func FuzzStackPacketHTTP(f *testing.F) { } hdr.SetMethod("GET") hdr.SetProtocol("HTTP/1.1") - hdr.SetRequestURI("/") + hdr.SetRequestTarget("/") data := hdr.AppendHeaders(nil) pktnum := 0