From 1884cfc9b701e91813eb7960f58cb2b04aadd2c7 Mon Sep 17 00:00:00 2001 From: Pat Whittingslow Date: Fri, 31 Jul 2026 12:47:33 -0300 Subject: [PATCH] Further improvements to httphi API (#173) * add Exchange.WriteBodyString * Mux.MaxPathValues and other improvements * Mux PathValue improvemnt and fixes * MuxSlice more method muxing improvements * diagram out interesting approach to form parsing for clanker * refactor RequestParseForm and achieve greatness in API design * explicit naming of headerCapacityKV value in kvBuffer.Reset * fix Mux bug not matching paths correctly; httpraw HTTP V1 naming applied * rename many examples,use httphi in examples,remove useless maxAwaitingConn field * add ipv4.String * add ipv4 UnspecifiedAddr and BroadcastAddr * add ethernet.String --- ethernet/definitions.go | 9 + ethernet/definitions_test.go | 42 ++ examples/berkeley-listener/berkeley_server.go | 6 +- examples/httphi-fuzz/fuzz.sh | 259 ++++++++++ examples/httphi-fuzz/main-httplinux.go | 475 ++++++++++++++++++ .../index.html | 0 .../main.go | 139 +++-- .../main-httplinux.go | 37 +- .../{httpclient => httpraw-client}/main.go | 2 +- examples/httptap/dhcp.go | 2 +- examples/xcurl/main.go | 2 +- http/httphi/README.md | 7 +- http/httphi/bench_test.go | 9 +- http/httphi/example_test.go | 8 +- http/httphi/exchange.go | 270 +++++++--- http/httphi/exchange_test.go | 198 +++++++- http/httphi/httphi_fuzz_test.go | 8 +- http/httphi/mux.go | 259 ++++++++-- http/httphi/mux_test.go | 309 +++++++++++- http/httphi/router.go | 48 +- http/httphi/router_test.go | 14 +- http/httpraw/cookie.go | 4 +- http/httpraw/form.go | 19 +- http/httpraw/form_test.go | 43 +- http/httpraw/{header.go => headerv1.go} | 121 ++--- .../{header_test.go => headerv1_test.go} | 62 +-- http/httpraw/kvbuffer.go | 13 +- http/httpraw/{parse.go => parsev1.go} | 60 ++- .../{parse_test.go => parsev1_test.go} | 141 ++++-- internet/pcap/capture.go | 2 +- internet/pcap/capture_test.go | 2 +- ipv4/definitions.go | 17 +- ipv4/definitions_test.go | 4 +- x/xnet/xnet_arp_test.go | 3 +- x/xnet/xnet_fuzz_test.go | 4 +- 35 files changed, 2192 insertions(+), 406 deletions(-) create mode 100644 ethernet/definitions_test.go create mode 100755 examples/httphi-fuzz/fuzz.sh create mode 100644 examples/httphi-fuzz/main-httplinux.go rename examples/{httpserver => httphi-over-ethernet}/index.html (100%) rename examples/{httpserver => httphi-over-ethernet}/main.go (68%) rename examples/{http-linux => httphi-server}/main-httplinux.go (77%) rename examples/{httpclient => httpraw-client}/main.go (98%) rename http/httpraw/{header.go => headerv1.go} (83%) rename http/httpraw/{header_test.go => headerv1_test.go} (95%) rename http/httpraw/{parse.go => parsev1.go} (83%) rename http/httpraw/{parse_test.go => parsev1_test.go} (80%) diff --git a/ethernet/definitions.go b/ethernet/definitions.go index a49c9bc..347abb6 100644 --- a/ethernet/definitions.go +++ b/ethernet/definitions.go @@ -25,6 +25,15 @@ const ( MinimumMTU = MinimumFrameLength - sizeHeaderNoVLAN ) +// String returns the colon-separated hexadecimal text representation of a hardware address. +func String(hwAddr [6]byte) string { + // See net/netip's (Addr).string4 pattern. + var buf [maxAddrStringLen]byte + return string(AppendAddr(buf[:0], hwAddr)) +} + +const maxAddrStringLen = len("ff:ff:ff:ff:ff:ff") + // AppendAddr appends the text representation of the hardware address to the destination buffer. func AppendAddr(dst []byte, hwAddr [6]byte) []byte { for i, b := range hwAddr { diff --git a/ethernet/definitions_test.go b/ethernet/definitions_test.go new file mode 100644 index 0000000..eaa4607 --- /dev/null +++ b/ethernet/definitions_test.go @@ -0,0 +1,42 @@ +package ethernet + +import ( + "net" + "testing" +) + +func TestString(t *testing.T) { + for _, tc := range []struct { + addr [6]byte + want string + }{ + {addr: [6]byte{}, want: "00:00:00:00:00:00"}, + {addr: [6]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, want: "ff:ff:ff:ff:ff:ff"}, + {addr: [6]byte{0xde, 0xad, 0xbe, 0xef, 0x00, 0x01}, want: "de:ad:be:ef:00:01"}, + {addr: [6]byte{0x01, 0x00, 0x5e, 0x7f, 0x00, 0x0f}, want: "01:00:5e:7f:00:0f"}, + } { + got := String(tc.addr) + if got != tc.want { + t.Errorf("String(%v): got %q, want %q", tc.addr, got, tc.want) + } + if want := net.HardwareAddr(tc.addr[:]).String(); got != want { + t.Errorf("String(%v) disagrees with net: got %q, want %q", tc.addr, got, want) + } + if got := string(AppendAddr(nil, tc.addr)); got != tc.want { + t.Errorf("AppendAddr(%v): got %q, want %q", tc.addr, got, tc.want) + } + } +} + +func TestString_singleAlloc(t *testing.T) { + addr := [6]byte{0xde, 0xad, 0xbe, 0xef, 0x00, 0x01} + var sink string + allocs := testing.AllocsPerRun(100, func() { + sink = String(addr) + }) + _ = sink + // Only the returned string allocates; the scratch buffer must stay on the stack. + if allocs != 1 { + t.Errorf("expected 1 alloc, got %v", allocs) + } +} diff --git a/examples/berkeley-listener/berkeley_server.go b/examples/berkeley-listener/berkeley_server.go index 310dffb..5a6ac7c 100644 --- a/examples/berkeley-listener/berkeley_server.go +++ b/examples/berkeley-listener/berkeley_server.go @@ -274,7 +274,7 @@ func handleConnNet(conn net.Conn) error { defer conn.Close() conn.SetDeadline(time.Now().Add(10 * time.Second)) - var hdr httpraw.Header + var hdr httpraw.HeaderV1 needMore := true for needMore { _, err := hdr.ReadFromLimited(conn, 1024) @@ -291,7 +291,7 @@ func handleConnNet(conn net.Conn) error { uri := string(hdr.RequestTarget()) fmt.Printf("< %s %s\n", method, uri) - var resp httpraw.Header + var resp httpraw.HeaderV1 resp.SetProtocol("HTTP/1.1") resp.SetStatus("200", "OK") resp.Set("Content-Type", "text/html") @@ -368,7 +368,7 @@ func mockClient(stack *xnet.StackAsync, port uint16, subnet netip.Prefix) { panic("mock client deadline exceeded to establish") } - var hdr httpraw.Header + var hdr httpraw.HeaderV1 hdr.SetMethod("GET") hdr.SetRequestTarget("/") hdr.SetProtocol("HTTP/1.1") diff --git a/examples/httphi-fuzz/fuzz.sh b/examples/httphi-fuzz/fuzz.sh new file mode 100755 index 0000000..2a99510 --- /dev/null +++ b/examples/httphi-fuzz/fuzz.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# fuzz.sh points ffuf at an already running main-httplinux server. Start it +# yourself first, in another terminal, so its log and any crash stay visible: +# +# go run ./examples/http-linux -port 8080 +# ./examples/http-linux/fuzz.sh # all cases +# ./examples/http-linux/fuzz.sh paths cookie # named cases only +# URL=http://localhost:9000 ./examples/http-linux/fuzz.sh +# +# Install ffuf with: go install github.com/ffuf/ffuf/v2@latest +# +# Every case ends by reporting the server is still up: a case that "finds +# nothing" because the process died is the failure this is looking for. +set -u + +URL="${URL:-http://localhost:8080}" +# Matched to the server's FixedNumGoroutines: in worker mode the router owns one +# exchange per goroutine and refuses a connection outright when none is free, so +# that count is what bounds concurrency. Going above it is correct backpressure, +# but it reaches ffuf as a connection error and hides the response a case was +# looking for. Raise it to exercise the drop path. +THREADS="${THREADS:-4}" + +WORDDIR="$(mktemp -d)" +trap 'rm -rf "$WORDDIR"' EXIT + +# --------------------------------------------------------------------------- +# Wordlists. Kept here rather than pulled from SecLists so a run is repeatable +# and every entry is aimed at the parser: percent escapes, separators the +# grammar gives meaning to, and lengths that cross the server's fixed buffers. +# --------------------------------------------------------------------------- +cat >"$WORDDIR/paths.txt" <<'EOF' +admin +login +search +health +echo +upload +users +files +users/alice +users/bob +users/carol +users/mallory +users/al%69ce +users/%zz +users/%2e%2e%2f +users/alice/extra +files/ +files/readme.txt +files/logo.png +files/notes.md +files/a/b/c +files/%2e%2e/%2e%2e/etc/passwd +EOF + +cat >"$WORDDIR/queries.txt" <<'EOF' +go +go+lang +go%20lang +%21%40%23 +%zz +% +%2 +a=b +a&b +a;b +"" +EOF + +cat >"$WORDDIR/passwords.txt" <<'EOF' +hunter2 +password +admin +letmein +hunter2%00 +hunter2+ +hun%74er2 +EOF + +cat >"$WORDDIR/tokens.txt" <<'EOF' +s3cr3t-session-token +admin +"" +"s3cr3t-session-token" +s3cr3t-session-token; debug +s3cr3t-session-token;debug +===== +;;;;; +EOF + +cat >"$WORDDIR/headers.txt" <<'EOF' +plain +with spaces +%00%01%02 +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +EOF + +cat >"$WORDDIR/names.txt" <<'EOF' +a.bin +report.pdf +../escape.txt +%2e%2e%2fescape.txt +EOF + +# grow prints a line of n 'A's, for the cases that walk a value past a buffer. +grow() { printf 'A%.0s' $(seq "$1"); printf '\n'; } +{ for n in 8 64 512 1024 2048 4096 8192; do grow "$n"; done; } >"$WORDDIR/long.txt" + +alive() { + if curl -s -o /dev/null --max-time 5 "$URL/health"; then + printf ' server alive\n\n' + else + printf ' *** SERVER DOWN after this case ***\n\n' + exit 1 + fi +} + +case_header() { printf '=== %s: %s\n' "$1" "$2"; } + +# --------------------------------------------------------------------------- +# Cases. Each one drives a different part of the request through the parser. +# --------------------------------------------------------------------------- + +# paths walks the mux: literal patterns, the "{id}" single-segment wildcard and +# the "{path...}" wildcard that swallows slashes. -mc all because a 404 from an +# unregistered path is a correct answer worth seeing next to the 200s. +fuzz_paths() { + case_header paths "mux patterns, wildcards and percent escapes in the path" + ffuf -u "$URL/FUZZ" -w "$WORDDIR/paths.txt" -t "$THREADS" -s -timeout 5 -mc all -fc 404 + alive +} + +# recursion follows the "{path...}" wildcard down, which is the pattern a +# directory scanner exercises hardest. +fuzz_recursion() { + case_header recursion "\"{path...}\" wildcard walked recursively" + ffuf -u "$URL/files/FUZZ" -w "$WORDDIR/paths.txt" -t "$THREADS" -s -timeout 5 \ + -recursion -recursion-depth 2 -recursion-strategy greedy -mc all -fc 404 + alive +} + +# longpath pushes the request-target past RequestHeaderBufferSize. The server +# should answer 431 or drop the connection, never serve a mangled path. +fuzz_longpath() { + case_header longpath "request-target grown past the request header buffer" + ffuf -u "$URL/FUZZ" -w "$WORDDIR/long.txt" -t 4 -s -timeout 5 -mc all + alive +} + +# query drives RequestQueryValue and the percent decoder, including escapes that +# do not decode, which must come back 400 and not half decoded. +fuzz_query() { + case_header query "query string values, valid and malformed escapes" + ffuf -u "$URL/search?q=FUZZ" -w "$WORDDIR/queries.txt" -t "$THREADS" -s -timeout 5 -mc all + ffuf -u "$URL/search?q=go&limit=FUZZ" -w "$WORDDIR/queries.txt" -t "$THREADS" -s -timeout 5 -mc all + case_header query "query value grown past the request header buffer" + ffuf -u "$URL/search?q=FUZZ" -w "$WORDDIR/long.txt" -t 4 -s -timeout 5 -mc all + alive +} + +# form posts "application/x-www-form-urlencoded" bodies, the case the credential +# check answers 200 for and everything else 401. +fuzz_form() { + case_header form "urlencoded body pairs; 200 is the credential that works" + ffuf -u "$URL/login" -X POST -w "$WORDDIR/passwords.txt" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'user=admin&pass=FUZZ' -t "$THREADS" -s -timeout 5 -mc all -fc 401 + case_header form "body grown past the form buffer, which may not grow" + ffuf -u "$URL/login" -X POST -w "$WORDDIR/long.txt" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'user=admin&pass=FUZZ' -t 4 -s -timeout 5 -mc all + case_header form "pair count driven past the form's fixed pair table" + ffuf -u "$URL/login?a=1&b=2&c=3&d=4&e=5&f=6&g=7&h=8&i=9&j=10&k=11&l=12&m=13&n=14&o=15&p=16&q=17" \ + -X POST -w "$WORDDIR/passwords.txt" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'user=admin&pass=FUZZ' -t "$THREADS" -s -timeout 5 -mc all + alive +} + +# cookie drives the Cookie header parser: quoting, valueless attributes and the +# separators the grammar splits on. +fuzz_cookie() { + case_header cookie "Cookie header values; 200 is the session that works" + ffuf -u "$URL/admin" -w "$WORDDIR/tokens.txt" -b 'session=FUZZ' \ + -t "$THREADS" -s -timeout 5 -mc all -fc 403 + case_header cookie "cookie grown past the cookie buffer" + ffuf -u "$URL/admin" -w "$WORDDIR/long.txt" -b 'session=FUZZ' -t 4 -s -timeout 5 -mc all + alive +} + +# headers fuzzes a header field value and the field count, /echo handing back +# the header block as the parser stored it. +fuzz_headers() { + case_header headers "header field values echoed back through the parser" + ffuf -u "$URL/echo" -w "$WORDDIR/headers.txt" -H 'X-Fuzz: FUZZ' \ + -t "$THREADS" -s -timeout 5 -mc all + case_header headers "header value grown past the request header buffer" + ffuf -u "$URL/echo" -w "$WORDDIR/long.txt" -H 'X-Fuzz: FUZZ' -t 4 -s -timeout 5 -mc all + alive +} + +# multipart fuzzes the part header block: the filename parameter picks whether a +# part is streamed to a sink or discarded. +fuzz_multipart() { + case_header multipart "multipart part headers and filenames" + ffuf -u "$URL/upload" -X POST -w "$WORDDIR/names.txt" \ + -H 'Content-Type: multipart/form-data; boundary=X' \ + -d $'--X\r\nContent-Disposition: form-data; name="f"; filename="FUZZ"\r\n\r\ndata\r\n--X--\r\n' \ + -t "$THREADS" -s -timeout 5 -mc all + case_header multipart "part header grown past the multipart buffer, expect 413" + ffuf -u "$URL/upload" -X POST -w "$WORDDIR/long.txt" \ + -H 'Content-Type: multipart/form-data; boundary=X' \ + -d $'--X\r\nContent-Disposition: form-data; name="f"; filename="FUZZ"\r\n\r\ndata\r\n--X--\r\n' \ + -t 4 -s -timeout 5 -mc all + alive +} + +# methods sends a method per registration and a few the server never names. +# "/echo" is registered without one, so any method reaches it; "/login" is +# POST only and everything else must 404 there. +fuzz_methods() { + case_header methods "registered, unregistered and extension methods" + printf 'GET\nPOST\nPUT\nDELETE\nPATCH\nHEAD\nOPTIONS\nTRACE\nPROPFIND\nBREW\n' >"$WORDDIR/methods.txt" + ffuf -u "$URL/echo" -w "$WORDDIR/methods.txt" -X FUZZ -t "$THREADS" -s -timeout 5 -mc all + ffuf -u "$URL/login" -w "$WORDDIR/methods.txt" -X FUZZ -t "$THREADS" -s -timeout 5 -mc all -fc 404 + alive +} + +# clusterbomb crosses a path wordlist with a query wordlist, so the two parsers +# are driven by unrelated inputs in the same request. +fuzz_clusterbomb() { + case_header clusterbomb "path and query fuzzed together, every combination" + ffuf -u "$URL/PATH?q=QUERY" -mode clusterbomb \ + -w "$WORDDIR/paths.txt:PATH" -w "$WORDDIR/queries.txt:QUERY" \ + -t "$THREADS" -s -timeout 5 -mc all -fc 404 + alive +} + +ALL=(paths recursion longpath query form cookie headers multipart methods clusterbomb) + +main() { + command -v ffuf >/dev/null || { + echo "ffuf not found: go install github.com/ffuf/ffuf/v2@latest" >&2 + exit 1 + } + curl -s -o /dev/null --max-time 5 "$URL/health" || { + echo "no server at $URL: start it with 'go run ./examples/http-linux'" >&2 + exit 1 + } + local cases=("$@") + [ ${#cases[@]} -eq 0 ] && cases=("${ALL[@]}") + for c in "${cases[@]}"; do + "fuzz_$c" || { echo "unknown case: $c" >&2; exit 1; } + done + echo "all cases done, server still up" +} + +main "$@" diff --git a/examples/httphi-fuzz/main-httplinux.go b/examples/httphi-fuzz/main-httplinux.go new file mode 100644 index 0000000..3373b5e --- /dev/null +++ b/examples/httphi-fuzz/main-httplinux.go @@ -0,0 +1,475 @@ +//go:build !tinygo && linux + +package main + +import ( + "flag" + "io" + "log/slog" + "net" + "os" + "strconv" + "sync/atomic" + "time" + + "github.com/soypat/lneto" + "github.com/soypat/lneto/http/httphi" + "github.com/soypat/lneto/http/httpraw" +) + +const ( + kB = 1 << 10 + listenPort = 8080 + bufferSizes = 2 * kB + // A browser sends around twenty header fields; a request carrying more + // than this is answered 431 rather than parsed into memory it was not + // given. Each field costs 8 bytes of table. + numHeaderFields = 32 + readTimeout = 2 * time.Second +) + +// Credentials the endpoints check. They are in the source on purpose: this is a +// target to point a fuzzer at, and a scan is only interesting when something is +// there to be found. +const ( + adminUser = "admin" + adminPass = "hunter2" + sessionToken = "s3cr3t-session-token" +) + +// Fixed corpora the handlers answer from, so a path scan separates hits from +// misses instead of finding one status code everywhere. +var ( + users = [...]string{"alice", "bob", "carol"} + files = [...]string{"readme.txt", "logo.png", "notes.md"} +) + +var ( + flagPort = flag.Int("port", listenPort, "TCP port to listen on") + flagVerbose = flag.Bool("v", false, "log every request to stderr; a fuzzer at full rate makes this expensive") + flagThreads = flag.Int("threads", 8, "Number of goroutines to spawn.") +) + +func main() { + flag.Parse() + if err := run(); err != nil { + println("Error:", err.Error()) + os.Exit(1) + } + println("DONE") +} + +func run() error { + ln, err := net.Listen("tcp", ":"+strconv.Itoa(*flagPort)) + if err != nil { + return err + } + defer ln.Close() + print("listening on http://localhost:", *flagPort, "\n") + + var server Server + // One scratch per router goroutine: the router serves that many requests at + // once, so a handler always finds one waiting for it. + server.initScratch(*flagThreads) + // "{$}" matches the empty path and nothing else, so an unregistered path + // gets a 404 instead of the homepage. A bare "/" is a catch-all. + server.Handle("GET /{$}", server.homepage) + server.Handle("GET /health", server.health) + server.Handle("GET /search", server.search) + server.Handle("POST /login", server.login) + server.Handle("GET /admin", server.admin) + server.Handle("GET /users/{id}", server.user) + server.Handle("GET /files/{path...}", server.file) + server.Handle("POST /upload", server.upload) + server.Handle("/echo", server.echo) // No method: any method matches. + + var router httphi.Router + err = router.Configure(httphi.RouterConfig{ + FixedNumGoroutines: *flagThreads, + RequestHeaderBufferSize: bufferSizes, + RequestNumHeaderKVCap: numHeaderFields, + ResponseHeaderMinBufferSize: bufferSizes, + Mux: &server.mux, + Logger: slog.Default(), + }) + if err != nil { + return err + } + defer router.Shutdown() + + for { + conn, err := ln.Accept() + if err != nil { + return err + } + // The connection owns the idle policy: a peer that opens a socket and + // then stalls fails its read instead of holding a router goroutine. + conn.SetReadDeadline(time.Now().Add(readTimeout)) + err = router.Handle(conn) + if err != nil { + // Every goroutine is busy and the queue is full. Dropping the + // connection is the backpressure: memory stays bounded. + slog.Warn("dropped connection", slog.String("remote", conn.RemoteAddr().String()), slog.String("err", err.Error())) + conn.Close() + } + } +} + +const ( + htmlHead = `
` + + `` + + `*** WELCOME TO MY HOMEPAGE ***` + + `

YOU ARE VISITOR #` + htmlTail = `!

` + + `Sign my guestbook!
` + + `/search | ` + + `/users/{id} | ` + + `/files/ | ` + + `/admin | ` + + `/echo | ` + + `/health` + + `
Best viewed in Netscape Navigator
` + // maxPage bounds the rendered page: both halves plus the visitor number. + maxPage = len(htmlHead) + 20 + len(htmlTail) +) + +// Compile-time check that a scratch's render buffer holds the largest page a +// handler builds. A page outgrowing it would grow the buffer on the heap, +// which is the one thing this server is written not to do. +const _ = uint(outBufferSize - maxPage) + +type Server struct { + // Visits counts served requests. + Visits atomic.Uint64 + mux httphi.MuxSlice + // scratch is a fixed pool of per-request working memory, see [scratch]. + scratch chan *scratch +} + +// Sizes of a [scratch]. Every one of these is memory spent once per pooled +// scratch, so the pool's size times the sum below is what the handlers cost. +const ( + formBufferSize = kB + formNumPairs = 16 + cookieBufferSize = 512 + cookieNumPairs = 8 + multipartBufferSize = kB + maxMultipartParts = 8 + outBufferSize = 2 * kB + tmpBufferSize = 256 +) + +// scratch is the memory a handler works in for the length of one request: the +// [Exchange] buffer holds the request header and the response header, and +// everything a handler parses or renders on top of that lives here. +// +// Parsers are handed their buffer once and forbidden to grow, so a request that +// sends more than the buffer holds is answered an error rather than served from +// memory the server never budgeted for. +type scratch struct { + form httpraw.Form + cookie httpraw.Cookie + // parts is reused across requests: its [httpraw.MultipartHeader] values keep + // the buffers their Name and Filename were copied into. + parts []httphi.MultipartSink + uploads countingSink + + formBuf [formBufferSize]byte + cookieBuf [cookieBufferSize]byte + mpBuf [multipartBufferSize]byte + // out renders the response body, tmp holds a decoded value being read out of + // the request. They are separate because a decode reads into one while the + // body is being built in the other. + out [outBufferSize]byte + tmp [tmpBufferSize]byte +} + +// initScratch fills the pool with n scratches and fixes each parser to its +// buffer. Sizing n to the router's goroutine count bounds handler memory the +// same way the router bounds its own. +func (sv *Server) initScratch(n int) { + sv.scratch = make(chan *scratch, n) + for range n { + s := new(scratch) + s.form.Reset(s.formBuf[:0], formNumPairs) + s.form.EnableBufferGrowth(false) + s.cookie.Reset(s.cookieBuf[:0], cookieNumPairs) + s.cookie.EnableBufferGrowth(false) + s.parts = make([]httphi.MultipartSink, 0, maxMultipartParts) + sv.scratch <- s + } +} + +// acquireScratch takes a scratch out of the pool, blocking while none is free. +// With the pool sized to the router's fixed goroutine count it never blocks: +// a handler running is a goroutine that has not returned its scratch yet. +func (sv *Server) acquireScratch() *scratch { return <-sv.scratch } + +func (sv *Server) releaseScratch(s *scratch) { sv.scratch <- s } + +// Handle registers a handler and wraps it in the middleware every request runs +// through: the visit counter and, when asked for, the request log. +func (sv *Server) Handle(pattern string, handler httphi.HandlerFunc) { + sv.mux.Handle(pattern, func(exch *httphi.Exchange) { + sv.Visits.Add(1) + if *flagVerbose { + println(exch.RequestMethod().String(), exch.MuxPattern()) + } + handler(exch) + }) +} + +func (sv *Server) homepage(exch *httphi.Exchange) { + s := sv.acquireScratch() + defer sv.releaseScratch(s) + page := append(s.out[:0], htmlHead...) + page = strconv.AppendUint(page, sv.Visits.Load(), 10) + page = append(page, htmlTail...) + exch.Respond(httphi.StatusOK, "text/html", page) +} + +func (sv *Server) health(exch *httphi.Exchange) { + exch.RespondString(httphi.StatusOK, "text/plain", "ok\n") +} + +// search reads the query string, i.e: "/search?q=go+lang&limit=2". Values are +// percent and '+' encoded on the wire, so this is the decoder's surface: a +// malformed escape is answered 400 and never half decoded into the response. +func (sv *Server) search(exch *httphi.Exchange) { + s := sv.acquireScratch() + defer sv.releaseScratch(s) + const decode = true + query, present := exch.RequestQueryAppend(s.tmp[:0], "q", decode) + if !present { + exch.RespondString(httphi.StatusBadRequest, "text/plain", "missing or malformed q parameter\n") + return + } + limit := len(users) + if raw, present := exch.RequestQueryValue("limit"); present { + n, ok := atoiBounded(raw, len(users)) + if !ok { + exch.RespondString(httphi.StatusBadRequest, "text/plain", "limit must be a non-negative integer\n") + return + } + limit = n + } + body := append(s.out[:0], "query: "...) + body = append(body, query...) + body = append(body, '\n') + for _, user := range users[:limit] { + body = append(body, user...) + body = append(body, '\n') + } + exch.Respond(httphi.StatusOK, "text/plain", body) +} + +// login reads "application/x-www-form-urlencoded" pairs out of the request body +// and the query string alike, the body winning a key both carry. It is where a +// credential scan lands: +// +// ffuf -X POST -u http://localhost:8080/login -d 'user=admin&pass=FUZZ' \ +// -H 'Content-Type: application/x-www-form-urlencoded' -w passwords.txt -fc 401 +func (sv *Server) login(exch *httphi.Exchange) { + s := sv.acquireScratch() + defer sv.releaseScratch(s) + // Checked ahead of the parse so a body in some other encoding is told what + // is wrong with it, the parser reporting only that it would not parse. + contentType := exch.RequestContentType() + if contentType != nil && !httpraw.MediaTypeIs(contentType, "application/x-www-form-urlencoded") { + exch.RespondString(httphi.StatusUnsupportedMediaType, "text/plain", "expected application/x-www-form-urlencoded\n") + return + } + const parseQuery, queryWins = true, false + err := exch.RequestParseForm(&s.form, parseQuery, queryWins) + if err != nil { + // A body larger than formBufferSize or more pairs than formNumPairs land + // here too: the form was told not to grow, so it refuses instead. + exch.RespondString(httphi.StatusBadRequest, "text/plain", "malformed or oversized form\n") + return + } + if err = s.form.Decode(); err != nil { + exch.RespondString(httphi.StatusBadRequest, "text/plain", "malformed percent escape in form\n") + return + } + user, pass := s.form.Get("user"), s.form.Get("pass") + if string(user) != adminUser || string(pass) != adminPass { + exch.RespondString(httphi.StatusUnauthorized, "text/plain", "bad credentials\n") + return + } + exch.StageHeader("Set-Cookie", "session="+sessionToken+"; Path=/; HttpOnly") + exch.RespondString(httphi.StatusOK, "text/plain", "welcome "+adminUser+"\n") +} + +// admin is gated on the cookie [Server.login] hands out, so a scan of it fuzzes +// the cookie parser: +// +// ffuf -u http://localhost:8080/admin -b 'session=FUZZ' -w tokens.txt -fc 403 +func (sv *Server) admin(exch *httphi.Exchange) { + s := sv.acquireScratch() + defer sv.releaseScratch(s) + err := exch.RequestParseCookie(&s.cookie, "Cookie") + if err != nil { + exch.RespondString(httphi.StatusUnauthorized, "text/plain", "no cookie\n") + return + } + if string(s.cookie.Get("session")) != sessionToken { + exch.RespondString(httphi.StatusForbidden, "text/plain", "forbidden\n") + return + } + body := append(s.out[:0], "admin panel\n"...) + // A valueless attribute, i.e: "session=...; debug", is stored with an empty + // key, so a plain Get would never find it. + if s.cookie.HasKeyOrSingleValue("debug") { + body = append(body, "requests served: "...) + body = strconv.AppendUint(body, sv.Visits.Load(), 10) + body = append(body, '\n') + } + exch.Respond(httphi.StatusOK, "text/plain", body) +} + +// user serves the "{id}" wildcard, a single path segment. Segments are bound +// raw, so the value is decoded here and "/users/al%69ce" reaches alice. +func (sv *Server) user(exch *httphi.Exchange) { + s := sv.acquireScratch() + defer sv.releaseScratch(s) + const decode = true + id, err := exch.PathValueAppend(s.tmp[:0], "id", decode) + if err != nil { + exch.RespondString(httphi.StatusBadRequest, "text/plain", "malformed percent escape in path\n") + return + } + for _, user := range users { + if string(id) == user { + body := append(s.out[:0], `{"user":"`...) + body = append(body, id...) + body = append(body, "\"}\n"...) + exch.Respond(httphi.StatusOK, "application/json", body) + return + } + } + exch.RespondString(httphi.StatusNotFound, "text/plain", "no such user\n") +} + +// file serves the "{path...}" wildcard, which takes the rest of the path +// including its slashes, so "/files/" and "/files/a/b" both reach here. That +// makes it what a recursive scan walks: ffuf -u http://localhost:8080/files/FUZZ -recursion. +func (sv *Server) file(exch *httphi.Exchange) { + s := sv.acquireScratch() + defer sv.releaseScratch(s) + path := exch.PathValue("path") + if len(path) == 0 { + body := append(s.out[:0], "index of /files/\n"...) + for _, file := range files { + body = append(body, file...) + body = append(body, '\n') + } + exch.Respond(httphi.StatusOK, "text/plain", body) + return + } + for _, file := range files { + if string(path) == file { + body := append(s.out[:0], "contents of "...) + body = append(body, path...) + body = append(body, '\n') + exch.Respond(httphi.StatusOK, "text/plain", body) + return + } + } + exch.RespondString(httphi.StatusNotFound, "text/plain", "no such file\n") +} + +// upload streams a "multipart/form-data" body, counting each file part instead +// of storing it. Parts declare no length, so the body is read a bufferful at a +// time and the header of a part that outgrows the buffer is refused 413. +// +// ffuf -X POST -u http://localhost:8080/upload -w names.txt \ +// -H 'Content-Type: multipart/form-data; boundary=X' \ +// -d $'--X\r\nContent-Disposition: form-data; name="f"; filename="FUZZ"\r\n\r\ndata\r\n--X--\r\n' +func (sv *Server) upload(exch *httphi.Exchange) { + s := sv.acquireScratch() + defer sv.releaseScratch(s) + if !httpraw.MediaTypeIs(exch.RequestContentType(), "multipart/form-data") { + exch.RespondString(httphi.StatusUnsupportedMediaType, "text/plain", "expected multipart/form-data\n") + return + } + s.uploads.n = 0 + // The sink is a field of the scratch, so handing it over as an io.WriteCloser + // boxes a pointer that is already on the heap and allocates nothing. + parts, err := exch.ReadMultiparts(s.parts[:0], s.mpBuf[:], func(hdr *httpraw.MultipartHeader) io.WriteCloser { + if len(hdr.Filename) == 0 { + return nil // A plain field, not a file: keep the header, drop the content. + } + return &s.uploads + }) + // Kept even on failure: the headers parsed so far own buffers worth reusing. + // The slice grows with the number of parts, which only the connection's read + // deadline bounds, so a real server would cap it. + s.parts = parts + if err != nil { + if err == lneto.ErrShortBuffer { + exch.RespondString(httphi.StatusRequestEntityTooLarge, "text/plain", "part header too large\n") + } else { + exch.RespondString(httphi.StatusBadRequest, "text/plain", "malformed multipart body\n") + } + return + } + body := append(s.out[:0], "parts: "...) + body = strconv.AppendInt(body, int64(len(parts)), 10) + body = append(body, '\n') + for i := range parts { + body = append(body, parts[i].Header.Name...) + if len(parts[i].Header.Filename) > 0 { + body = append(body, " -> "...) + body = append(body, parts[i].Header.Filename...) + } + body = append(body, '\n') + } + body = append(body, "bytes stored: "...) + body = strconv.AppendInt(body, s.uploads.n, 10) + body = append(body, '\n') + exch.Respond(httphi.StatusOK, "text/plain", body) +} + +// echo hands back the request line and the header block as the parser stored +// it, which is what tells a header fuzzer what its input turned into. +func (sv *Server) echo(exch *httphi.Exchange) { + s := sv.acquireScratch() + defer sv.releaseScratch(s) + body := append(s.out[:0], exch.RequestMethodRaw()...) + body = append(body, ' ') + body = append(body, exch.RequestTarget()...) + body = append(body, '\n') + if value := exch.RequestHeader("X-Fuzz"); value != nil { + body = append(body, "x-fuzz: "...) + body = append(body, value...) + body = append(body, '\n') + } + body = append(body, "-- parsed header --\n"...) + body = exch.RequestHeaderV1Raw().AppendHeaders(body) + exch.Respond(httphi.StatusOK, "text/plain", body) +} + +// countingSink discards a multipart part and counts what it discarded, standing +// in for the file a real upload would write. +type countingSink struct{ n int64 } + +func (c *countingSink) Write(b []byte) (int, error) { c.n += int64(len(b)); return len(b), nil } +func (c *countingSink) Close() error { return nil } + +// atoiBounded parses a decimal number and clamps it to max, reporting false for +// anything that is not one. It works off the bytes rather than converting to a +// string, which would allocate on a path every request takes. +func atoiBounded(b []byte, max int) (int, bool) { + const maxDigits = 9 // Bounded so the accumulator below cannot overflow. + if len(b) == 0 || len(b) > maxDigits { + return 0, false + } + n := 0 + for _, c := range b { + if c < '0' || c > '9' { + return 0, false + } + n = n*10 + int(c-'0') + } + return min(n, max), true +} diff --git a/examples/httpserver/index.html b/examples/httphi-over-ethernet/index.html similarity index 100% rename from examples/httpserver/index.html rename to examples/httphi-over-ethernet/index.html diff --git a/examples/httpserver/main.go b/examples/httphi-over-ethernet/main.go similarity index 68% rename from examples/httpserver/main.go rename to examples/httphi-over-ethernet/main.go index 7f55fff..dab3a02 100644 --- a/examples/httpserver/main.go +++ b/examples/httphi-over-ethernet/main.go @@ -18,11 +18,12 @@ import ( "runtime" "strconv" "strings" + "sync/atomic" "time" "github.com/soypat/lneto" "github.com/soypat/lneto/ethernet" - "github.com/soypat/lneto/http/httpraw" + "github.com/soypat/lneto/http/httphi" "github.com/soypat/lneto/internal" "github.com/soypat/lneto/internal/ltesto" "github.com/soypat/lneto/internet/pcap" @@ -34,6 +35,23 @@ import ( //go:embed index.html var indexhtml string +// Router memory. The router allocates all of it on Configure and never again, +// so these are the whole cost of serving HTTP over the stack. +const ( + // A browser sends around 700 bytes of header on a landing page request. + requestHeaderBuffer = 1024 + // Response headers reuse whatever the request left unused on top of this, + // and the status line does not count towards it. + responseHeaderBuffer = 256 + numHeaderFields = 16 + // One exchange is allocated per worker, and a worker holds its exchange for + // the whole request, so this is what bounds requests served at once. + numWorkers = 2 + // requestTimeout drops a peer that opens a connection and then stalls, + // rather than letting it hold one of the workers. + requestTimeout = 10 * time.Second +) + var softRand = time.Now().Unix() func main() { @@ -225,6 +243,28 @@ func run() (err error) { svPort := uint16(flagPort) fmt.Printf("Listening on %s:%d\n", ipv4.AppendFormatAddr(nil, stack.Addr4()), svPort) + // Routes are registered before Configure: the router reads the mux to size + // the exchanges it allocates, and refuses a mux with nothing registered. + server := httpServer{start: time.Now()} + // "{$}" matches the empty path and nothing else, so anything unregistered + // gets a 404 rather than the index page. + server.handle("GET /{$}", server.index) + server.handle("GET /stats", server.stats) + + var router httphi.Router + err = router.Configure(httphi.RouterConfig{ + FixedNumGoroutines: numWorkers, + RequestHeaderBufferSize: requestHeaderBuffer, + ResponseHeaderMinBufferSize: responseHeaderBuffer, + RequestNumHeaderKVCap: numHeaderFields, + Mux: &server.mux, + Logger: slog.Default(), + }) + if err != nil { + return fmt.Errorf("configuring HTTP router: %w", err) + } + defer router.Shutdown() + // Serve connections in a loop. for { var conn tcp.Conn @@ -254,65 +294,56 @@ func run() (err error) { continue } fmt.Println("connection established from", net.IP(conn.RemoteAddr()).String()) - go func() { - err = handleConnection(&conn) - if err != nil { - fmt.Println("handle error:", err) - } - }() + // The connection owns the idle policy: a peer that stalls fails its read + // instead of holding a worker. conn is declared inside the loop, so the + // worker keeps serving this one while the next iteration listens anew. + conn.SetDeadline(time.Now().Add(requestTimeout)) + err = router.Handle(&conn) + if err != nil { + // Every worker is busy. Dropping is the backpressure that keeps the + // stack's memory bounded, see numWorkers. + slog.Warn("dropped connection", slog.String("err", err.Error())) + conn.Abort() + } } } -func handleConnection(conn *tcp.Conn) error { - conn.SetDeadline(time.Now().Add(10 * time.Second)) +// httpServer holds what the handlers answer with. Routes are registered on its +// mux before [httphi.Router.Configure] runs, which reads the mux to size the +// path values every exchange must hold. +type httpServer struct { + mux httphi.MuxSlice + served atomic.Uint64 + start time.Time +} - // Read HTTP request. - var hdr httpraw.Header - var needMore bool = true - for needMore { - _, err := hdr.ReadFromLimited(conn, 1024) - if err != nil { - return fmt.Errorf("reading request: %w", err) - } - const asResponse = false - needMore, err = hdr.TryParse(asResponse) - if err != nil && !needMore { - return fmt.Errorf("parsing request: %w", err) - } - } +// handle registers handler and wraps it in the logging and counting every +// request goes through, i.e: the "< GET /" line this example has always printed. +func (sv *httpServer) handle(pattern string, handler httphi.HandlerFunc) { + sv.mux.Handle(pattern, func(exch *httphi.Exchange) { + sv.served.Add(1) + fmt.Printf("< %s %s\n", exch.RequestMethodRaw(), exch.RequestTarget()) + handler(exch) + }) +} - method := string(hdr.Method()) - uri := string(hdr.RequestTarget()) - fmt.Printf("< %s %s\n", method, uri) +// index serves the embedded page. The body goes straight to the connection, so +// only its header ever sits in the exchange's buffer and the page's size does +// not enter into how the router is configured. +func (sv *httpServer) index(exch *httphi.Exchange) { + exch.RespondString(httphi.StatusOK, "text/html", indexhtml) +} - // Build response body. - - // Build HTTP response. - var resp httpraw.Header - resp.SetProtocol("HTTP/1.1") - resp.SetStatus("200", "OK") - resp.Set("Content-Type", "text/html") - resp.Set("Content-Length", strconv.Itoa(len(indexhtml))) - resp.Set("Connection", "close") - response, err := resp.AppendResponse(nil) - if err != nil { - return fmt.Errorf("building response: %w", err) - } - response = append(response, indexhtml...) - - // Send response. - _, err = conn.Write(response) - if err != nil { - return fmt.Errorf("writing response: %w", err) - } - err = conn.Flush() - if err != nil { - return fmt.Errorf("flushing response: %w", err) - } - fmt.Printf("> %d bytes sent\n", len(response)) - - conn.Close() - return nil +// stats reports what the stack has served, which is the quickest way to tell a +// working link from a page that came out of a browser cache. +func (sv *httpServer) stats(exch *httphi.Exchange) { + var buf [128]byte + body := append(buf[:0], "requests served: "...) + body = strconv.AppendUint(body, sv.served.Load(), 10) + body = append(body, "\nuptime: "...) + body = append(body, prettyDuration(time.Since(sv.start))...) + body = append(body, '\n') + exch.Respond(httphi.StatusOK, "text/plain", body) } func clear(buf []byte) { diff --git a/examples/http-linux/main-httplinux.go b/examples/httphi-server/main-httplinux.go similarity index 77% rename from examples/http-linux/main-httplinux.go rename to examples/httphi-server/main-httplinux.go index a4548d9..c73a4e1 100644 --- a/examples/http-linux/main-httplinux.go +++ b/examples/httphi-server/main-httplinux.go @@ -41,8 +41,8 @@ func run() error { defer ln.Close() print("listening on http://localhost:", listenPort, "\n") - var mux httphi.MuxSlice - mux.Handle("GET /", homepage) + var server Server + server.Handle("GET /", server.homepage) var router httphi.Router err = router.Configure(httphi.RouterConfig{ @@ -50,8 +50,7 @@ func run() error { RequestHeaderBufferSize: bufferSizes, RequestNumHeaderKVCap: numHeaderFields, ResponseHeaderMinBufferSize: bufferSizes, - MaxAwaitingConns: 256, - Mux: &mux, + Mux: &server.mux, Logger: slog.Default(), }) if err != nil { @@ -89,18 +88,28 @@ const ( maxPage = len(htmlHead) + 20 + len(htmlTail) ) -// visits counts served requests. Handlers run on the router's goroutines, so -// every visitor gets their own number. -var visits atomic.Uint64 +type Server struct { + // visits counts served requests. + Visits atomic.Uint64 + mux httphi.MuxSlice +} -func homepage(exch *httphi.Exchange) { +func (sv *Server) Handle(pattern string, handler httphi.HandlerFunc) { + // Middleware for all incoming requests declared here. + sv.mux.Handle(pattern, func(exch *httphi.Exchange) { + println(exch.RequestMethod().String(), exch.MuxPattern()) + handler(exch) + }) +} + +func (sv *Server) homepage(exch *httphi.Exchange) { var page [maxPage]byte n := copy(page[:], htmlHead) - n += len(strconv.AppendUint(page[n:n], visits.Add(1), 10)) + n += len(strconv.AppendUint(page[n:n], sv.Visits.Add(1), 10)) n += copy(page[n:], htmlTail) - - exch.StageHeader("Content-Type", "text/html") - exch.StageHeaderInt("Content-Length", int64(n), 10) - exch.WriteHeader(int(httphi.StatusOK)) - exch.WriteBody(page[:n]) + exch.Respond(200, "text/html", page[:n]) +} + +func (sv *Server) form(exch *httphi.Exchange) { + } diff --git a/examples/httpclient/main.go b/examples/httpraw-client/main.go similarity index 98% rename from examples/httpclient/main.go rename to examples/httpraw-client/main.go index 8fbd6be..5aa8eb9 100644 --- a/examples/httpclient/main.go +++ b/examples/httpraw-client/main.go @@ -25,7 +25,7 @@ func run() error { flag.IntVar(&port, "lport", 13337, "Local port over which to hit server") flag.Parse() // Prepare GET request. - var hdr httpraw.Header + var hdr httpraw.HeaderV1 hdr.SetMethod("GET") hdr.SetRequestTarget("/") hdr.SetProtocol("HTTP/1.1") diff --git a/examples/httptap/dhcp.go b/examples/httptap/dhcp.go index d3cd97f..b0f1394 100644 --- a/examples/httptap/dhcp.go +++ b/examples/httptap/dhcp.go @@ -168,7 +168,7 @@ func (d *dhcpInterceptor) buildDHCPResponse(buf []byte) (int, error) { // DHCP responses must be broadcast since the client doesn't have // an IP configured yet and the stack would drop unicast packets. *efrm.DestinationHardwareAddr() = ethernet.BroadcastAddr() - *ifrm.DestinationAddr() = [4]byte{255, 255, 255, 255} + *ifrm.DestinationAddr() = ipv4.BroadcastAddr() ifrm.SetTotalLength(totalIPLen) ufrm.SetLength(udpLen) // Source and destination IPs already set by dhcpv4.Server.Encapsulate. diff --git a/examples/xcurl/main.go b/examples/xcurl/main.go index 000d991..9817926 100644 --- a/examples/xcurl/main.go +++ b/examples/xcurl/main.go @@ -305,7 +305,7 @@ func run() (err error) { }) timeHTTPCreate := timer("create HTTP GET request") - var hdr httpraw.Header + var hdr httpraw.HeaderV1 hdr.SetMethod("GET") hdr.SetRequestTarget("/") hdr.SetProtocol("HTTP/1.1") diff --git a/http/httphi/README.md b/http/httphi/README.md index 1b6ac35..6a43c5e 100644 --- a/http/httphi/README.md +++ b/http/httphi/README.md @@ -24,11 +24,9 @@ mux.Handle("GET /", func(ex *httphi.Exchange) { var router httphi.Router err := router.Configure(httphi.RouterConfig{ FixedNumGoroutines: 4, // 4 workers, 4 exchanges, allocated here and never again. - MaxAwaitingConns: 8, // Queue depth. Full queue drops connections. RequestHeaderBufferSize: 1024, ResponseHeaderMinBufferSize: 32, // Shares the request buffer. RequestNumHeaderKVCap: 32, - Backoff: func(uint) time.Duration { return time.Millisecond }, Mux: &mux, }) if err != nil { @@ -49,3 +47,8 @@ for { Runnable server over raw Linux sockets, plus query, form and multipart handlers: [`example_test.go`](./example_test.go). + + +## Naming + +Gonna be honest with y'all. I initially wanted it to be named `httplo` until I saw I could write `httphi.MethHead` with a small change. \ No newline at end of file diff --git a/http/httphi/bench_test.go b/http/httphi/bench_test.go index 99fb33c..4bf5dff 100644 --- a/http/httphi/bench_test.go +++ b/http/httphi/bench_test.go @@ -68,7 +68,7 @@ func BenchmarkHandle(b *testing.B) { request: "GET /?abc=123 HTTP/1.1\r\nHost: tinygo.org\r\nUser-Agent: bench\r\nAccept: */*\r\nConnection: close\r\n\r\n", handler: func(ex *Exchange) { ex.StageHeader("Content-Type", "text/plain") - ex.StageHeaderInt("Content-Length", int64(len(benchBody)), 10) + ex.StageHeaderIntBase("Content-Length", int64(len(benchBody)), 10) data, present := ex.RequestQueryAppend(buf[:0], "abc", true) if !present || !internal.BytesEqual(data, expect) { panic("invalid result") @@ -112,10 +112,13 @@ func BenchmarkRequestParseForm(b *testing.B) { const request = "POST /f HTTP/1.1\r\nHost: tinygo.org\r\n" + "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 27\r\n\r\n" + "user=gopher&msg=hello+world" - buf := make([]byte, 64) + // The form owns its memory now, so pre-size it and forbid growth: an + // allocation on this path is the failure the benchmark is watching for. + benchForm.Reset(make([]byte, 0, 64), 2) + benchForm.EnableBufferGrowth(false) var mux MuxSlice mux.Handle("POST /f", func(ex *Exchange) { - err := ex.RequestParseForm(&benchForm, buf) + err := ex.RequestParseForm(&benchForm, false, false) if err != nil || benchForm.Len() != 2 { panic("invalid result") } diff --git a/http/httphi/example_test.go b/http/httphi/example_test.go index 1094469..33944e7 100644 --- a/http/httphi/example_test.go +++ b/http/httphi/example_test.go @@ -77,10 +77,12 @@ func ExampleMuxSlice_query_forms_multipart() { }) mux.Handle("GET /form", func(ex *httphi.Exchange) { - // Request Body Form. - formbuf := make([]byte, 1024) + // Request Body Form. The form owns the memory: hand it a buffer and + // forbid growth to bound what a request may spend. var form httpraw.Form - err := ex.RequestParseForm(&form, formbuf) + form.Reset(make([]byte, 0, 1024), 8) // Room for 8 pairs. + form.EnableBufferGrowth(false) + err := ex.RequestParseForm(&form, false, false) if err != nil { ex.WriteHeader(httphi.StatusInternalServerError) return diff --git a/http/httphi/exchange.go b/http/httphi/exchange.go index 59d182a..289c28b 100644 --- a/http/httphi/exchange.go +++ b/http/httphi/exchange.go @@ -6,6 +6,7 @@ import ( "slices" "strconv" "sync/atomic" + "unsafe" "github.com/soypat/lneto" "github.com/soypat/lneto/http/httpraw" @@ -33,8 +34,11 @@ type Exchange struct { rawbuf []byte respHeaderOff uint16 respHeaderLen uint16 - reqHdr httpraw.Header - pathValues []pathValue + reqHdr httpraw.HeaderV1 + pathValues []PathValue + // bodyRW is the reader handed to [httpraw.Form.ReadLimited], kept here so + // boxing it into an io.Reader allocates nothing per request. + bodyRW ExchangeRW hijacked bool rw conn @@ -52,24 +56,28 @@ type Exchange struct { // ExchangeConfig is the memory an [Exchange] is fixed to for the rest of its // life by [Exchange.Configure]. A [Router] derives one per exchange from its // [RouterConfig], which is what bounds the router's memory. +// +// Fields open with Required, Conditional or Optional and the constraint in +// brackets, as in [RouterConfig]. type ExchangeConfig struct { - // RawBuf is the single buffer holding the request header, the response + // Required [non-empty] single buffer holding the request header, the response // header and any surplus body. See [Exchange.UnsafeRawBuffer]. RawBuf []byte - // RequestBufferLim reserves the first bytes of RawBuf for the request - // header, the rest being the response. Configure panics if it exceeds RawBuf. + // Required [<=len(RawBuf)] bytes of RawBuf reserved for the request header, + // the rest being the response. Configure panics if it exceeds RawBuf. RequestBufferLim int - // NumHeaderKVCap is how many request header fields may be parsed. A request - // carrying more is answered 431, see [httpraw.ErrHeaderTooMany]. + // Required [>0] request header fields that may be parsed. A request carrying + // more is answered 431, see [httpraw.ErrHeaderTooMany]. NumHeaderKVCap int - // NormalizeOutgoingKeys normalizes staged response header keys as they are + // Optional [any] normalization of staged response header keys as they are // written, i.e: "content-type" becomes "Content-Type". NormalizeOutgoingKeys bool - // NoRequestBufferGrowth holds the request header to RequestBufferLim rather - // than growing it. A header outgrowing it is answered 431, see [httpraw.ErrBufferExhausted]. + // Optional [any] cap holding the request header to RequestBufferLim rather than + // growing it. A header outgrowing it is answered 431, see [httpraw.ErrBufferExhausted]. NoRequestBufferGrowth bool - // MaxPathValues is how many wildcards a single pattern may bind, read back with - // [Exchange.PathValue]. A pattern with more never matches, see [SetPathValues]. + // Conditional [>=the most wildcards any one registered pattern binds] number of + // path values bindable, read back with [Exchange.PathValue]. A pattern binding + // more never matches, see [SetPathValues]. Zero suits a mux of literal patterns. MaxPathValues int } @@ -163,13 +171,18 @@ func (exch *Exchange) Release() { // written to and used without modifying the staged response first line. // // Staging headers will write to this buffer so use mindfully. -// To access only the request header buffer portion use [httpraw.Header.BufferRaw] limited -// to [httpraw.Header.BufferParsed] as returned by [Exchange.RequestHeaderRaw]. +// To access only the request header buffer portion use [httpraw.HeaderV1.BufferRaw] limited +// to [httpraw.HeaderV1.BufferParsed] as returned by [Exchange.requestHeaderRaw]. // Writing to this section will not change the contents read by [Exchange.ReadBody]. // // In [Router] context, the size of this buffer is influenced directly by [RouterConfig] HeaderBufferSize fields. func (exch *Exchange) UnsafeRawBuffer() []byte { return exch.rawbuf } +// RequestHeaderV1Raw returns the parsed request header for access beyond the +// Request* methods, such as [httpraw.HeaderV1.ForEach]. Valid until the exchange +// is released, and writing to it corrupts the response. +func (exch *Exchange) RequestHeaderV1Raw() *httpraw.HeaderV1 { return &exch.reqHdr } + // StageHeader stages a response header field, written on the first // [Exchange.FlushHeader], [Exchange.WriteHeader] or [Exchange.WriteBody]. // Returns false and drops the field if the response buffer cannot fit it. @@ -200,11 +213,23 @@ func (exch *Exchange) StageHeader(key, value string) (enoughMemory bool) { return true } -// StageHeaderInt is [Exchange.StageHeader] with an integer value, i.e: Content-Length. +// StageHeaderBytes is [Exchange.StageHeader] with a byte slice value, i.e: a +// field copied out of the request. The value is not retained. +func (exch *Exchange) StageHeaderBytes(key string, value []byte) (enoughMemory bool) { + return exch.StageHeader(key, b2s(value)) +} + +// StageHeaderInt is [Exchange.StageHeaderIntBase] in base 10, which is the base +// every HTTP field value carrying a number uses, i.e: Content-Length. +func (exch *Exchange) StageHeaderInt(key string, value int64) (enoughMemory bool) { + return exch.StageHeaderIntBase(key, value, 10) +} + +// StageHeaderIntBase is [Exchange.StageHeader] with an integer value, i.e: Content-Length. // It formats the value directly into the response buffer without allocating. // base must be in the range 10..36; lower bases are dropped, no HTTP header // field value is written below base 10. -func (exch *Exchange) StageHeaderInt(key string, value int64, base int) (enoughMemory bool) { +func (exch *Exchange) StageHeaderIntBase(key string, value int64, base int) (enoughMemory bool) { if exch.headerWritten || base < 10 || base > 36 { return false } @@ -252,11 +277,54 @@ func (exch *Exchange) StageStatus(code int) { // WriteHeader sends the status line for code along with the staged header // fields. Only the first call reaches the wire, as in http.ResponseWriter. -func (exch *Exchange) WriteHeader(code int) { +func (exch *Exchange) WriteHeader(code int) (n int, err error) { if !exch.headerWritten { exch.StageStatus(code) - exch.FlushHeader() + n, err = exch.FlushHeader() } + return n, err +} + +// Respond writes a complete response in one call: Content-Type, a Content-Length +// taken from len(body), the status line and the body. An empty contentType +// stages no Content-Type field, for a code that carries no entity. +// +// It also stages "Connection: close", the router serving one exchange per +// connection, so a peer never waits on a response that is not coming. +// +// Returns [Exchange.ResponseError]: staged fields that did not fit and failed +// writes are both reported there, so a truncated response cannot pass silently. +func (exch *Exchange) Respond(code int, contentType string, body []byte) error { + exch.stageResponse(code, contentType, len(body)) + exch.WriteBody(body) // Reports through respErr, checked below. + return exch.respErr +} + +// RespondString is [Exchange.Respond] with a string body, saving the conversion. +func (exch *Exchange) RespondString(code int, contentType, body string) error { + exch.stageResponse(code, contentType, len(body)) + exch.WriteBodyString(body) // Reports through respErr, checked below. + return exch.respErr +} + +// stageResponse stages the fields and status line a complete response needs. +// Drops are recorded on respErr by the Stage* calls, so [Exchange.WriteBody] +// declines to write a partial header afterwards. +func (exch *Exchange) stageResponse(code int, contentType string, bodyLen int) { + if contentType != "" { + exch.StageHeader("Content-Type", contentType) + } + exch.StageHeaderInt("Content-Length", int64(bodyLen)) + // One exchange per connection today, so the peer is told not to wait for a + // second response on it. Revisit once the router loops exchanges. + exch.StageHeader("Connection", "close") + exch.StageStatus(code) +} + +// ResponseError returns any error encountered during staging of headers or during writing of response. +// Provides an ergonomic way of checking if one ran out of buffer space after staging all headers with [Exchange.StageHeader]. +func (exch *Exchange) ResponseError() error { + return exch.respErr } // FlushHeader writes the status line and staged header fields to the connection @@ -319,6 +387,14 @@ func (rw *ExchangeRW) Write(buf []byte) (int, error) { return rw.exch.WriteBody(buf) } +// WriteString wraps [Exchange.WriteBodyString]. Fails if handle no longer valid. +func (rw *ExchangeRW) WriteString(s string) (int, error) { + if err := rw.validate(); err != nil { + return 0, err + } + return rw.exch.WriteBodyString(s) +} + // Read reads request body bytes. See [Exchange.ReadBody]. // Fails with [net.ErrClosed] once the handle is no longer valid. func (rw *ExchangeRW) Read(buf []byte) (int, error) { @@ -346,7 +422,13 @@ func (exch *Exchange) ReadWriter(dst *ExchangeRW) { dst.exch = exch } -// Write writes response body bytes, flushing the header first if the handler +// WriteBodyString implements [io.StringWriter] by unsafe conversion. +// Most underlying [io.Writer] implementations are TCP transport and not modify/own the underlying buffer. +func (exch *Exchange) WriteBodyString(buf string) (int, error) { + return exch.WriteBody(unsafe.Slice(unsafe.StringData(buf), len(buf))) +} + +// WriteBody writes response body bytes, flushing the header first if the handler // has not written it yet. Once a write to the connection fails the response is // unrecoverable and every later write returns that same error, so a body never // reaches the wire without its header. @@ -401,13 +483,6 @@ func (exch *Exchange) MuxPattern() string { return exch.matchedPattern } -// RequestHeaderRaw returns the parsed request header for access beyond the -// Request* methods, such as [httpraw.Header.ForEach]. Valid until the exchange -// is released, and writing to it corrupts the response. -func (exch *Exchange) RequestHeaderRaw() *httpraw.Header { - return &exch.reqHdr -} - // RequestParseCookie parses the request's key header field into dst, i.e: // "Cookie". The caller owns dst and its buffer, so it may be reused between // requests. @@ -422,62 +497,119 @@ func (exch *Exchange) RequestParseCookie(dst *httpraw.Cookie, key string) error func (exch *Exchange) RequestContentType() []byte { // Folded: field names are case insensitive and HTTP/2 mandates lowercase, so // a proxy translating h2 to h1 sends "content-type", RFC 9110 5.1. - return exch.RequestHeaderRaw().GetFold("Content-Type") + return exch.RequestHeaderV1Raw().GetFold("Content-Type") } // RequestContentLength returns the body length declared by the request's // Content-Length field. An absent field is signalled with present=false and no error. -// See [httpraw.Header.ContentLength]. +// See [httpraw.HeaderV1.ContentLength]. func (exch *Exchange) RequestContentLength() (_ int64, present bool, _ error) { - return exch.RequestHeaderRaw().ContentLength() + return exch.RequestHeaderV1Raw().ContentLength() } -// RequestParseForm reads the request body into buf and parses it as -// "application/x-www-form-urlencoded" into dst. buf is the only storage used and -// the only limit: a body longer than buf is refused with [lneto.ErrBufferFull] -// before a single byte is read, leaving the caller free to answer 413. Pairs are -// left as they arrived, call [httpraw.Form.Decode] to decode them in place. +// RequestParseForm parses "application/x-www-form-urlencoded" pairs into dst +// from the request body and, when parseURL is set, from the query string as +// well. Pairs are stored as they arrived, call [httpraw.Form.Decode] to decode +// them in place. // -// Unlike http.Request.ParseForm the query string is not folded in, reach it with -// [Exchange.RequestQuery] or [Exchange.RequestQueryAppend]. The body is consumed, so -// call this before [Exchange.ReadBody]. +// dst owns the memory: both sources are read into its buffer and parsed together +// once. Hand it a preallocated buffer with [httpraw.Form.Reset] and turn growth +// off with [httpraw.Form.EnableBufferGrowth] to bound it, which then reports +// [httpraw.ErrBufferExhausted] instead of allocating. It grows by default. // -// A request with no Content-Length has no body, RFC 9112 6.3, and yields an -// empty form. Use [Exchange.RequestContentLength] to tell that apart from a body -// that arrived empty. -func (exch *Exchange) RequestParseForm(dst *httpraw.Form, buf []byte) error { - if !httpraw.MediaTypeIs(exch.RequestContentType(), "application/x-www-form-urlencoded") { +// prioritizeURL reads the query ahead of the body, so a key carried by both +// resolves to the query's value: [httpraw.Form.Get] answers with the first pair +// holding a key. Both stay readable in wire order through [httpraw.Form.Pair]. +// The body is consumed, so call this before [Exchange.ReadBody]. +// +// A request with no Content-Length has no body, RFC 9112 6.3, and one with no +// Content-Type declares no encoding to parse, RFC 9110 8.3. Neither is an error, +// a bodiless POST being legal, and the query is still parsed when asked for. A +// Content-Type that is present and not form encoded is [errNotFormEncoded]. +func (exch *Exchange) RequestParseForm(dst *httpraw.Form, parseURL, prioritizeURL bool) error { + dst.Reset(nil, 0) // Reuse whatever buffer dst holds, discarding old pairs. + if parseURL && prioritizeURL { + if err := exch.readQueryForm(dst); err != nil { + return err + } + } + if err := exch.readBodyForm(dst); err != nil { + return err + } + if parseURL && !prioritizeURL { + if err := exch.readQueryForm(dst); err != nil { + return err + } + } + return dst.Parse() +} + +// formSeparator joins two sources inside one form buffer. Shared so appending it +// converts no literal per call. +var formSeparator = []byte{'&'} + +// readQueryForm appends the request's query string to dst's buffer. +func (exch *Exchange) readQueryForm(dst *httpraw.Form) error { + query := exch.RequestQuery() + if len(query) == 0 { + return nil + } else if err := separateForm(dst); err != nil { + return err + } + return dst.ReadFromBytes(query) +} + +// readBodyForm appends the request body to dst's buffer, reading until +// Content-Length bytes have arrived. +func (exch *Exchange) readBodyForm(dst *httpraw.Form) error { + contentType := exch.RequestContentType() + if contentType == nil { + return nil // No declared encoding is no form, RFC 9110 8.3. + } else if !httpraw.MediaTypeIs(contentType, "application/x-www-form-urlencoded") { return errNotFormEncoded - } else if exch.RequestHeaderRaw().GetFold("Transfer-Encoding") != nil { + } else if exch.RequestHeaderV1Raw().GetFold("Transfer-Encoding") != nil { // Chunked bodies are framed, so reading Content-Length bytes off the // wire would parse chunk sizes as form data. httpraw does not decode them. return errUnsupportedTransferCoding } - length, present, err := exch.RequestContentLength() - if !present { - dst.Reset(nil, 0) - return nil // No length is no body, RFC 9112 6.3. - } else if err != nil { + if err != nil { return err - } else if length > int64(len(buf)) { - return lneto.ErrShortBuffer // Refuse before reading, caller may answer 413. + } else if !present || length == 0 { + return nil // No length is no body, RFC 9112 6.3. } - buf = buf[:length] - for read := 0; read < len(buf); { - n, err := exch.ReadBody(buf[read:]) + if err = separateForm(dst); err != nil { + return err + } + // Reuse the exchange's own handle: a local would escape when boxed into the + // io.Reader [httpraw.Form.ReadLimited] takes, costing an allocation a request. + exch.ReadWriter(&exch.bodyRW) + // A single read may fall short of the limit, the body arriving a TCP segment + // at a time, so read until the declared length is in hand. + for read := 0; read < int(length); { + n, err := dst.ReadLimited(&exch.bodyRW, int(length)-read) read += n if n == 0 { if err == nil { err = io.ErrNoProgress } else if err == io.EOF { - break + break // Peer sent less than it declared. } return err + } else if err != nil && err != io.EOF { + return err } } - dst.Reset(buf, 0) - return dst.Parse() + return nil +} + +// separateForm appends the '&' keeping two sources from merging into one pair, +// doing nothing while dst holds no bytes yet. +func separateForm(dst *httpraw.Form) error { + if dst.BufferUsed() == 0 { + return nil + } + return dst.ReadFromBytes(formSeparator) } // RequestMultipart returns a parser prepared from the boundary parameter of the @@ -589,26 +721,26 @@ func (exch *Exchange) ReadMultiparts(dst []MultipartSink, buf []byte, newSink fu // RequestHeader returns the value of the first request header field matching // key, or nil if absent. Key matching is case sensitive. func (exch *Exchange) RequestHeader(key string) []byte { - header := exch.RequestHeaderRaw() + header := exch.RequestHeaderV1Raw() return header.Get(key) } // RequestTarget returns the request-target (URI) of the request line, i.e: -// "/search?q=go". See [httpraw.Header.RequestTarget]. +// "/search?q=go". See [httpraw.HeaderV1.RequestTarget]. func (exch *Exchange) RequestTarget() []byte { - return exch.RequestHeaderRaw().RequestTarget() + return exch.RequestHeaderV1Raw().RequestTarget() } // RequestPath returns the request-target (URI) up to the query string. This is // what the [Mux] matches on, i.e: "/search" for a request to "/search?q=go". func (exch *Exchange) RequestPath() []byte { - return exch.RequestHeaderRaw().RequestPath() + return exch.RequestHeaderV1Raw().RequestPath() } // RequestQuery returns the request's query string as it appears on the wire. -// Iterate it with [httpraw.NextQueryPair]. See [httpraw.Header.RequestQuery]. +// Iterate it with [httpraw.NextQueryPair]. See [httpraw.HeaderV1.RequestQuery]. func (exch *Exchange) RequestQuery() []byte { - return exch.RequestHeaderRaw().RequestQuery() + return exch.RequestHeaderV1Raw().RequestQuery() } // RequestQueryValue returns an undecoded view of the first query parameter @@ -694,14 +826,18 @@ func (exch *Exchange) PathValueAppend(dst []byte, key string, decoded bool) ([]b return dst[:base+n], nil } -// RequestMethod returns the request line's method, i.e: "GET". See -// [MethodFromBytes] to compare it against a [Method]. -func (exch *Exchange) RequestMethod() []byte { - return exch.RequestHeaderRaw().Method() +// RequestMethod returns the request's [Method] enum. +func (exch *Exchange) RequestMethod() Method { + return MethodFromBytes(exch.RequestMethodRaw()) +} + +// RequestMethod returns the request line's method as a []byte view, i.e: "GET". +func (exch *Exchange) RequestMethodRaw() []byte { + return exch.RequestHeaderV1Raw().Method() } // RequestConnectionClose returns true if the client asked for the connection to // be closed after this exchange with a "Connection: close" header field. func (exch *Exchange) RequestConnectionClose() bool { - return exch.RequestHeaderRaw().ConnectionClose() + return exch.RequestHeaderV1Raw().ConnectionClose() } diff --git a/http/httphi/exchange_test.go b/http/httphi/exchange_test.go index cf908f2..ed55202 100644 --- a/http/httphi/exchange_test.go +++ b/http/httphi/exchange_test.go @@ -23,6 +23,10 @@ func nopBackoff(consecutiveBackoffs uint) time.Duration { return lneto.BackoffFl // where it says so. const defaultNumHeaderKVCap = 32 +// defaultKVCap is the pair table size tests hand to [httpraw.Form.Reset], a +// bounded form needing room for the pairs it parses. See [httpraw.Form.Reset]. +const defaultKVCap = 8 + // newExchange returns an Exchange acquired on conn, ready to serve a request. func newExchange(t *testing.T, conn conn, cfg ExchangeConfig) *Exchange { t.Helper() @@ -214,7 +218,7 @@ func TestHandleRequestFields(t *testing.T) { var sm MuxSlice route, _, _ := strings.Cut(test.wantURI, "?") // Mux matches on path. sm.Handle(route, func(ex *Exchange) { - gotMethod = string(ex.RequestMethod()) + gotMethod = string(ex.RequestMethodRaw()) gotURI = string(ex.RequestTarget()) gotHost = string(ex.RequestHeader("Host")) ex.WriteHeader(200) @@ -317,7 +321,9 @@ func TestHandleHTTP10Served(t *testing.T) { // No registered handler must yield 404, not an empty response. func TestHandleNoHandler(t *testing.T) { var sm MuxSlice - sm.Handle("GET /", func(ex *Exchange) { t.Error("handler must not run") }) + // "/{$}" is the root and nothing else; a bare "/" is a catch-all that would + // match /nowhere too, see [SetPathValues]. + sm.Handle("GET /{$}", func(ex *Exchange) { t.Error("handler must not run") }) conn := serve(t, "GET /nowhere HTTP/1.1\r\nHost: h\r\n\r\n", &sm) const want = "HTTP/1.1 404 Not Found\r\n\r\n" if got := conn.ViewWritten(); got != want { @@ -580,7 +586,7 @@ func TestExchangeSetHeaderInt(t *testing.T) { t.Run(name, func(t *testing.T) { conn := newConn("") exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*256), RequestBufferLim: 256}) - exch.StageHeaderInt("N", test.value, test.base) + exch.StageHeaderIntBase("N", test.value, test.base) exch.WriteHeader(200) got, _ := strings.CutPrefix(conn.ViewWritten(), "HTTP/1.1 200 OK\r\n") if got != test.want { @@ -594,7 +600,7 @@ func TestExchangeSetHeaderInt(t *testing.T) { func TestExchangeSetHeaderIntNoAlloc(t *testing.T) { exch := newExchange(t, newConn(""), ExchangeConfig{RawBuf: make([]byte, 2*256), RequestBufferLim: 256}) allocs := testing.AllocsPerRun(100, func() { - exch.StageHeaderInt("Content-Length", 1234567890, 10) + exch.StageHeaderIntBase("Content-Length", 1234567890, 10) }) if allocs != 0 { t.Errorf("SetHeaderInt allocated %v times, want 0", allocs) @@ -808,10 +814,13 @@ func TestExchangeRequestParseForm(t *testing.T) { contentType: "text/plain", wantErr: errNotFormEncoded, }, { + // An absent field is not a wrong one: no media type is no body, the + // same answer "no content length" gets above. Only a type that is + // present and not form encoded is an error. name: "no media type", formVals: []formPair{{key: "a", value: "1"}}, noContentType: true, - wantErr: errNotFormEncoded, + wantVals: []formPair{}, }, { // The coding is refused on the field alone, so the body stays off. name: "chunked", @@ -819,10 +828,12 @@ func TestExchangeRequestParseForm(t *testing.T) { extraHeaders: "Transfer-Encoding: chunked\r\n", wantErr: errUnsupportedTransferCoding, }, { + // The form bounds itself now, so an oversized body is the form + // refusing to grow rather than a short buffer handed in. name: "body larger than buffer", formVals: []formPair{{key: "a", value: "1"}, {key: "b", value: "2"}, {key: "c", value: "3"}}, bufsize: 4, - wantErr: lneto.ErrShortBuffer, + wantErr: httpraw.ErrBufferExhausted, }, } { t.Run(test.name, func(t *testing.T) { @@ -862,12 +873,16 @@ func TestExchangeRequestParseForm(t *testing.T) { builder.WriteString("\r\n") builder.Write(body) + // The form owns the memory: bufSize bounds it here, growth off so an + // oversized body is reported rather than allocated for. var form httpraw.Form + form.Reset(make([]byte, 0, bufSize), defaultKVCap) + form.EnableBufferGrowth(false) var gotErr error var sm MuxSlice sm.Reset(1) sm.Handle("/f", func(exch *Exchange) { - gotErr = exch.RequestParseForm(&form, make([]byte, bufSize)) + gotErr = exch.RequestParseForm(&form, false, false) if gotErr == nil && test.callDecode { gotErr = form.Decode() } @@ -910,7 +925,7 @@ func TestExchangeRequestParseFormSplit(t *testing.T) { var sm MuxSlice sm.Reset(1) sm.Handle("/f", func(exch *Exchange) { - gotErr = exch.RequestParseForm(&form, make([]byte, 64)) + gotErr = exch.RequestParseForm(&form, false, false) }) exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 2*1024), RequestBufferLim: 1024}) if err := Handle(exch, &sm, nopBackoff); err != nil { @@ -930,7 +945,7 @@ func TestExchangeRequestParseFormDecode(t *testing.T) { var sm MuxSlice sm.Reset(1) sm.Handle("/f", func(exch *Exchange) { - if err := exch.RequestParseForm(&form, make([]byte, 64)); err != nil { + if err := exch.RequestParseForm(&form, false, false); err != nil { t.Error(err) } else if err = form.Decode(); err != nil { t.Error(err) @@ -1267,7 +1282,7 @@ func TestHandleBrowserSizedRequest(t *testing.T) { sm.Reset(1) sm.Handle("GET /echo", func(exch *Exchange) { gotMode = string(exch.RequestHeader("X-Mode")) - exch.RequestHeaderRaw().ForEach(func(key, value []byte) bool { + exch.RequestHeaderV1Raw().ForEach(func(key, value []byte) bool { fields++ return true }) @@ -1430,7 +1445,7 @@ func TestExchangeRequestContentTypeFolded(t *testing.T) { sm.Reset(1) sm.Handle("/f", func(exch *Exchange) { gotType = string(exch.RequestContentType()) - gotErr = exch.RequestParseForm(&form, make([]byte, 64)) + gotErr = exch.RequestParseForm(&form, false, false) }) serve(t, "POST /f HTTP/1.1\r\nHost: h\r\n"+name+": "+formType+"\r\nContent-Length: 3\r\n\r\na=1", &sm) @@ -1459,7 +1474,7 @@ func TestExchangeRequestParseFormFoldedTransferEncoding(t *testing.T) { var sm MuxSlice sm.Reset(1) sm.Handle("/f", func(exch *Exchange) { - gotErr = exch.RequestParseForm(&form, make([]byte, 64)) + gotErr = exch.RequestParseForm(&form, false, false) }) serve(t, "POST /f HTTP/1.1\r\nHost: h\r\nContent-Type: application/x-www-form-urlencoded\r\n"+ name+": chunked\r\nContent-Length: "+strconv.Itoa(len(body))+"\r\n\r\n"+body, &sm) @@ -1470,3 +1485,162 @@ func TestExchangeRequestParseFormFoldedTransferEncoding(t *testing.T) { }) } } + +// Respond replaces the stage/stage/stage/write boilerplate every handler paid, +// deriving Content-Length from the body so it cannot disagree with what is sent. +func TestExchangeRespond(t *testing.T) { + for _, test := range []struct { + name string + code int + contentType string + body string + want string + }{ + { + name: "html", code: 200, contentType: "text/html", body: "

hi

", + want: "HTTP/1.1 200 OK\r\nContent-Type:text/html\r\nContent-Length:11\r\nConnection:close\r\n\r\n

hi

", + }, + { + name: "empty body still declares zero length", code: 200, contentType: "text/plain", body: "", + want: "HTTP/1.1 200 OK\r\nContent-Type:text/plain\r\nContent-Length:0\r\nConnection:close\r\n\r\n", + }, + { + name: "no content type staged when empty", code: 204, contentType: "", body: "", + want: "HTTP/1.1 204 No Content\r\nContent-Length:0\r\nConnection:close\r\n\r\n", + }, + { + name: "error code carries a body", code: 500, contentType: "text/plain", body: "boom", + want: "HTTP/1.1 500 Internal Server Error\r\nContent-Type:text/plain\r\nContent-Length:4\r\nConnection:close\r\n\r\nboom", + }, + } { + t.Run(test.name+"/bytes", func(t *testing.T) { + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 512), RequestBufferLim: 256}) + if err := exch.Respond(test.code, test.contentType, []byte(test.body)); err != nil { + t.Fatalf("Respond: %s", err) + } + if got := conn.ViewWritten(); got != test.want { + t.Errorf("want %q, got %q", test.want, got) + } + }) + t.Run(test.name+"/string", func(t *testing.T) { + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 512), RequestBufferLim: 256}) + if err := exch.RespondString(test.code, test.contentType, test.body); err != nil { + t.Fatalf("RespondString: %s", err) + } + if got := conn.ViewWritten(); got != test.want { + t.Errorf("want %q, got %q", test.want, got) + } + }) + } +} + +// A response that does not fit must be reported, not shipped truncated: the +// whole point of folding the boilerplate into one call. +func TestExchangeRespondReportsOverflow(t *testing.T) { + conn := newConn("") + exch := newExchange(t, conn, ExchangeConfig{RawBuf: make([]byte, 64), RequestBufferLim: 32}) + err := exch.Respond(200, strings.Repeat("t", 200), []byte("body")) + if err == nil { + t.Fatal("want an error for a response header that cannot fit") + } + if got := conn.ViewWritten(); got != "" { + t.Errorf("nothing must reach the wire, got %q", got) + } + if exch.ResponseError() == nil { + t.Error("want the failure recorded on the exchange too") + } +} + +// Query and body are read into one form buffer and parsed together, so both +// sources are present at once and read order decides which value a key resolves +// to. A key carried by both keeps both pairs, in wire order. +func TestExchangeRequestParseFormFoldsQuery(t *testing.T) { + const body = "cnt=body&only=b" + const target = "/f?cnt=query&page=2" + for _, test := range []struct { + name string + parseURL, prioritizeURL bool + wantCnt string + wantPage string + wantRendered string + }{ + { + name: "body only", parseURL: false, + wantCnt: "body", wantPage: "", wantRendered: "cnt=body|only=b", + }, + { + name: "query first wins", parseURL: true, prioritizeURL: true, + wantCnt: "query", wantPage: "2", wantRendered: "cnt=query|page=2|cnt=body|only=b", + }, + { + name: "body first wins", parseURL: true, prioritizeURL: false, + wantCnt: "body", wantPage: "2", wantRendered: "cnt=body|only=b|cnt=query|page=2", + }, + } { + t.Run(test.name, func(t *testing.T) { + var form httpraw.Form + var gotErr error + var sm MuxSlice + sm.Reset(1) + sm.Handle("POST /f", func(exch *Exchange) { + gotErr = exch.RequestParseForm(&form, test.parseURL, test.prioritizeURL) + }) + serve(t, "POST "+target+" HTTP/1.1\r\nHost: h\r\n"+ + "Content-Type: application/x-www-form-urlencoded\r\n"+ + "Content-Length: "+strconv.Itoa(len(body))+"\r\n\r\n"+body, &sm) + if gotErr != nil { + t.Fatalf("RequestParseForm: %s", gotErr) + } + if got := string(form.Get("cnt")); got != test.wantCnt { + t.Errorf("want cnt=%q, got %q", test.wantCnt, got) + } + if got := string(form.Get("page")); got != test.wantPage { + t.Errorf("want page=%q, got %q", test.wantPage, got) + } + if got := formString(&form); got != test.wantRendered { + t.Errorf("want pairs %q, got %q", test.wantRendered, got) + } + }) + } +} + +// A GET with a query and no body must fold the query alone: no Content-Type +// means no body to parse, which is not an error. +func TestExchangeRequestParseFormQueryWithoutBody(t *testing.T) { + var form httpraw.Form + var gotErr error + var sm MuxSlice + sm.Reset(1) + sm.Handle("GET /f", func(exch *Exchange) { + gotErr = exch.RequestParseForm(&form, true, true) + }) + serve(t, "GET /f?a=1&b=2 HTTP/1.1\r\nHost: h\r\n\r\n", &sm) + if gotErr != nil { + t.Fatalf("want the query parsed with no body, got %s", gotErr) + } + if got := formString(&form); got != "a=1|b=2" { + t.Errorf("want a=1|b=2, got %q", got) + } +} + +// The separator must not merge the two sources into one pair: without it the +// last query pair and the first body pair run together. +func TestExchangeRequestParseFormSourcesNotMerged(t *testing.T) { + var form httpraw.Form + var sm MuxSlice + sm.Reset(1) + sm.Handle("POST /f", func(exch *Exchange) { + if err := exch.RequestParseForm(&form, true, true); err != nil { + t.Fatal(err) + } + }) + const body = "second=2" + serve(t, "POST /f?first=1 HTTP/1.1\r\nHost: h\r\n"+ + "Content-Type: application/x-www-form-urlencoded\r\n"+ + "Content-Length: "+strconv.Itoa(len(body))+"\r\n\r\n"+body, &sm) + if got := formString(&form); got != "first=1|second=2" { + t.Errorf("want first=1|second=2, got %q", got) + } +} diff --git a/http/httphi/httphi_fuzz_test.go b/http/httphi/httphi_fuzz_test.go index 209f452..012a0be 100644 --- a/http/httphi/httphi_fuzz_test.go +++ b/http/httphi/httphi_fuzz_test.go @@ -64,7 +64,7 @@ const ( // segSizes are the chunk sizes a request may be delivered in, index 0 meaning // "all at once". Splitting a request mid-CRLF or before a colon is what drives -// [httpraw.Header.TryParse]'s resumption path. Entries may be appended, never +// [httpraw.HeaderV1.TryParse]'s resumption path. Entries may be appended, never // changed: an existing index must keep splitting exactly as it does today. var segSizes = [16]int{0, 1, 2, 3, 5, 7, 11, 16, 23, 37, 64, 101, 173, 256, 509, 1024} @@ -152,7 +152,7 @@ func checkResponse(t *testing.T, written string) { if !strings.Contains(written, "\r\n\r\n") { t.Fatalf("header block never terminated: %q", written) } - var resp httpraw.Header + var resp httpraw.HeaderV1 const asResponse = true if err := resp.ParseBytes(asResponse, []byte(written)); err != nil { t.Fatalf("response does not parse back: %s in %q", err, written) @@ -190,7 +190,7 @@ func FuzzHandleRequest(f *testing.F) { // escaping, not this package's framing. for i := range stage { exch.StageHeader("X-Fuzz", "value") - exch.StageHeaderInt("X-Fuzz-Int", int64(i), 10) + exch.StageHeaderIntBase("X-Fuzz-Int", int64(i), 10) } } if ops&opReadBody != 0 { @@ -251,7 +251,7 @@ func FuzzQueryAndForm(f *testing.F) { var form httpraw.Form buf := make([]byte, scratchLen) - if err := exch.RequestParseForm(&form, buf); err != nil { + if err := exch.RequestParseForm(&form, false, false); err != nil { return } total := 0 diff --git a/http/httphi/mux.go b/http/httphi/mux.go index 7ca7e6f..bd38a53 100644 --- a/http/httphi/mux.go +++ b/http/httphi/mux.go @@ -66,6 +66,7 @@ func Handle(exch *Exchange, mux Mux, backoff lneto.BackoffStrategy) error { // Mux on the request path: the query string is the handler's business. path := reqhdr.RequestPath() meth := reqhdr.Method() + clear(exch.pathValues) matchedPattern, handler := mux.LookupHandler(MethodFromBytes(meth), path, exch.pathValues) if handler != nil { exch.matchedPattern = matchedPattern @@ -80,6 +81,18 @@ func Handle(exch *Exchange, mux Mux, backoff lneto.BackoffStrategy) error { } func (exch *Exchange) handleError(err error) { + if err == lneto.ErrUnsupported { + // httpraw refused a first line naming a version it does not speak, before + // spending the field loop on it. An empty protocol is a HTTP/0.9 + // simple-request, RFC 9112 3: a malformed 1.x request-line rather than a + // version there is any point naming back. + if len(exch.reqHdr.Protocol()) == 0 { + exch.WriteHeader(int(StatusBadRequest)) + } else { + exch.WriteHeader(int(StatusHTTPVersionNotSupported)) + } + return + } if err == httpraw.ErrHeaderTooMany || err == httpraw.ErrBufferExhausted || exch.reqHdr.BufferFree() == 0 { // The peer is owed an answer: no larger buffer is coming, so // say so instead of dropping the connection, RFC 6585 5. @@ -101,22 +114,15 @@ type Mux interface { // LookupHandler matches the requestPath and method to a handler and returns it and the // pattern it matched. dstPathVals are set to non-zero values by Mux and can later be accessed by [Exchange.PathValue] // requestPath is a buffer owned by the [Exchange] usually and should not be held after LookupHandler returns. - LookupHandler(get Method, requestPath []byte, dstPathVals []pathValue) (matchedPattern string, handler HandlerFunc) + LookupHandler(get Method, requestPath []byte, dstPathVals []PathValue) (matchedPattern string, handler HandlerFunc) + // MaxPathValues specifies the required size of dstPathVals in a call to [Mux.LookupHandler]. + // MaxPathValues should return -1 if no paths have been configured to catch situation + // where the Mux has been passed to a [Router.Configuration] before registering paths. + MaxPathValues() int } -// MuxSlice is a [Mux] backed by a slice of registered endpoints, matched by -// exact path. Lookup is linear in the number of registrations. -type MuxSlice struct { - // TODO: binary search worth it? - _handlers []struct { - method Method - path string - handler HandlerFunc - setPathVal bool - } -} - -type pathValue struct { +// PathValue used to implement [Mux] interface. Stores http.Request.PathValue-like values. +type PathValue struct { Key string // owned by mux. Value []byte // points to raw exchange buffer. } @@ -133,17 +139,30 @@ var pathSeparator = []byte{'/'} // Unlike ServeMux, segments are compared and bound raw, so "/users/{id}" binds // "x%2Fy" and not "x/y". Which paths match is unaffected. Bound values alias // requestPath rather than copy it. -func SetPathValues(dstPathVals []pathValue, pattern string, requestPath []byte) (matched, pathValSliceTooShort bool) { +// +// Values are bound while walking, before the match is known, so on failure +// SetPathValues clears what it bound. A [Mux] may then try patterns in turn +// without a matching one inheriting values from one that failed. +func SetPathValues(dstPathVals []PathValue, pattern string, requestPath []byte) (matched, pathValSliceTooShort bool) { + n, matched, pathValSliceTooShort := setPathValues(dstPathVals, pattern, requestPath) + if !matched { + clear(dstPathVals[:n]) + } + return matched, pathValSliceTooShort +} + +// setPathValues is [SetPathValues] reporting how many values it bound, so its +// caller can discard them when the pattern turns out not to match. +func setPathValues(dstPathVals []PathValue, pattern string, requestPath []byte) (n int, matched, pathValSliceTooShort bool) { if len(pattern) == 0 || pattern[0] != '/' || len(requestPath) == 0 || requestPath[0] != '/' { - return false, false + return n, false, false } pattern, requestPath = pattern[1:], requestPath[1:] - n := 0 for { if len(pattern) == 0 { // Nothing left after a slash: an anonymous "..." taking the rest, // which is why "/files/" matches "/files/a/b" and "/" matches all. - return true, false + return n, true, false } patSeg, patRest, patMore := strings.Cut(pattern, "/") reqSeg, reqRest, reqMore := bytes.Cut(requestPath, pathSeparator) @@ -152,40 +171,40 @@ func SetPathValues(dstPathVals []pathValue, pattern string, requestPath []byte) case isWildcard && name == "$": // Matches the end of the path and nothing else, so it must be the // last segment of the pattern and leave no path behind. - return !patMore && len(requestPath) == 0, false + return n, !patMore && len(requestPath) == 0, false case isWildcard && isMulti: // Takes the remainder including slashes, possibly empty. if name != "" { if n >= len(dstPathVals) { - return false, true + return n, false, true } - dstPathVals[n] = pathValue{Key: name, Value: requestPath} + dstPathVals[n] = PathValue{Key: name, Value: requestPath} n++ } - return true, false + return n, true, false case isWildcard: if len(reqSeg) == 0 { - return false, false // One segment means a non-empty one. + return n, false, false // One segment means a non-empty one. } if n >= len(dstPathVals) { - return false, true + return n, false, true } - dstPathVals[n] = pathValue{Key: name, Value: reqSeg} + dstPathVals[n] = PathValue{Key: name, Value: reqSeg} n++ default: if b2s(reqSeg) != patSeg { - return false, false + return n, false, false } } if patMore != reqMore { // One side has a further segment and the other does not, so // "/health" misses "/health/" and "/files/" misses "/files". - return false, false + return n, false, false } else if !patMore { - return true, false // Both spent on the same segment. + return n, true, false // Both spent on the same segment. } pattern, requestPath = patRest, reqRest } @@ -205,48 +224,204 @@ func pathWildcard(segment string) (name string, isMulti, ok bool) { return name, false, true } +// MuxSlice is a [Mux] implementation backed by a slice of registered endpoints, matched by +// exact path. Lookup is linear in the number of registrations. +type MuxSlice struct { + // TODO: binary search worth it? + _handlers []struct { + method Method + path string + handler HandlerFunc + pathVals int + spec int + } +} + // Reset discards all registered handlers, reusing the backing array and growing // it to fit capacity registrations. func (sm *MuxSlice) Reset(capacity int) { internal.SliceReuse(&sm._handlers, capacity) } -// LookupHandler returns the handler registered for request path, or nil if none matches. -// The first registration matching both method and uri wins. -func (sm *MuxSlice) LookupHandler(method Method, path []byte, dstPathVals []pathValue) (matched string, _ HandlerFunc) { - for _, endpoint := range sm._handlers { +// LookupHandler returns the handler registered for request path, or nil if none +// matches. The most specific matching registration wins, not the first, so the +// catch-all "/" may be registered alongside the endpoints it backs without +// shadowing them, as in http.ServeMux, see [patternSpecificity]. Registrations +// of equal specificity are resolved in registration order. +// +// Every method this package does not name is [MethUnknown], so a request with an +// extension method matches a bare-path registration and any registration naming +// an extension method, whichever it names. Tell PROPFIND from MKCOL inside the +// handler with [Exchange.RequestMethodRaw]. +func (sm *MuxSlice) LookupHandler(method Method, path []byte, dstPathVals []PathValue) (matched string, _ HandlerFunc) { + best := -1 + bestSpec := 0 + for i, endpoint := range sm._handlers { if endpoint.method != MethUndefined && endpoint.method != method { continue + } else if best >= 0 && endpoint.spec <= bestSpec { + continue // Cannot beat the incumbent, so do not pay to match it. } - // Method matches. - if endpoint.setPathVal { - if ok, _ := SetPathValues(dstPathVals, endpoint.path, path); ok { - return endpoint.path, endpoint.handler - } - } else if b2s(path) == endpoint.path { - return endpoint.path, endpoint.handler + // Method matches. A pattern ending in '/' is a wildcard despite binding no + // values: the trailing slash is an anonymous "{...}", so it must go + // through the matcher and not a literal compare, see [SetPathValues]. + var ok bool + if isWildcardPattern(endpoint.path) { + // dstPathVals is scratch during the scan: a candidate that matches and + // is then beaten, or one that is beaten and clears on failure, would + // leave the winner's values wrong, so the winner is bound below. + ok, _ = SetPathValues(dstPathVals, endpoint.path, path) + } else { + ok = b2s(path) == endpoint.path + } + if ok { + best, bestSpec = i, endpoint.spec } } - return "", nil + if best < 0 { + return "", nil + } + winner := sm._handlers[best] + if isWildcardPattern(winner.path) { + clear(dstPathVals) // The scan may have bound more values than the winner does. + SetPathValues(dstPathVals, winner.path, path) + } + return winner.path, winner.handler +} + +// MaxPathValues returns the maximum number of path values any endpoint could have. +func (sm *MuxSlice) MaxPathValues() (maxPathValues int) { + if len(sm._handlers) == 0 { + return -1 // Signal no handlers registered. + } + for _, endpoint := range sm._handlers { + maxPathValues = max(maxPathValues, endpoint.pathVals) + } + return maxPathValues } // Handle registers handler for reg, either a bare path matching any method or a // method and path separated by a space, i.e: "/health" or "GET /health". -// Handle does not check for duplicate registrations: the first one added wins. +// +// Handle panics on a registration that could never serve a request: a method +// token carrying lowercase (methods are case sensitive and uppercase, RFC 9110 +// 9.1, so "Get" matches no GET request), a path not rooted at '/', or an exact +// duplicate of an earlier registration, which the first one always shadows. +// Registration is program startup, so a fault belongs there and not in a +// permanent silent 404. func (sm *MuxSlice) Handle(optMethodAndPath string, handler HandlerFunc) { - v := internal.SliceReclaim(&sm._handlers) method := MethUndefined methodOrURL, url, methodFound := strings.Cut(optMethodAndPath, " ") if methodFound { + if hasLowerASCII(methodOrURL) { + panic("httphi: method must be uppercase in registration " + optMethodAndPath) + } method = MethodFrom(methodOrURL) } else { url = methodOrURL } + if len(url) == 0 || url[0] != '/' { + panic("httphi: path must begin with '/' in registration " + optMethodAndPath) + } + for _, endpoint := range sm._handlers { + if endpoint.method == method && endpoint.path == url { + if method == MethUnknown { + // Two extension methods are both MethUnknown, so the second is + // unreachable. Register one and branch in the handler, see + // [MuxSlice.LookupHandler]. + panic("httphi: extension method already registered on path in " + optMethodAndPath) + } + panic("httphi: duplicate registration " + optMethodAndPath) + } + } + v := internal.SliceReclaim(&sm._handlers) + v.pathVals = countPathValues(url) + v.spec = patternSpecificity(url) v.method = method v.path = url v.handler = handler } +// patternSpecificity scores how tightly pattern pins a path, letting +// [MuxSlice.LookupHandler] prefer the most specific match over the first one +// registered. A literal segment pins harder than a wildcard segment, and a +// pattern left open at the end ("/", "/files/", "/{p...}") pins less than one +// spent on the whole path, so "/cnt" outscores "/" and "/users/me" outscores +// "/users/{id}". Scoring at registration keeps lookup to an integer compare. +// +// The score is a total order over patterns, which the subset relation is not: +// neither of "/a/{x}/c" and "/a/b/{y}" is more specific than the other, and they +// tie here where http.ServeMux rejects the pair as conflicting. A tie is settled +// by registration order rather than by a panic. +func patternSpecificity(pattern string) (spec int) { + if len(pattern) == 0 || pattern[0] != '/' { + return 0 + } + pattern = pattern[1:] + for { + if len(pattern) == 0 { + return spec // Nothing after a slash: an anonymous "{...}" taking the rest. + } + segment, rest, more := strings.Cut(pattern, "/") + name, isMulti, isWildcard := pathWildcard(segment) + switch { + case isWildcard && name == "$": + return spec + 1 // Ends the path, so nothing is left open. + case isWildcard && isMulti: + return spec // Takes the remainder, pinning nothing more. + case isWildcard: + spec++ + default: + spec += 2 + } + if !more { + return spec + 1 // Spent on the last segment: the pattern is exact. + } + pattern = rest + } +} + +// countPathValues is how many values pattern can bind, which is what sizes the +// slice [SetPathValues] writes into. Only a named wildcard segment binds: "{$}" +// marks the path's end, an anonymous "{...}" has no name to bind under, and a +// brace inside a literal segment is not a wildcard at all. +func countPathValues(pattern string) (n int) { + if len(pattern) == 0 || pattern[0] != '/' { + return 0 + } + pattern = pattern[1:] + for len(pattern) > 0 { + segment, rest, more := strings.Cut(pattern, "/") + if name, _, ok := pathWildcard(segment); ok && name != "" && name != "$" { + n++ + } + if !more { + break + } + pattern = rest + } + return n +} + +// isWildcardPattern reports whether pattern must go through [SetPathValues] +// rather than a literal comparison. Distinct from the value count: "{$}" and a +// trailing slash match by walking segments while binding nothing. +func isWildcardPattern(pattern string) bool { + return strings.IndexByte(pattern, '{') >= 0 || strings.HasSuffix(pattern, "/") +} + +// hasLowerASCII reports whether s carries an ASCII lowercase letter, which a +// method token registered by mistake ("Get") does and a legal extension method +// ("PROPFIND") does not. +func hasLowerASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= 'a' && s[i] <= 'z' { + return true + } + } + return false +} + // Method is a HTTP request method, parsed by [MethodFrom]. type Method uint8 diff --git a/http/httphi/mux_test.go b/http/httphi/mux_test.go index 23d5135..0a55f21 100644 --- a/http/httphi/mux_test.go +++ b/http/httphi/mux_test.go @@ -54,7 +54,7 @@ func TestSetPathValues(t *testing.T) { {pattern: "/a/{x}/b", path: "/a//b", match: false}, } { t.Run(test.pattern+"__"+test.path, func(t *testing.T) { - vals := make([]pathValue, 8) + vals := make([]PathValue, 8) match, tooShort := SetPathValues(vals, test.pattern, []byte(test.path)) if tooShort { t.Fatal("8 slots must be enough for these patterns") @@ -76,7 +76,7 @@ func TestSetPathValues(t *testing.T) { // exchange owns that memory and a copy would allocate per request. func TestSetPathValuesAliasesRequestBuffer(t *testing.T) { path := []byte("/users/42/edit") - vals := make([]pathValue, 4) + vals := make([]PathValue, 4) match, _ := SetPathValues(vals, "/users/{id}/edit", path) if !match { t.Fatal("want match") @@ -94,7 +94,7 @@ func TestSetPathValuesAliasesRequestBuffer(t *testing.T) { // A destination too small to hold every wildcard must say so rather than bind a // partial set or write out of range. func TestSetPathValuesSliceTooShort(t *testing.T) { - match, tooShort := SetPathValues(make([]pathValue, 1), "/{a}/{b}", []byte("/x/y")) + match, tooShort := SetPathValues(make([]PathValue, 1), "/{a}/{b}", []byte("/x/y")) if !tooShort { t.Error("want pathValSliceTooShort for 2 wildcards in 1 slot") } @@ -112,11 +112,11 @@ func TestSetPathValuesSliceTooShort(t *testing.T) { // segment, so "/users/x%2Fy" binds id="x/y" there and id="x%2Fy" here. Matching // agrees either way; only the bound bytes differ. func TestSetPathValuesEscaping(t *testing.T) { - vals := make([]pathValue, 4) + vals := make([]PathValue, 4) if match, _ := SetPathValues(vals, "/a%2Fb/{x}", []byte("/a%2Fb/v")); !match { t.Error("want literal escape in pattern to match the same bytes in path") } - vals = make([]pathValue, 4) + vals = make([]PathValue, 4) match, _ := SetPathValues(vals, "/users/{id}", []byte("/users/x%2Fy")) if !match { t.Fatal("want match") @@ -128,7 +128,7 @@ func TestSetPathValuesEscaping(t *testing.T) { // renderPathValues joins the bound pairs for comparison, stopping at the first // unused slot. -func renderPathValues(vals []pathValue) string { +func renderPathValues(vals []PathValue) string { var sb strings.Builder for _, v := range vals { if v.Key == "" { @@ -147,7 +147,7 @@ func renderPathValues(vals []pathValue) string { // Matching a request must not allocate: keys alias the mux's pattern and values // alias the request buffer, so nothing is copied per request. func TestSetPathValuesNoAlloc(t *testing.T) { - vals := make([]pathValue, 8) + vals := make([]PathValue, 8) path := []byte("/b/bk/o/a/b/c") allocs := testing.AllocsPerRun(100, func() { SetPathValues(vals, "/b/{bucket}/o/{obj...}", path) @@ -164,7 +164,11 @@ type pathValueMux struct { handler HandlerFunc } -func (m *pathValueMux) LookupHandler(method Method, path []byte, dst []pathValue) (string, HandlerFunc) { +func (m *pathValueMux) MaxPathValues() int { + return -1 +} + +func (m *pathValueMux) LookupHandler(method Method, path []byte, dst []PathValue) (string, HandlerFunc) { if ok, _ := SetPathValues(dst, m.pattern, path); ok { return m.pattern, m.handler } @@ -220,3 +224,292 @@ func TestExchangePathValueClearedBetweenRequests(t *testing.T) { t.Errorf("want no path value on a literal route, got id=%q from the previous request", leaked) } } + +// A lookup tries each endpoint in turn and [SetPathValues] binds as it walks, so +// a pattern that binds values and then fails must not leave them behind for the +// pattern that does match: a handler would read a wildcard no matched pattern has. +func TestMuxSliceNoStaleBindingsAcrossCandidates(t *testing.T) { + var sm MuxSlice + sm.Reset(2) + sm.Handle("/a/{x}/{y}/z", func(ex *Exchange) { t.Error("non-matching handler ran") }) + var gotP, gotX, gotY string + sm.Handle("/a/{p}/b", func(ex *Exchange) { + gotP = string(ex.PathValue("p")) + gotX = string(ex.PathValue("x")) + gotY = string(ex.PathValue("y")) + ex.WriteHeader(200) + }) + + exch := new(Exchange) + exch.Configure(ExchangeConfig{ + RawBuf: make([]byte, 2048), RequestBufferLim: 1024, + NumHeaderKVCap: defaultNumHeaderKVCap, MaxPathValues: sm.MaxPathValues(), + }) + conn := newConn("GET /a/1/b HTTP/1.1\r\nHost: h\r\n\r\n") + conn.Hangup() + if !exch.Acquire(conn) { + t.Fatal("fresh exchange failed to acquire") + } + if err := Handle(exch, &sm, nopBackoff); err != nil { + t.Fatal(err) + } + if gotP != "1" { + t.Errorf("want p=1 from the matched pattern, got %q", gotP) + } + if gotX != "" || gotY != "" { + t.Errorf("want no x/y from the failed candidate, got x=%q y=%q", gotX, gotY) + } +} + +// A pattern ending in '/' carries no brace but is still a wildcard: the trailing +// slash is an anonymous "{...}", see [SetPathValues]. MuxSlice must route it +// through the same matcher rather than comparing the path literally. +func TestMuxSliceTrailingSlashPattern(t *testing.T) { + for _, test := range []struct { + pattern string + path string + want bool + }{ + {pattern: "/files/", path: "/files/a/b", want: true}, + {pattern: "/files/", path: "/files/", want: true}, + {pattern: "/files/", path: "/files", want: false}, + {pattern: "/files/", path: "/other/a", want: false}, + {pattern: "/", path: "/anything/at/all", want: true}, + {pattern: "/", path: "/", want: true}, + // Without the trailing slash a pattern stays literal. + {pattern: "/files", path: "/files/a", want: false}, + {pattern: "/files", path: "/files", want: true}, + } { + t.Run(test.pattern+"__"+test.path, func(t *testing.T) { + var sm MuxSlice + sm.Reset(1) + var served bool + sm.Handle(test.pattern, func(ex *Exchange) { served = true; ex.WriteHeader(200) }) + // MuxSlice must agree with the matcher it delegates to. + if ok, _ := SetPathValues(nil, test.pattern, []byte(test.path)); ok != test.want { + t.Fatalf("SetPathValues disagrees with the table: got %v", ok) + } + exch := new(Exchange) + exch.Configure(ExchangeConfig{ + RawBuf: make([]byte, 2048), RequestBufferLim: 1024, + NumHeaderKVCap: defaultNumHeaderKVCap, MaxPathValues: sm.MaxPathValues(), + }) + conn := newConn("GET " + test.path + " HTTP/1.1\r\nHost: h\r\n\r\n") + conn.Hangup() + if !exch.Acquire(conn) { + t.Fatal("fresh exchange failed to acquire") + } + if err := Handle(exch, &sm, nopBackoff); err != nil { + t.Fatal(err) + } + if served != test.want { + t.Errorf("want served=%v, got %v", test.want, served) + } + }) + } +} + +// A malformed registration is a programming error, and one that otherwise costs +// a permanent silent 404 at runtime: "Get /x" parses to MethUnknown, which no +// GET request ever matches but every extension-method request does. Fail at +// registration, where the stack points at the offending line. +func TestMuxSliceHandlePanicsOnBadRegistration(t *testing.T) { + for _, test := range []struct { + name string + reg string + }{ + {name: "lowercase method", reg: "Get /x"}, + {name: "all lower method", reg: "get /x"}, + {name: "mixed case method", reg: "pOsT /x"}, + {name: "no leading slash", reg: "GET x"}, + {name: "bare path no slash", reg: "x"}, + {name: "empty path after method", reg: "GET "}, + } { + t.Run(test.name, func(t *testing.T) { + var sm MuxSlice + sm.Reset(1) + defer func() { + if recover() == nil { + t.Errorf("want panic registering %q", test.reg) + } + }() + sm.Handle(test.reg, func(ex *Exchange) {}) + }) + } +} + +// An exact duplicate is unreachable code: the first registration always wins. +func TestMuxSliceHandlePanicsOnDuplicate(t *testing.T) { + var sm MuxSlice + sm.Reset(2) + sm.Handle("GET /x", func(ex *Exchange) {}) + defer func() { + if recover() == nil { + t.Error("want panic registering the same method and path twice") + } + }() + sm.Handle("GET /x", func(ex *Exchange) {}) +} + +// Extension methods are legal and uppercase, so they must still register: only +// the case-mangled forms are rejected. +func TestMuxSliceHandleAllowsExtensionMethod(t *testing.T) { + var sm MuxSlice + sm.Reset(2) + sm.Handle("PROPFIND /dav", func(ex *Exchange) {}) + sm.Handle("/any-method", func(ex *Exchange) {}) // Bare path matches any method. + if sm.MaxPathValues() != 0 { + t.Errorf("want 0 path values, got %d", sm.MaxPathValues()) + } +} + +// A catch-all registered before the endpoints it sits above must not swallow +// them. "/" is a wildcard pattern: its trailing slash is an anonymous "{...}", +// so a purely first-match-wins scan hands every request to it and the specific +// registrations below become dead code, answered with the root page instead of +// their own body. Registering the site root first is the ordinary way to write +// a mux, so the more specific pattern has to win regardless of order, as in +// http.ServeMux. +func TestMuxSliceSpecificPatternBeatsEarlierCatchAll(t *testing.T) { + for _, test := range []struct { + name string + register []string + path string + want string + }{ + { + name: "root registered first", + register: []string{"/", "/hello", "/cnt", "/6"}, + path: "/cnt", + want: "/cnt", + }, { + name: "root registered last", + register: []string{"/hello", "/cnt", "/6", "/"}, + path: "/cnt", + want: "/cnt", + }, { + name: "root still serves the root path", + register: []string{"/", "/cnt"}, + path: "/", + want: "/", + }, { + name: "root still catches the unregistered", + register: []string{"/", "/cnt"}, + path: "/nowhere", + want: "/", + }, { + name: "subtree wildcard loses to its own literal", + register: []string{"/files/", "/files/index"}, + path: "/files/index", + want: "/files/index", + }, { + name: "subtree wildcard keeps the rest", + register: []string{"/files/", "/files/index"}, + path: "/files/a/b", + want: "/files/", + }, { + name: "longer literal prefix wins over shorter subtree", + register: []string{"/", "/files/", "/files/a/b"}, + path: "/files/a/b", + want: "/files/a/b", + }, { + name: "named wildcard loses to the literal it covers", + register: []string{"/users/{id}", "/users/me"}, + path: "/users/me", + want: "/users/me", + }, { + name: "named wildcard keeps everything else", + register: []string{"/users/{id}", "/users/me"}, + path: "/users/42", + want: "/users/{id}", + }, + } { + t.Run(test.name, func(t *testing.T) { + var sm MuxSlice + sm.Reset(len(test.register)) + for _, pattern := range test.register { + sm.Handle(pattern, func(ex *Exchange) { ex.WriteHeader(200) }) + } + pathVals := make([]PathValue, max(sm.MaxPathValues(), 0)) + got, handler := sm.LookupHandler(MethGet, []byte(test.path), pathVals) + if handler == nil { + t.Fatalf("%s matched no handler, want %q", test.path, test.want) + } + if got != test.want { + t.Errorf("%s matched pattern %q, want %q", test.path, got, test.want) + } + }) + } +} + +// pathVals sizes the slice SetPathValues writes into, so it must count exactly +// the wildcards that bind. Counting braces over-reports: "{$}" marks the path's +// end, an anonymous "{...}" has no name, and a brace inside a literal segment is +// not a wildcard at all. Each of those binds nothing. +func TestMuxSliceMaxPathValuesCountsOnlyBindingWildcards(t *testing.T) { + for _, test := range []struct { + pattern string + want int + }{ + {pattern: "/health", want: 0}, + {pattern: "/", want: 0}, + {pattern: "/files/", want: 0}, // Anonymous trailing wildcard. + {pattern: "/{$}", want: 0}, // End-of-path marker. + {pattern: "/a/{$}", want: 0}, // + {pattern: "/b_{bucket}", want: 0}, // Literal: brace not a whole segment. + {pattern: "/{...}", want: 0}, // Multi wildcard with no name. + {pattern: "/users/{id}", want: 1}, // + {pattern: "/files/{p...}", want: 1}, // + {pattern: "/{a}/{b}", want: 2}, // + {pattern: "/b/{bucket}/o/{obj...}", want: 2}, + } { + t.Run(test.pattern, func(t *testing.T) { + var sm MuxSlice + sm.Reset(1) + sm.Handle(test.pattern, func(ex *Exchange) {}) + if got := sm.MaxPathValues(); got != test.want { + t.Errorf("want %d path values, got %d", test.want, got) + } + }) + } +} + +// Sizing and routing are different questions: "{$}" binds no values yet still +// needs the matcher, so an exact pathVals count must not send it to the literal +// comparison instead. +func TestMuxSliceZeroValueWildcardStillMatches(t *testing.T) { + for _, test := range []struct { + pattern string + path string + want bool + }{ + {pattern: "/{$}", path: "/", want: true}, + {pattern: "/{$}", path: "/x", want: false}, + {pattern: "/a/{$}", path: "/a/", want: true}, + {pattern: "/a/{$}", path: "/a/b", want: false}, + {pattern: "/{...}", path: "/any/thing", want: true}, + } { + t.Run(test.pattern+"__"+test.path, func(t *testing.T) { + var sm MuxSlice + sm.Reset(1) + var served bool + sm.Handle(test.pattern, func(ex *Exchange) { served = true; ex.WriteHeader(200) }) + exch := new(Exchange) + exch.Configure(ExchangeConfig{ + RawBuf: make([]byte, 2048), RequestBufferLim: 1024, + NumHeaderKVCap: defaultNumHeaderKVCap, MaxPathValues: sm.MaxPathValues(), + }) + conn := newConn("GET " + test.path + " HTTP/1.1\r\nHost: h\r\n\r\n") + conn.Hangup() + if !exch.Acquire(conn) { + t.Fatal("acquire") + } + if err := Handle(exch, &sm, nopBackoff); err != nil { + t.Fatal(err) + } + if served != test.want { + t.Errorf("want served=%v, got %v", test.want, served) + } + }) + } +} diff --git a/http/httphi/router.go b/http/httphi/router.go index bed1f06..b86a4d8 100644 --- a/http/httphi/router.go +++ b/http/httphi/router.go @@ -70,34 +70,32 @@ type job struct { } // RouterConfig configures a [Router]. See [Router.Configure]. +// +// Each field below opens with Required, Conditional or Optional followed by the +// constraint in brackets, that being what [RouterConfig.Validate] rejects on. type RouterConfig struct { - // FixedNumGoroutines must be set to either -1 (freely allocate new goroutines) or to the number of goroutines - // to spawn on [Router.Configure] being called. + // Required [-1 or >0] number of goroutines to spawn on [Router.Configure], + // -1 meaning allocate them freely per connection instead. FixedNumGoroutines int - // RequestHeaderBufferSize determines the buffer allocated - // for processing request HTTP headers including request-target (URI), protocol and key/value pairs. + // Required [>=32, sum with ResponseHeaderMinBufferSize <=65535] buffer for the + // request header: request-target (URI), protocol and key/value pairs. RequestHeaderBufferSize int - // ResponseHeaderMinBufferSize determines buffer allocated for processing response headers. - // Response buffer will reuse unused request memory so this is not a strict limit. - // "HTTP/1.1 200 OK\r\n" does not count towards this memory, only actual Headers key/value pairs use this memory. - // After memory is fully consumed [Exchange.StageHeader] will not append more headers. + // Required [>=2, <=65535] buffer for response headers. Reuses unused request + // memory so it is not a strict limit, and the status line does not count + // towards it. Once consumed [Exchange.StageHeader] appends no more fields. ResponseHeaderMinBufferSize int - // Number of request header key/value pairs to parse before failing and returning [StatusRequestHeaderFieldsTooLarge]. + // Required [>0] request header key/value pairs to parse before failing with + // [StatusRequestHeaderFieldsTooLarge]. RequestNumHeaderKVCap int - // Sets maximum number of PathValue pairs that can be set on an exchange. Accessed via [Exchange.PathValue]. - MaxPathValues int - // NormalizeOutgoingKeys normalizes response header field keys as they are + // Optional [any] normalization of response header field keys as they are // staged, i.e: "content-type" becomes "Content-Type". NormalizeOutgoingKeys bool - // MaxAwaitingConns is the depth of the queue connections wait in for a free - // goroutine. [Router.Handle] drops connections once it is full. Required and - // must be non-zero when running a fixed number of goroutines, unused otherwise. - MaxAwaitingConns int - // Mux resolves each request's method and path to the handler serving it. Required. + // Required [non-nil] resolver of each request's method and path to the handler + // serving it. Routes must be registered before Configure, see [Mux.MaxPathValues]. Mux Mux - // Logger receives failed exchanges. Optional, nil disables logging. + // Optional [nil disables] sink for failed exchanges. Logger *slog.Logger } @@ -120,7 +118,6 @@ func (cfg RouterConfig) Validate() error { switch { case cfg.Mux == nil, !workerMode && cfg.FixedNumGoroutines != -1, - workerMode && cfg.MaxAwaitingConns <= 0, cfg.RequestNumHeaderKVCap <= 0, cfg.RequestHeaderBufferSize < minRequestHeaderBuffer, cfg.ResponseHeaderMinBufferSize < minResponseHeaderBuffer, @@ -186,7 +183,11 @@ func (r *Router) Configure(cfg RouterConfig) error { r.respBuf = cfg.ResponseHeaderMinBufferSize r.mux = cfg.Mux r.log = cfg.Logger - r.maxPathValues = cfg.MaxPathValues + maxPathValues := cfg.Mux.MaxPathValues() + if maxPathValues < 0 { + return errors.New("Mux paths must be registered before configuring Router") + } + r.maxPathValues = maxPathValues r.normalizeKeys = cfg.NormalizeOutgoingKeys // Freelist entries were sized by the outgoing configuration: recycling one // would serve a request with buffer limits cfg never asked for. @@ -197,7 +198,7 @@ func (r *Router) Configure(cfg RouterConfig) error { return nil } if workerMode { - jobqueue := make(chan job, cfg.MaxAwaitingConns) + jobqueue := make(chan job, cfg.FixedNumGoroutines) if gen > 1 { // Exchange buffers below are reused: the previous generation must be // done serving before they may be handed to the new one. @@ -210,6 +211,7 @@ func (r *Router) Configure(cfg RouterConfig) error { r.exchs = r.exchs[:numgoro] rawBuflen := cfg.RequestHeaderBufferSize + cfg.ResponseHeaderMinBufferSize internal.SliceReuse(&r.globbuf, numgoro*rawBuflen) + for i := range numgoro { // TODO exchange buffer alloc goff := i * rawBuflen @@ -220,7 +222,7 @@ func (r *Router) Configure(cfg RouterConfig) error { NumHeaderKVCap: cfg.RequestNumHeaderKVCap, NormalizeOutgoingKeys: cfg.NormalizeOutgoingKeys, NoRequestBufferGrowth: true, // Hard memory limit. - MaxPathValues: cfg.MaxPathValues, + MaxPathValues: maxPathValues, }) go r.goroWorker(gen, jobqueue, cfg.Mux) } @@ -306,7 +308,7 @@ func (r *Router) goroWorker(gen uint32, queue chan job, mux Mux) { for job := range queue { exch := job.exch if exch == nil { - panic("httplo: unreachable nil job") + panic("httphi: unreachable nil job") } else if gen != r.gen.Load() { // Not released with freeExch since generation torn down, // new buffer may have been allocated for Exchanges. diff --git a/http/httphi/router_test.go b/http/httphi/router_test.go index db78e2e..d688774 100644 --- a/http/httphi/router_test.go +++ b/http/httphi/router_test.go @@ -196,8 +196,10 @@ func TestRouterRequestVisibleToHandler(t *testing.T) { router Router ) var gotMethod, gotURI, gotHost string + var gotMethodEnum Method sm.Handle("GET /index.html", func(ex *Exchange) { - gotMethod = string(ex.RequestMethod()) + gotMethod = string(ex.RequestMethodRaw()) + gotMethodEnum = ex.RequestMethod() gotURI = string(ex.RequestTarget()) gotHost = string(ex.RequestHeader("Host")) ex.WriteHeader(200) @@ -213,6 +215,9 @@ func TestRouterRequestVisibleToHandler(t *testing.T) { if gotMethod != "GET" { t.Errorf("want method %q, got %q", "GET", gotMethod) } + if gotMethodEnum != MethGet { + t.Errorf("want method enum %q, got %q", MethGet, gotMethodEnum) + } if gotURI != "/index.html" { t.Errorf("want URI %q, got %q", "/index.html", gotURI) } @@ -242,7 +247,9 @@ func TestRouterMux(t *testing.T) { sm MuxSlice router Router ) - sm.Handle("GET /", staticPage(t, "root")) + // "/{$}" is the root alone: a bare "/" is a catch-all and would serve + // "root" for /page and /nowhere as well, see [SetPathValues]. + sm.Handle("GET /{$}", staticPage(t, "root")) sm.Handle("GET /page", staticPage(t, "page")) sm.Handle("/any", staticPage(t, "any")) // No method: matches any. configSynchronousRouter(t, &router, bufferSize, &sm) @@ -383,7 +390,6 @@ func TestRouterHandleAfterTeardown(t *testing.T) { sm.Handle("GET /", staticPage(t, "ok")) err := router.Configure(RouterConfig{ FixedNumGoroutines: 2, - MaxAwaitingConns: 4, Mux: &sm, RequestHeaderBufferSize: 512, RequestNumHeaderKVCap: 16, @@ -416,7 +422,6 @@ func TestRouterTeardownReleasesQueuedConns(t *testing.T) { sm.Handle("GET /", staticPage(t, "ok")) cfg := RouterConfig{ FixedNumGoroutines: numGoro, - MaxAwaitingConns: 4, Mux: &sm, RequestHeaderBufferSize: 512, RequestNumHeaderKVCap: 16, @@ -463,7 +468,6 @@ func TestRouterConfigureDuringWorkerHandle(t *testing.T) { sm.Handle("GET /", staticPage(t, "ok")) cfg := RouterConfig{ FixedNumGoroutines: 2, - MaxAwaitingConns: 4, Mux: &sm, RequestHeaderBufferSize: 512, RequestNumHeaderKVCap: 16, diff --git a/http/httpraw/cookie.go b/http/httpraw/cookie.go index 00c3717..6b22a27 100644 --- a/http/httpraw/cookie.go +++ b/http/httpraw/cookie.go @@ -4,7 +4,7 @@ import ( "bytes" ) -// Cookie implements cookie key-value parsing. Methods function similarly to eponymous [Header] methods. +// Cookie implements cookie key-value parsing. Methods function similarly to eponymous [HeaderV1] methods. // Cookie represents a single-line Cookie header value in a HTTP header, much like the standard library Cookie. type Cookie struct { kv kvBuffer @@ -16,7 +16,7 @@ func (c *Cookie) EnableBufferGrowth(enableBufferGrowth bool) { c.kv.EnableBufferGrowth(enableBufferGrowth) } -// Reset functions very similarly to [Header.Reset]. Can be used for in-place cookie parsing. +// Reset functions very similarly to [HeaderV1.Reset]. Can be used for in-place cookie parsing. func (c *Cookie) Reset(buf []byte, capKV int) { c.kv.Reset(buf, capKV) } func (c *Cookie) valid() bool { diff --git a/http/httpraw/form.go b/http/httpraw/form.go index 314b539..f186427 100644 --- a/http/httpraw/form.go +++ b/http/httpraw/form.go @@ -1,6 +1,9 @@ package httpraw -import "bytes" +import ( + "bytes" + "io" +) // Form holds "application/x-www-form-urlencoded" key-value pairs, the encoding // HTML forms use for POST bodies and query strings alike. Methods function @@ -19,10 +22,24 @@ func (f *Form) EnableBufferGrowth(enableGrowth bool) { f.kv.EnableBufferGrowth(e // Reset discards parsed pairs and sets the buffer to parse in place. // If buf is nil the current buffer is reused. +// +// capKV sizes the pair table. With growth disabled it is a hard limit, so a +// capKV of 0 leaves no room for a single pair and [Form.Parse] answers +// [ErrBufferExhausted]; size it to the pairs expected. func (f *Form) Reset(buf []byte, capKV int) { f.kv.Reset(buf, capKV) } +// ReadFromBytes appends buf to the underlying buffer, accumulating data to parse. Returns ErrBufferExhausted when buf does not fit and growth is disabled. +func (f *Form) ReadFromBytes(b []byte) error { return f.kv.ReadFromBytes(b) } + +// BufferUsed returns bytes accumulated by the Read* methods and awaiting a +// [Form.Parse]. See [kvBuffer.BufferUsed]. +func (f *Form) BufferUsed() int { return f.kv.BufferUsed() } + +// ReadLimited appends at most limit bytes read from r to the underlying buffer. A read returning data alongside io.EOF reports a nil error, later ones io.EOF. +func (f *Form) ReadLimited(r io.Reader, limit int) (int, error) { return f.kv.ReadLimited(r, limit) } + // ParseBytes copies the argument bytes to the Form's underlying buffer and parses them. func (f *Form) ParseBytes(b []byte) error { f.Reset(nil, 0) diff --git a/http/httpraw/form_test.go b/http/httpraw/form_test.go index 7545118..445693e 100644 --- a/http/httpraw/form_test.go +++ b/http/httpraw/form_test.go @@ -148,10 +148,51 @@ func TestFormParseReuseNoAlloc(t *testing.T) { t.Fatal(err) } allocs := testing.AllocsPerRun(100, func() { - f.Reset(body, 0) + f.Reset(body, 0) // 0 preserves the pair storage warmed up above, the reuse under test. f.Parse() }) if allocs != 0 { t.Errorf("reused Form allocated %v times, want 0", allocs) } } + +// BufferUsed reports buffered bytes, not parsed pairs, so a caller appending +// from several sources can tell whether a separator is needed before the next +// one. Form.Len is zero until Parse runs and cannot answer that. +func TestFormBufferUsed(t *testing.T) { + var f Form + f.Reset(nil, defaultKVCap) + if got := f.BufferUsed(); got != 0 { + t.Errorf("want 0 on a fresh form, got %d", got) + } + if err := f.ReadFromBytes([]byte("a=1")); err != nil { + t.Fatal(err) + } + if got := f.BufferUsed(); got != 3 { + t.Errorf("want 3 buffered, got %d", got) + } + if got := f.Len(); got != 0 { + t.Errorf("Len must stay 0 until Parse, got %d", got) + } + // A second source appended behind a separator. + if err := f.ReadFromBytes([]byte("&b=2")); err != nil { + t.Fatal(err) + } + if got := f.BufferUsed(); got != 7 { + t.Errorf("want 7 buffered, got %d", got) + } + if err := f.Parse(); err != nil { + t.Fatal(err) + } + if got := render(&f); got != "a=1|b=2" { + t.Errorf("want a=1|b=2, got %q", got) + } + if got := f.BufferUsed(); got != 7 { + t.Errorf("BufferUsed must not change on Parse, got %d", got) + } + // Reset discards the pairs and the buffered bytes with them. + f.Reset(nil, defaultKVCap) + if got := f.BufferUsed(); got != 0 { + t.Errorf("want 0 after Reset, got %d", got) + } +} diff --git a/http/httpraw/header.go b/http/httpraw/headerv1.go similarity index 83% rename from http/httpraw/header.go rename to http/httpraw/headerv1.go index 8c40377..6750981 100644 --- a/http/httpraw/header.go +++ b/http/httpraw/headerv1.go @@ -8,7 +8,8 @@ import ( const ( methodGet = "GET" - strHTTP11 = "HTTP/1.1" + strHTTP1 = "HTTP/1." + strHTTP11 = strHTTP1 + "1" strCRLF = "\r\n" headerCookie = "Cookie" headerConnection = "Connection" @@ -19,7 +20,7 @@ const ( // Flags is a bitset of signals gathered while parsing or building a header, // such as a status code having been set or the peer requesting connection -// close. See [Header.Flags]. +// close. See [HeaderV1.Flags]. type Flags uint16 const ( @@ -40,15 +41,15 @@ func (f Flags) HasAny(checkThese Flags) bool { return f&checkThese != 0 } -// Header implements "raw" HTTP header key-value parsing, validation and marshalling. +// HeaderV1 implements "raw" HTTP header key-value parsing, validation and marshalling. // // It does NOT implement: // - Normalization. // - Cookies (see [Cookie]). // - Special header optimizations. // - Content-Length validation and other special header field value validation. -type Header struct { - hbuf headerBuf +type HeaderV1 struct { + hbuf headerv1Buf // Request fields. method view @@ -62,18 +63,18 @@ type Header struct { } // Flags returns [Flags] to signal status code has been set, Connection:Close or other useful signals provided by flags. -func (h *Header) Flags() Flags { return h.hbuf.kv.flags } +func (h *HeaderV1) Flags() Flags { return h.hbuf.kv.flags } // ConfigBufferGrowth configures the memory the header may use. Setting -// outlives [Header.Reset]. Call before parsing/reading. +// outlives [HeaderV1.Reset]. Call before parsing/reading. // // enableBufferGrowth enables growing both the header buffer and the header key/value pair slice. -func (h *Header) ConfigBufferGrowth(enableBufferGrowth bool) { +func (h *HeaderV1) ConfigBufferGrowth(enableBufferGrowth bool) { h.hbuf.kv.EnableBufferGrowth(enableBufferGrowth) } // ParseBytes copies the bytes into buffer and parses the HTTP header. It fails if HTTP header data is incomplete. -func (h *Header) ParseBytes(asResponse bool, b []byte) error { +func (h *HeaderV1) ParseBytes(asResponse bool, b []byte) error { h.Reset(nil, 0) err := h.hbuf.kv.ReadFromBytes(b) if err != nil { @@ -82,9 +83,9 @@ func (h *Header) ParseBytes(asResponse bool, b []byte) error { return h.parse(asResponse) } -// Parse parses accumulated data in-place with no copying. One can set HTTP header data buffer with [Header.Reset]. +// Parse parses accumulated data in-place with no copying. One can set HTTP header data buffer with [HeaderV1.Reset]. // It fails if HTTP data is incomplete. -func (h *Header) Parse(asResponse bool) error { +func (h *HeaderV1) Parse(asResponse bool) error { debuglog("http:parse:reset") h.Reset(h.hbuf.kv.buf, 0) debuglog("http:parse:start") @@ -93,7 +94,7 @@ func (h *Header) Parse(asResponse bool) error { // TryParse begins parsing or resumes parsing from a failed previous attempt from any of the Parse* methods. // As long as needMoreData returns true future calls to TryParse may succeed and the header is not done parsing. -// Users may call [Header.ForEach] in-between TryParse calls so as to validate values before header is completely parsed. +// Users may call [HeaderV1.ForEach] in-between TryParse calls so as to validate values before header is completely parsed. // // needMoreData := true // var err error @@ -107,7 +108,7 @@ func (h *Header) Parse(asResponse bool) error { // if err != nil { // return err // } -func (h *Header) TryParse(asResponse bool) (needMoreData bool, err error) { +func (h *HeaderV1) TryParse(asResponse bool) (needMoreData bool, err error) { flags := h.Flags() if flags.HasAny(flagDoneParsingHeader) { return false, errAlreadyParsed @@ -125,26 +126,26 @@ func (h *Header) TryParse(asResponse bool) (needMoreData bool, err error) { } // ParsingSuccess returns true if TryParse was successful, that is to say it returned needMoreData==false and err==nil. -func (h *Header) ParsingSuccess() bool { +func (h *HeaderV1) ParsingSuccess() bool { return h.Flags().HasAny(flagDoneParsingHeader) } // ReadFromLimited reads at most maxBytesToRead from reader and appends them to underlying buffer. -// Used to accumulate HTTP header for later parsing with [Header.TryParse]. +// Used to accumulate HTTP header for later parsing with [HeaderV1.TryParse]. // If read is successful (read length>0) and reader returns [io.EOF] then ReadFromLimited will return a nil error. -func (h *Header) ReadFromLimited(r io.Reader, maxBytesToRead int) (int, error) { +func (h *HeaderV1) ReadFromLimited(r io.Reader, maxBytesToRead int) (int, error) { return h.hbuf.kv.ReadLimited(r, maxBytesToRead) } // ReadFromBytes appends argument buffer to underlying buffer. -// Used to accumulate HTTP header for later parsing with [Header.TryParse]. -func (h *Header) ReadFromBytes(b []byte) error { +// Used to accumulate HTTP header for later parsing with [HeaderV1.TryParse]. +func (h *HeaderV1) ReadFromBytes(b []byte) error { return h.hbuf.kv.ReadFromBytes(b) } // BufferReceived returns the amoung of bytes read during calls to Read* methods. // Returns 0 if buffer is invalid/mangled. -func (h *Header) BufferReceived() int { +func (h *HeaderV1) BufferReceived() int { if h.Flags().HasAny(flagMangledBuffer | flagOOMReached) { return 0 } @@ -154,7 +155,7 @@ func (h *Header) BufferReceived() int { // BufferParsed returns the amount of bytes parsed during a call to Parse* methods. // If the Parse* method completed without error then BufferParsed returns the header's length including the final "\r\n\r\n" text. // BufferParsed returns 0 if the buffer is invalid/mangled or if no header data has been parsed succesfully. -func (h *Header) BufferParsed() int { +func (h *HeaderV1) BufferParsed() int { if h.Flags().HasAny(flagMangledBuffer | flagOOMReached) { return 0 } @@ -162,56 +163,56 @@ func (h *Header) BufferParsed() int { } // BufferRaw returns the undeerlying buffer as stored currently in memory. -// The length of the returned buffer is the used portion. Capacity of returned slice is [Header.BufferCapacity]. -func (h *Header) BufferRaw() []byte { return h.hbuf.kv.BufferRaw() } +// The length of the returned buffer is the used portion. Capacity of returned slice is [HeaderV1.BufferCapacity]. +func (h *HeaderV1) BufferRaw() []byte { return h.hbuf.kv.BufferRaw() } // BufferUsed returns the raw memory used. // // BufferUsed + BufferFree == BufferCapacity -func (h *Header) BufferUsed() int { +func (h *HeaderV1) BufferUsed() int { return len(h.hbuf.kv.BufferRaw()) } // BufferFree returns amount of bytes free in underlying buffer. // // BufferUsed + BufferFree == BufferCapacity -func (h *Header) BufferFree() int { +func (h *HeaderV1) BufferFree() int { return h.hbuf.free() } // BufferCapacity returns the total capacity of the underlying buffer. // // BufferUsed + BufferFree == BufferCapacity -func (h *Header) BufferCapacity() int { +func (h *HeaderV1) BufferCapacity() int { return cap(h.hbuf.kv.BufferRaw()) } // ForEach iterates over header key-value field tuples. -func (h *Header) ForEach(cb func(key, value []byte) bool) { +func (h *HeaderV1) ForEach(cb func(key, value []byte) bool) { h.hbuf.kv.ForEach(cb) } // Reset discards all parsed data and sets the buffer data to buf. This method -// can be used to avoid copying and growing buffers. Call [Header.Parse] after setting buffer +// can be used to avoid copying and growing buffers. Call [HeaderV1.Parse] after setting buffer // data with Reset to parse data in-place. // If buf is nil then the current buffer is reused. There are 3 ways to use Reset: // // h.Reset(prealloc[:0], 16); h.ParseBytes(httpHeader) // Tell header to use a pre-allocated buffer capacity. // h.Reset(httpHeader, 16); h.Parse() // Parse bytes in place with no copying. // h.Reset(nil) // Reuse buffer previously set in a call to Reset. -func (h *Header) Reset(buf []byte, numHeaderCapacity int) { +func (h *HeaderV1) Reset(buf []byte, numHeaderCapacity int) { const persistentFlags = flagNoBufferGrow debuglog("http:reset:hbuf") h.hbuf.reset(buf, numHeaderCapacity) if h.Flags().HasAny(flagNoBufferGrow) && h.BufferCapacity() < 32 { panic("small buffer and flagNoBufferGrow set") } - *h = Header{hbuf: h.hbuf} + *h = HeaderV1{hbuf: h.hbuf} debuglog("http:reset:done") } // Body returns the surplus data following headers. It is only valid as long as Parse* or Reset methods are not called. -func (h *Header) Body() ([]byte, error) { +func (h *HeaderV1) Body() ([]byte, error) { debuglog("http:body") flags := h.Flags() if flags.HasAny(flagMangledBuffer) { @@ -222,16 +223,16 @@ func (h *Header) Body() ([]byte, error) { return nil, errUnparsed } -// SetBytes is equivalent to [Header.Set] but with a []byte value. Does not keep reference to value slice. +// SetBytes is equivalent to [HeaderV1.Set] but with a []byte value. Does not keep reference to value slice. // Calling SetBytes Mangles the buffer. -func (h *Header) SetBytes(key string, value []byte) { +func (h *HeaderV1) SetBytes(key string, value []byte) { h.Set(key, b2s(value)) } -// SetInt is equivalent to [Header.Set] but with an integer value i.e: Content-Length header key. +// SetInt is equivalent to [HeaderV1.Set] but with an integer value i.e: Content-Length header key. // base must be in the range 2..36 (as accepted by [strconv.AppendInt]); other bases are dropped. // SetInt formats the value directly into the header buffer without heap allocation. -func (h *Header) SetInt(key string, value int64, base int) { +func (h *HeaderV1) SetInt(key string, value int64, base int) { if base < 2 || base > 36 { return // strconv.AppendInt only supports base 2..36. } @@ -240,25 +241,25 @@ func (h *Header) SetInt(key string, value int64, base int) { // Set sets a key-value pair in the HTTP header. // Calling Set mangles the buffer. -func (h *Header) Set(key, value string) (enoughSpace bool) { +func (h *HeaderV1) Set(key, value string) (enoughSpace bool) { return h.hbuf.kv.Set(key, value) } -// Get gets the first exact-match value of a key found in the headers. Use [Header.ForEach] to find multiple values corresponding to same key. -func (h *Header) Get(key string) []byte { +// Get gets the first exact-match value of a key found in the headers. Use [HeaderV1.ForEach] to find multiple values corresponding to same key. +func (h *HeaderV1) Get(key string) []byte { return h.hbuf.kv.Get(key) } // GetFold gets the first value whose key matches key under ASCII case-insensitive // comparison, i.e: "content-length" matches "Content-Length". -// Use [Header.Get] for exact match and [Header.ForEach] to find multiple values +// Use [HeaderV1.Get] for exact match and [HeaderV1.ForEach] to find multiple values // corresponding to same key. -func (h *Header) GetFold(key string) []byte { +func (h *HeaderV1) GetFold(key string) []byte { return h.hbuf.kv.GetFold(key) } // NormalizeKeys normalizes all header keys. i.e: CONTENT-type -> Content-Type -func (h *Header) NormalizeKeys() { +func (h *HeaderV1) NormalizeKeys() { for i, kv := range h.hbuf.kv.kvs { if kv.isValidHeader() { NormalizeHeaderKey(h.hbuf.kv.AtKey(i)) @@ -268,7 +269,7 @@ func (h *Header) NormalizeKeys() { // ContentLength returns the body length declared by the Content-Length field. // If the field is not present then the returned bool is false. Will return error for invalid or non-integer value. -func (h *Header) ContentLength() (_ int64, present bool, _ error) { +func (h *HeaderV1) ContentLength() (_ int64, present bool, _ error) { value := h.GetFold(headerContentLength) if value == nil { return 0, false, nil @@ -283,34 +284,34 @@ func (h *Header) ContentLength() (_ int64, present bool, _ error) { } // Add adds a new key-value pair to the HTTP header. Calling Add mangles the buffer. -func (h *Header) Add(key, value string) { +func (h *HeaderV1) Add(key, value string) { h.hbuf.kv.appendPair(key, value) } // Method returns HTTP request method. -func (h *Header) Method() []byte { +func (h *HeaderV1) Method() []byte { return h.getNonEmptyValue(h.method) } // SetMethod sets the request header's method. -func (h *Header) SetMethod(method string) { +func (h *HeaderV1) SetMethod(method string) { h.method = h.hbuf.kv.reuseOrAppend(h.method, method) } // SetRequestTarget sets request-target (URI) for the first HTTP request line. -func (h *Header) SetRequestTarget(requestTarget string) { +func (h *HeaderV1) SetRequestTarget(requestTarget string) { h.requestTarget = h.hbuf.kv.reuseOrAppend(h.requestTarget, requestTarget) } // RequestTarget returns a view of the request-target (URI) of the first HTTP request line. // Called Request-URI in the obsolete RFC 2616, renamed request-target by RFC 9112. -func (h *Header) RequestTarget() []byte { +func (h *HeaderV1) RequestTarget() []byte { return h.getNonEmptyValue(h.requestTarget) } // RequestPath returns the request-target (URI) up to the query string, i.e: "/search" // for "/search?q=go". Returns the whole target if it contains no query string. -func (h *Header) RequestPath() []byte { +func (h *HeaderV1) RequestPath() []byte { target := h.RequestTarget() before, _, ok := bytes.Cut(target, []byte{'?'}) if !ok { @@ -322,7 +323,7 @@ func (h *Header) RequestPath() []byte { // RequestQuery returns the request-target (URI) query string as it appears on the // wire, percent-encoded and with '+' undecoded, i.e: "q=go" for "/search?q=go". // Returns nil if the target has no query string. Iterate it with [NextQueryPair]. -func (h *Header) RequestQuery() []byte { +func (h *HeaderV1) RequestQuery() []byte { target := h.RequestTarget() _, after, ok := bytes.Cut(target, []byte{'?'}) if !ok { @@ -332,17 +333,17 @@ func (h *Header) RequestQuery() []byte { } // Protocol returns the request header's HTTP protocol. Usually "HTTP/1.1". -func (h *Header) Protocol() []byte { +func (h *HeaderV1) Protocol() []byte { return h.getNonEmptyValue(h.proto) } // SetProtocol sets the request header's protocol. Usually "HTTP/1.1". -func (h *Header) SetProtocol(protocol string) { +func (h *HeaderV1) SetProtocol(protocol string) { h.proto = h.hbuf.kv.reuseOrAppend(h.proto, protocol) } // Status returns the response header's status code and status text. i.e: "200" "OK". -func (h *Header) Status() (code, statusText []byte) { +func (h *HeaderV1) Status() (code, statusText []byte) { if h.statusCode.len == 0 { return nil, nil } @@ -350,20 +351,20 @@ func (h *Header) Status() (code, statusText []byte) { } // SetStatus sets the response header's status code and status text. i.e: "200" "OK". -func (h *Header) SetStatus(code, statusText string) { +func (h *HeaderV1) SetStatus(code, statusText string) { h.hbuf.kv.flags |= FlagStatusSet h.statusCode = h.hbuf.kv.reuseOrAppend(h.statusCode, code) h.statusText = h.hbuf.kv.reuseOrAppend(h.statusText, statusText) } -// SetStatusInt is identical to [Header.SetStatus] but performs integer to text conversion for status code. -func (h *Header) SetStatusInt(code int64, statusText string) { +// SetStatusInt is identical to [HeaderV1.SetStatus] but performs integer to text conversion for status code. +func (h *HeaderV1) SetStatusInt(code int64, statusText string) { h.hbuf.kv.flags |= FlagStatusSet h.statusCode = h.hbuf.kv.reuseOrAppendInt(h.statusCode, code, 10) h.statusText = h.hbuf.kv.reuseOrAppend(h.statusText, statusText) } -func (h *Header) getNonEmptyValue(s view) []byte { +func (h *HeaderV1) getNonEmptyValue(s view) []byte { if s.len == 0 { return nil // If empty then value is invalid, return nil. } @@ -371,7 +372,7 @@ func (h *Header) getNonEmptyValue(s view) []byte { } // AppendRequest appends the request header representation to the buffer and returns the result. -func (h *Header) AppendRequest(dst []byte) ([]byte, error) { +func (h *HeaderV1) AppendRequest(dst []byte) ([]byte, error) { proto := h.Protocol() if h.hbuf.kv.flags.HasAny(flagOOMReached) { return dst, ErrBufferExhausted @@ -401,7 +402,7 @@ func (h *Header) AppendRequest(dst []byte) ([]byte, error) { } // AppendResponse appends the response header representation to the buffer and returns the result. -func (h *Header) AppendResponse(dst []byte) ([]byte, error) { +func (h *HeaderV1) AppendResponse(dst []byte) ([]byte, error) { dst, err := h.AppendResponseNoHeaders(dst) if err != nil { return dst, err @@ -411,7 +412,7 @@ func (h *Header) AppendResponse(dst []byte) ([]byte, error) { } // AppendResponseNoHeaders appends the first line of the response containing protocol and status code/text: i.e: "HTTP/1.1 200 OK\r\n" -func (h *Header) AppendResponseNoHeaders(dst []byte) ([]byte, error) { +func (h *HeaderV1) AppendResponseNoHeaders(dst []byte) ([]byte, error) { proto := h.Protocol() if h.hbuf.kv.flags.HasAny(flagOOMReached) { return dst, ErrBufferExhausted @@ -433,7 +434,7 @@ func (h *Header) AppendResponseNoHeaders(dst []byte) ([]byte, error) { // AppendHeaders appends headers to buffer. Use AppendRequest and AppendResponse over this. // Does not append extra \r\n to end. Appends nothing if contains no headers. -func (h *Header) AppendHeaders(dst []byte) []byte { +func (h *HeaderV1) AppendHeaders(dst []byte) []byte { for i, kv := range h.hbuf.kv.kvs { if kv.isValidHeader() { k, v := h.hbuf.kv.At(i) @@ -446,7 +447,7 @@ func (h *Header) AppendHeaders(dst []byte) []byte { // String returns the header's wire representation, as a request if it has a // request line and as a response otherwise. Returns the error text if neither // can be built. Allocates, so it is meant for debugging and logging only. -func (h *Header) String() string { +func (h *HeaderV1) String() string { buf, err := h.AppendRequest(nil) if err != nil { buf, err = h.AppendResponse(nil) diff --git a/http/httpraw/header_test.go b/http/httpraw/headerv1_test.go similarity index 95% rename from http/httpraw/header_test.go rename to http/httpraw/headerv1_test.go index 449f494..8948613 100644 --- a/http/httpraw/header_test.go +++ b/http/httpraw/headerv1_test.go @@ -11,7 +11,11 @@ import ( "time" ) -const numHeaderCapacity = 16 +// defaultKVCap is the key/value table size tests hand to Reset. A Reset with 0 +// preserves whatever capacity the value already had, which is what production +// code wants on reuse but leaves a fresh value unable to hold a single pair when +// growth is disabled, see [Form.Reset]. +const defaultKVCap = 16 func TestHeaderParseRequest(t *testing.T) { const ( @@ -40,7 +44,7 @@ func TestHeaderParseRequest(t *testing.T) { var buf bytes.Buffer req.Write(&buf) - var hdr Header + var hdr HeaderV1 msg := buf.Bytes() start := time.Now() @@ -62,7 +66,7 @@ func TestHeaderParseRequest(t *testing.T) { } var c Cookie cookie := hdr.Get("Cookie") - c.Reset(cookie, 0) + c.Reset(cookie, defaultKVCap) err = c.Parse() if err != nil { t.Error(err) @@ -124,7 +128,7 @@ func BenchmarkParseBytes(b *testing.B) { // allocating on every iteration. Declaring it inside the loop causes // two allocs per iteration: one for the headers slice (make in reset) // and one for the data buffer (append in readFromBytes). - var hdr Header + var hdr HeaderV1 b.StartTimer() for b.Loop() { @@ -165,7 +169,7 @@ func TestHeaderRequestPath(t *testing.T) { {uri: "/a/b/c?x=1&y=2", want: "/a/b/c"}, {uri: "/?q=go", want: "/"}, } { - var h Header + var h HeaderV1 err := h.ParseBytes(false, []byte("GET "+test.uri+" HTTP/1.1\r\nHost: h\r\n\r\n")) if err != nil { t.Fatal(err) @@ -192,7 +196,7 @@ func TestHeaderContentLength(t *testing.T) { {field: "Content-Length: 1 2", wantErr: errBadContentLength}, // Not a list. {field: "Content-Length: 9223372036854775808", wantErr: errBadContentLength}, } { - var h Header + var h HeaderV1 raw := "POST / HTTP/1.1\r\nHost: h\r\n" if test.field != "" { raw += test.field + "\r\n" @@ -230,7 +234,7 @@ func TestNextQueryPair(t *testing.T) { {uri: "/x?a%20b=c%20d", want: "a%20b=c%20d"}, // Raw, undecoded. {uri: "/x?a=b=c", want: "a=b=c"}, // Only first '=' splits. } { - var h Header + var h HeaderV1 err := h.ParseBytes(false, []byte("GET "+test.uri+" HTTP/1.1\r\nHost: h\r\n\r\n")) if err != nil { t.Fatal(err) @@ -393,8 +397,8 @@ func TestCopyDecodedPercentURLInPlace(t *testing.T) { } func TestHeaderSetOverwrite(t *testing.T) { - var h Header - h.Reset(nil, numHeaderCapacity) + var h HeaderV1 + h.Reset(nil, defaultKVCap) h.SetMethod("GET") h.SetRequestTarget("/") h.SetProtocol("HTTP/1.1") @@ -416,8 +420,8 @@ func TestHeaderSetOverwrite(t *testing.T) { } func TestHeaderSetBytesEmptyValue(t *testing.T) { - var h Header - h.Reset(nil, numHeaderCapacity) + var h HeaderV1 + h.Reset(nil, defaultKVCap) h.SetBytes("X-Empty", nil) if got := h.Get("X-Empty"); len(got) != 0 { t.Errorf("want empty value, got %q", got) @@ -436,7 +440,7 @@ func TestHeader_LargeBufferOverflow(t *testing.T) { "X-Canary: " + wantVal + "\r\n" + "\r\n" - var h Header + var h HeaderV1 err := h.ParseBytes(false, []byte(raw)) if err != nil { // Clean rejection of the oversized header is the intended behavior: @@ -454,7 +458,7 @@ func TestHeader_LargeBufferOverflow(t *testing.T) { // not ErrNeedMoreData (which makes a streaming parser wait forever). func TestHeader_ColonlessLineIsHardError(t *testing.T) { raw := "GET / HTTP/1.1\r\nBadHeaderNoColon\r\n\r\n" - var h Header + var h HeaderV1 err := h.ParseBytes(false, []byte(raw)) if err == nil { t.Fatal("want error on colonless header line, got nil") @@ -471,8 +475,8 @@ func TestHeader_SplitBeforeColonStillParses(t *testing.T) { const part1 = "GET / HTTP/1.1\r\nHost" // split mid-key, before colon+newline const part2 = ": example.com\r\n\r\n" - var h Header - h.Reset(nil, numHeaderCapacity) + var h HeaderV1 + h.Reset(nil, defaultKVCap) if err := h.ReadFromBytes([]byte(part1)); err != nil { t.Fatal(err) } @@ -504,8 +508,8 @@ func TestHeader_SplitBeforeColonStillParses(t *testing.T) { func TestHeader_AppendHeaderExactCapNoPanic(t *testing.T) { const key, value = "K", "V" buf := make([]byte, 0, len(key)+len(value)) // exact cap, no slack. - var h Header - h.Reset(buf, numHeaderCapacity) + var h HeaderV1 + h.Reset(buf, defaultKVCap) defer func() { if r := recover(); r != nil { t.Fatalf("appendHeader panicked on exact-cap buffer: %v", r) @@ -521,8 +525,8 @@ func TestHeader_AppendHeaderExactCapNoPanic(t *testing.T) { // never panic. Panicking is unacceptable for this package. func TestHeader_AddFullBufferNoPanic(t *testing.T) { buf := make([]byte, 0, 40) // Small cap; enough for Reset (len 0) but not the field below. - var h Header - h.Reset(buf, numHeaderCapacity) + var h HeaderV1 + h.Reset(buf, defaultKVCap) h.ConfigBufferGrowth(false) h.SetMethod("GET") h.SetRequestTarget("/") @@ -556,8 +560,8 @@ func TestHeader_SetInt(t *testing.T) { {"hex", 255, 16, "ff"}, } { t.Run(tc.name, func(t *testing.T) { - var h Header - h.Reset(nil, numHeaderCapacity) + var h HeaderV1 + h.Reset(nil, defaultKVCap) h.SetInt("Content-Length", tc.value, tc.base) if got := string(h.Get("Content-Length")); got != tc.want { t.Fatalf("want %q, got %q", tc.want, got) @@ -568,8 +572,8 @@ func TestHeader_SetInt(t *testing.T) { // SetInt on an existing key must reuse the slot in place (single field, latest value). func TestHeader_SetIntOverwrite(t *testing.T) { - var h Header - h.Reset(nil, numHeaderCapacity) + var h HeaderV1 + h.Reset(nil, defaultKVCap) h.SetMethod("GET") h.SetRequestTarget("/") h.SetProtocol("HTTP/1.1") @@ -592,8 +596,8 @@ func TestHeader_SetIntOverwrite(t *testing.T) { // SetInt must not heap-allocate: it must format directly into the header buffer. func TestHeader_SetIntNoAlloc(t *testing.T) { buf := make([]byte, 0, 256) - var h Header - h.Reset(buf, numHeaderCapacity) + var h HeaderV1 + h.Reset(buf, defaultKVCap) h.ConfigBufferGrowth(false) h.Add("Content-Length", "0000000000000000000000") // pre-size a reusable slot. allocs := testing.AllocsPerRun(100, func() { @@ -622,8 +626,8 @@ func TestHeader_FieldTableSizedFromBuffer(t *testing.T) { } raw.WriteString("X-Canary: " + wantVal + "\r\n\r\n") - var h Header - h.Reset(make([]byte, 0, 8192), numHeaderCapacity) // Room for the block with plenty to spare. + var h HeaderV1 + h.Reset(make([]byte, 0, 8192), defaultKVCap) // Room for the block with plenty to spare. err := h.ParseBytes(false, []byte(raw.String())) if err != nil { t.Fatalf("parsing a 42 field request into an 8kB buffer: %s", err) @@ -645,8 +649,8 @@ func TestHeader_FieldTableFullIsReported(t *testing.T) { raw.WriteString(":v\r\n") } raw.WriteString("\r\n") - var h Header - h.Reset(make([]byte, 0, 512), numHeaderCapacity) + var h HeaderV1 + h.Reset(make([]byte, 0, 512), defaultKVCap) h.ConfigBufferGrowth(false) err := h.ParseBytes(false, []byte(raw.String())) if !errors.Is(err, ErrHeaderTooMany) { diff --git a/http/httpraw/kvbuffer.go b/http/httpraw/kvbuffer.go index b709c77..0072db6 100644 --- a/http/httpraw/kvbuffer.go +++ b/http/httpraw/kvbuffer.go @@ -23,6 +23,11 @@ func (kvb *kvBuffer) free() int { return cap(kvb.buf) - len(kvb.buf) } // Stored pairs alias it, so writing to it mangles them. func (kvb *kvBuffer) BufferRaw() []byte { return kvb.buf } +// BufferUsed returns the raw memory used, which is what a caller appending from +// several sources checks to know whether a separator is needed. Counts buffered +// bytes and not parsed pairs, so it is set before a Parse and unchanged by one. +func (kvb *kvBuffer) BufferUsed() int { return len(kvb.buf) } + // EnableBufferGrowth allows the buffer to grow past the memory [kvBuffer.Reset] // was handed. The setting outlives Reset; with growth off callers get [ErrBufferExhausted]. func (kvb *kvBuffer) EnableBufferGrowth(enableGrowth bool) { @@ -285,17 +290,17 @@ func (kvb *kvBuffer) getIdx(key string) int { func (kvb *kvBuffer) getFoldIdx(key string) int { for i, pair := range kvb.kvs { - if pair.isValid() && asciiEqualFold(key, b2s(kvb.AtKey(i))) { + if pair.isValid() && EqualFoldASCII(key, b2s(kvb.AtKey(i))) { return i } } return -1 } -// asciiEqualFold reports whether a and b are equal under ASCII case folding. +// EqualFoldASCII reports whether a and b are equal under ASCII case folding. // Unlike strings.EqualFold it does not fold non-ASCII runes, so no multi-byte // rune such as U+212A KELVIN SIGN can alias a header key. -func asciiEqualFold(a, b string) bool { +func EqualFoldASCII(a, b string) bool { if len(a) != len(b) { return false } @@ -463,7 +468,7 @@ func (pair pairKV) isValid() bool { return pair.key.len > 0 || pair.value.len > 0 } -// isValidHeader is for the append-built [Header] store, where mustAppendSlice +// isValidHeader is for the append-built [HeaderV1] store, where mustAppendSlice // burns byte 0 so a zero offset means absent. Drops offset-0 pairs otherwise. func (pair pairKV) isValidHeader() bool { return pair.key.start > 0 } diff --git a/http/httpraw/parse.go b/http/httpraw/parsev1.go similarity index 83% rename from http/httpraw/parse.go rename to http/httpraw/parsev1.go index 4016fba..5c0202a 100644 --- a/http/httpraw/parse.go +++ b/http/httpraw/parsev1.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" + "github.com/soypat/lneto" "github.com/soypat/lneto/internal" ) @@ -48,7 +49,11 @@ var ( // returning the wrong bytes or panicking on a wrapped slice bound. const maxBufLen = 0xffff -type headerBuf struct { +func protoIsV1(s string) bool { + return s[:min(len(strHTTP1), len(s))] == strHTTP1 +} + +type headerv1Buf struct { kv kvBuffer // buf[:len] holds entire HTTP header data, which may be normalized by [flags]. buf[off:len] holds data not yet processed during parsing. // buf []byte @@ -61,7 +66,7 @@ type headerBuf struct { // reset sets the buffer data and discards all parsed data. The field table is // grown to match the new buffer's capacity and never shrinks, so a header // reused across requests settles on its largest buffer and stops allocating. -func (h *headerBuf) reset(buf []byte, numHeaderCapacity int) { +func (h *headerv1Buf) reset(buf []byte, numHeaderCapacity int) { h.kv.Reset(buf, numHeaderCapacity) h.off = 0 } @@ -80,7 +85,7 @@ type scannerState struct { initialized bool } -func (h *Header) parse(asResponse bool) (err error) { +func (h *HeaderV1) parse(asResponse bool) (err error) { debuglog("http:firstline:start") err = h.parseFirstLine(asResponse) if err != nil { @@ -93,7 +98,7 @@ func (h *Header) parse(asResponse bool) (err error) { return err } -func (h *Header) parseFirstLine(asResponse bool) (err error) { +func (h *HeaderV1) parseFirstLine(asResponse bool) (err error) { if len(h.hbuf.kv.buf) > maxBufLen { return errBufferTooLarge // Offsets would overflow uint16 tokint. } @@ -107,7 +112,7 @@ func (h *Header) parseFirstLine(asResponse bool) (err error) { return err } -func (h *Header) parseNextHeaders(flags Flags) error { +func (h *HeaderV1) parseNextHeaders(flags Flags) error { var ss scannerState h.hbuf.parseNextHeaders(&ss, flags) if ss.err != nil { @@ -118,9 +123,9 @@ func (h *Header) parseNextHeaders(flags Flags) error { return nil } -func (hb *headerBuf) free() int { return hb.kv.free() } +func (hb *headerv1Buf) free() int { return hb.kv.free() } -func (hb *headerBuf) parseNextHeaders(ss *scannerState, flags Flags) { +func (hb *headerv1Buf) parseNextHeaders(ss *scannerState, flags Flags) { debuglog("http:nexthdr:loop") for kv := hb.next(ss); kv.isValidHeader(); kv = hb.next(ss) { if !hb.kv.canAddOneKV() { @@ -132,17 +137,17 @@ func (hb *headerBuf) parseNextHeaders(ss *scannerState, flags Flags) { debuglog("http:nexthdr:done") } -func (hb *headerBuf) offBuf() []byte { +func (hb *headerv1Buf) offBuf() []byte { return hb.kv.buf[hb.off:] } -func (hb *headerBuf) skipLeadingCRLF() { +func (hb *headerv1Buf) skipLeadingCRLF() { for hb.off < len(hb.kv.buf) && (hb.kv.buf[hb.off] == '\n' || hb.kv.buf[hb.off] == '\r') { hb.off++ } } -func (hb *headerBuf) scanLine() []byte { +func (hb *headerv1Buf) scanLine() []byte { buf := hb.scanUntilByte('\n') if len(buf) > 0 && buf[len(buf)-1] == '\r' { buf = buf[:len(buf)-1] // exclude carriage return. @@ -153,7 +158,7 @@ func (hb *headerBuf) scanLine() []byte { return buf } -func (hb *headerBuf) scanUntilByte(c byte) []byte { +func (hb *headerv1Buf) scanUntilByte(c byte) []byte { buf := hb.offBuf() idx := bytes.IndexByte(buf, c) if idx >= 0 { @@ -163,7 +168,7 @@ func (hb *headerBuf) scanUntilByte(c byte) []byte { return buf } -func (hb *headerBuf) parseFirstLineRequest(initFlags Flags) (method, uri, proto view, flags Flags, err error) { +func (hb *headerv1Buf) parseFirstLineRequest(initFlags Flags) (method, uri, proto view, flags Flags, err error) { debuglog("http:req:scan") hb.off = 0 // Parsing first line resets offset. hb.skipLeadingCRLF() @@ -183,21 +188,30 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags Flags) (method, uri, proto reqURIEnd += methodEnd + 1 uri = hb.kv.view(b[methodEnd+1 : reqURIEnd]) proto = hb.kv.view(b[reqURIEnd+1:]) // Skip space before protocol. - if b2s(b[reqURIEnd+1:]) != strHTTP11 { - flags |= flagNoHTTP11 + protoText := b2s(b[reqURIEnd+1:]) + if !protoIsV1(protoText) { + // Refused here rather than after the fields: the field loop is nearly + // all of the parse cost and none of it serves a version this type + // does not speak. proto is set so the caller can name it, i.e: 505. + method = hb.kv.view(b[:methodEnd]) + return method, uri, proto, flags | flagNoHTTP11, lneto.ErrUnsupported + } else if protoText != strHTTP11 { + flags |= flagNoHTTP11 // HTTP/1.0, which defaults to closing the connection. } } else if reqURIEnd == 0 { return method, uri, proto, flags, errEmptyURI } else { - // No version provided. - flags |= flagNoHTTP11 + // No version at all is a HTTP/0.9 simple-request, not a 1.x request-line, + // RFC 9112 3. proto stays empty, telling it apart from a named version. uri = hb.kv.view(b[methodEnd+1:]) + method = hb.kv.view(b[:methodEnd]) + return method, uri, proto, flags | flagNoHTTP11, lneto.ErrUnsupported } method = hb.kv.view(b[:methodEnd]) return method, uri, proto, flags, nil } -func (hb *headerBuf) parseFirstLineResponse(initFlags Flags) (statusCode, statusText view, flags Flags, err error) { +func (hb *headerv1Buf) parseFirstLineResponse(initFlags Flags) (statusCode, statusText view, flags Flags, err error) { debuglog("http:resp:scan") hb.off = 0 // Parsing first line resets offset. hb.skipLeadingCRLF() @@ -216,7 +230,11 @@ func (hb *headerBuf) parseFirstLineResponse(initFlags Flags) (statusCode, status if protoEnd < 0 { return statusCode, statusText, flags, ErrNeedMoreData } - if b2s(b[:protoEnd]) != strHTTP11 { + if !protoIsV1(b2s(b[:protoEnd])) { + // Refused before the fields, as on the request side: a response naming + // another version is not one this type can read. + return statusCode, statusText, flags | flagNoHTTP11, lneto.ErrUnsupported + } else if b2s(b[:protoEnd]) != strHTTP11 { flags |= flagNoHTTP11 } b = b[protoEnd+1:] // Advance past protocol and space. @@ -243,7 +261,7 @@ func (hb *headerBuf) parseFirstLineResponse(initFlags Flags) (statusCode, status return statusCode, statusText, flags, nil } -func (hb *headerBuf) next(ss *scannerState) pairKV { +func (hb *headerv1Buf) next(ss *scannerState) pairKV { if !ss.initialized { ss.nextColon = -1 ss.nextNewLine = -1 @@ -328,7 +346,7 @@ func (hb *headerBuf) next(ss *scannerState) pairKV { } // ConnectionClose returns true if 'Connection: close' header is set or if a invalid header was found. -func (h *Header) ConnectionClose() bool { +func (h *HeaderV1) ConnectionClose() bool { flags := h.Flags() closed := flags.HasAny(flagConnClose) || h.hasConnectionToken(strClose) || @@ -342,7 +360,7 @@ func (h *Header) ConnectionClose() bool { // hasConnectionToken reports whether the Connection field lists token, which // must be lowercase. The field name, its comma list and each token all compare // case insensitively, RFC 9110 5.1 and 7.6.1. -func (h *Header) hasConnectionToken(token string) bool { +func (h *HeaderV1) hasConnectionToken(token string) bool { value := h.GetFold(headerConnection) for len(value) > 0 { item := value diff --git a/http/httpraw/parse_test.go b/http/httpraw/parsev1_test.go similarity index 80% rename from http/httpraw/parse_test.go rename to http/httpraw/parsev1_test.go index 0a3160e..d4e2d59 100644 --- a/http/httpraw/parse_test.go +++ b/http/httpraw/parsev1_test.go @@ -2,15 +2,18 @@ package httpraw import ( "bytes" + "errors" "strings" "testing" + + "github.com/soypat/lneto" ) func TestTryParse_IncrementalRequest(t *testing.T) { // Full HTTP request split across multiple ReadFromBytes calls. full := "GET /index.html HTTP/1.1\r\nHost: example.com\r\nContent-Type: text/html\r\n\r\nbody here" - var hdr Header - hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) + var hdr HeaderV1 + hdr.Reset(make([]byte, 0, 256), defaultKVCap) // Feed data in small chunks to exercise incremental parsing. chunks := splitInto(full, 10) @@ -78,8 +81,8 @@ func TestTryParse_IncrementalRequest(t *testing.T) { func TestTryParse_IncrementalResponse(t *testing.T) { full := "HTTP/1.1 200 OK\r\nContent-Length: 5\r\nServer: lneto\r\n\r\nhello" - var hdr Header - hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) + var hdr HeaderV1 + hdr.Reset(make([]byte, 0, 256), defaultKVCap) chunks := splitInto(full, 8) var done bool @@ -130,8 +133,8 @@ func TestReadFromLimited(t *testing.T) { data := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" r := strings.NewReader(data) - var hdr Header - hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) + var hdr HeaderV1 + hdr.Reset(make([]byte, 0, 256), defaultKVCap) // Read in one shot. n, err := hdr.ReadFromLimited(r, 256) @@ -156,8 +159,8 @@ func TestReadFromLimited(t *testing.T) { } func TestReadFromLimited_MaxBytes(t *testing.T) { - var hdr Header - hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) + var hdr HeaderV1 + hdr.Reset(make([]byte, 0, 256), defaultKVCap) // Zero maxBytesToRead should error. _, err := hdr.ReadFromLimited(strings.NewReader("data"), 0) @@ -167,8 +170,8 @@ func TestReadFromLimited_MaxBytes(t *testing.T) { } func TestReadFromBytes_Empty(t *testing.T) { - var hdr Header - hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) + var hdr HeaderV1 + hdr.Reset(make([]byte, 0, 256), defaultKVCap) err := hdr.ReadFromBytes(nil) if err == nil { @@ -177,8 +180,8 @@ func TestReadFromBytes_Empty(t *testing.T) { } func TestBufferFreeAndCapacity(t *testing.T) { - var hdr Header - hdr.Reset(make([]byte, 0, 100), numHeaderCapacity) + var hdr HeaderV1 + hdr.Reset(make([]byte, 0, 100), defaultKVCap) if hdr.BufferCapacity() != 100 { t.Errorf("capacity = %d; want 100", hdr.BufferCapacity()) @@ -194,9 +197,9 @@ func TestBufferFreeAndCapacity(t *testing.T) { } func TestEnableBufferGrowth(t *testing.T) { - var hdr Header + var hdr HeaderV1 buf := make([]byte, 0, 64) - hdr.Reset(buf, numHeaderCapacity) + hdr.Reset(buf, defaultKVCap) hdr.ConfigBufferGrowth(false) // With growth disabled, reading more than capacity should fail. big := make([]byte, 128) @@ -211,7 +214,7 @@ func TestEnableBufferGrowth(t *testing.T) { func TestHeader_Add(t *testing.T) { full := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 err := hdr.ParseBytes(false, []byte(full)) if err != nil { t.Fatal(err) @@ -238,7 +241,7 @@ func TestHeader_Add(t *testing.T) { func TestHeader_SetBytes(t *testing.T) { full := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 err := hdr.ParseBytes(false, []byte(full)) if err != nil { t.Fatal(err) @@ -254,7 +257,7 @@ func TestHeader_SetBytes(t *testing.T) { func TestConnectionClose(t *testing.T) { t.Run("HTTP11_NoConnectionHeader", func(t *testing.T) { full := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 hdr.ParseBytes(false, []byte(full)) if hdr.ConnectionClose() { t.Error("HTTP/1.1 without Connection:close should not close") @@ -263,7 +266,7 @@ func TestConnectionClose(t *testing.T) { t.Run("ExplicitClose", func(t *testing.T) { full := "GET / HTTP/1.1\r\nConnection: close\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 hdr.ParseBytes(false, []byte(full)) if !hdr.ConnectionClose() { t.Error("Connection:close header should trigger close") @@ -272,7 +275,7 @@ func TestConnectionClose(t *testing.T) { t.Run("HTTP10_NoKeepAlive", func(t *testing.T) { full := "GET / HTTP/1.0\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 hdr.ParseBytes(false, []byte(full)) if !hdr.ConnectionClose() { t.Error("HTTP/1.0 without keep-alive should close") @@ -281,7 +284,7 @@ func TestConnectionClose(t *testing.T) { t.Run("HTTP10_KeepAlive", func(t *testing.T) { full := "GET / HTTP/1.0\r\nConnection: keep-alive\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 hdr.ParseBytes(false, []byte(full)) if hdr.ConnectionClose() { t.Error("HTTP/1.0 with keep-alive should not close") @@ -291,7 +294,7 @@ func TestConnectionClose(t *testing.T) { func TestTryParse_AlreadyParsed(t *testing.T) { full := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 hdr.ParseBytes(false, []byte(full)) // Calling TryParse again should return error. @@ -303,7 +306,7 @@ func TestTryParse_AlreadyParsed(t *testing.T) { func TestParseResponse_BadStatusCode(t *testing.T) { full := "HTTP/1.1 abc Bad\r\n\r\n" - var hdr Header + var hdr HeaderV1 err := hdr.ParseBytes(true, []byte(full)) if err == nil { t.Fatal("expected error for non-numeric status code") @@ -370,7 +373,7 @@ func TestCookie_ForEach(t *testing.T) { func TestHeader_MultilineValue(t *testing.T) { // RFC 7230: obsolete line folding with \r\n followed by space/tab. full := "GET / HTTP/1.1\r\nX-Multi: line1\r\n\tline2\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 err := hdr.ParseBytes(false, []byte(full)) if err != nil { t.Fatal(err) @@ -386,8 +389,8 @@ func TestHeader_MultilineValue(t *testing.T) { } func TestHeader_ResponseRoundTrip(t *testing.T) { - var hdr Header - hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) + var hdr HeaderV1 + hdr.Reset(make([]byte, 0, 256), defaultKVCap) hdr.SetProtocol("HTTP/1.1") hdr.SetStatus("404", "Not Found") hdr.Add("Content-Type", "text/plain") @@ -411,7 +414,7 @@ func TestHeader_ResponseRoundTrip(t *testing.T) { } // Parse back the generated response. - var hdr2 Header + var hdr2 HeaderV1 err = hdr2.ParseBytes(true, buf) if err != nil { t.Fatalf("re-parse response: %v", err) @@ -426,8 +429,8 @@ func TestHeader_ResponseRoundTrip(t *testing.T) { } func TestHeader_RequestRoundTrip(t *testing.T) { - var hdr Header - hdr.Reset(make([]byte, 0, 256), numHeaderCapacity) + var hdr HeaderV1 + hdr.Reset(make([]byte, 0, 256), defaultKVCap) hdr.SetProtocol("HTTP/1.1") hdr.SetMethod("POST") hdr.SetRequestTarget("/api/data") @@ -444,7 +447,7 @@ func TestHeader_RequestRoundTrip(t *testing.T) { } // Parse back the generated request. - var hdr2 Header + var hdr2 HeaderV1 err = hdr2.ParseBytes(false, buf) if err != nil { t.Fatalf("re-parse request: %v", err) @@ -463,7 +466,7 @@ func TestHeader_RequestRoundTrip(t *testing.T) { func TestParseResponse_StatusCodeOnly(t *testing.T) { // Response with status code but no status text. full := "HTTP/1.1 204\r\n\r\n" - var hdr Header + var hdr HeaderV1 err := hdr.ParseBytes(true, []byte(full)) if err != nil { t.Fatal(err) @@ -477,7 +480,7 @@ func TestParseResponse_StatusCodeOnly(t *testing.T) { func TestParseResponse_HTTP10(t *testing.T) { full := "HTTP/1.0 200 OK\r\nServer: old\r\n\r\n" - var hdr Header + var hdr HeaderV1 err := hdr.ParseBytes(true, []byte(full)) if err != nil { t.Fatal(err) @@ -492,12 +495,14 @@ func TestParseResponse_HTTP10(t *testing.T) { } func TestParseRequest_NoProtocol(t *testing.T) { - // HTTP/0.9 style: just method and URI, no version. + // HTTP/0.9 style: just method and URI, no version. Refused, but the + // request-line it did read stays readable so a caller can answer 400 and say + // what it saw. full := "GET /simple\r\nHost: test\r\n\r\n" - var hdr Header + var hdr HeaderV1 err := hdr.ParseBytes(false, []byte(full)) - if err != nil { - t.Fatal(err) + if !errors.Is(err, lneto.ErrUnsupported) { + t.Fatalf("want lneto.ErrUnsupported for a version-less request, got %v", err) } if string(hdr.Method()) != "GET" { t.Errorf("method = %q; want GET", hdr.Method()) @@ -505,6 +510,8 @@ func TestParseRequest_NoProtocol(t *testing.T) { if string(hdr.RequestTarget()) != "/simple" { t.Errorf("URI = %q; want /simple", hdr.RequestTarget()) } + // An empty protocol is what tells HTTP/0.9 apart from a named version, which + // is how [httphi] picks 400 over 505. if hdr.Protocol() != nil { t.Errorf("protocol should be nil for version-less request, got %q", hdr.Protocol()) } @@ -521,7 +528,7 @@ func TestCookie_QuotedValue(t *testing.T) { func TestParseRequest_InvalidHeaderSpaceBeforeColon(t *testing.T) { // RFC 7230 ยง3.2.4: No whitespace allowed between header name and colon. full := "GET / HTTP/1.1\r\nBad Header : value\r\n\r\n" - var hdr Header + var hdr HeaderV1 err := hdr.ParseBytes(false, []byte(full)) if err == nil { t.Fatal("expected error for space before colon in header name") @@ -567,7 +574,7 @@ func TestConnectionCloseFolded(t *testing.T) { {proto: "HTTP/1.0", field: "Host: h", wantClose: true}, } { t.Run(test.proto+" "+test.field, func(t *testing.T) { - var hdr Header + var hdr HeaderV1 full := "GET / " + test.proto + "\r\nHost: h\r\n" + test.field + "\r\n\r\n" if err := hdr.ParseBytes(false, []byte(full)); err != nil { t.Fatal(err) @@ -578,3 +585,63 @@ func TestConnectionCloseFolded(t *testing.T) { }) } } + +// HeaderV1 speaks HTTP/1.x and nothing else, so a request or response naming +// another version is refused on the first line. That skips the field loop, which +// is where nearly all the parse cost is, and refuses the h2c preface a modern +// client opens with before it is mistaken for a request. +func TestHeaderV1RejectsUnsupportedVersion(t *testing.T) { + for _, test := range []struct { + name string + raw string + asResponse bool + }{ + {name: "request http2", raw: "GET / HTTP/2.0\r\nHost: h\r\n\r\n"}, + {name: "request http3", raw: "GET / HTTP/3.0\r\nHost: h\r\n\r\n"}, + {name: "h2c preface", raw: "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"}, + {name: "request http09 no version", raw: "GET /index.html\r\nHost: h\r\n\r\n"}, + {name: "request bogus proto", raw: "GET / BANANA\r\nHost: h\r\n\r\n"}, + {name: "response http2", raw: "HTTP/2.0 200 OK\r\nServer: s\r\n\r\n", asResponse: true}, + {name: "response http09", raw: "HTTP/0.9 200 OK\r\nServer: s\r\n\r\n", asResponse: true}, + } { + t.Run(test.name, func(t *testing.T) { + var h HeaderV1 + err := h.ParseBytes(test.asResponse, []byte(test.raw)) + if !errors.Is(err, lneto.ErrUnsupported) { + t.Fatalf("want lneto.ErrUnsupported, got %v", err) + } + // The field loop must not have run: refusing early is the point. + fields := 0 + h.ForEach(func(key, value []byte) bool { fields++; return true }) + if fields != 0 { + t.Errorf("want no fields parsed, got %d", fields) + } + if h.ParsingSuccess() { + t.Error("a refused header must not report a successful parse") + } + }) + } +} + +// Both HTTP/1 versions stay supported: only non-1.x is refused. +func TestHeaderV1AcceptsV1Versions(t *testing.T) { + for _, test := range []struct { + raw string + asResponse bool + }{ + {raw: "GET / HTTP/1.1\r\nHost: h\r\n\r\n"}, + {raw: "GET / HTTP/1.0\r\nHost: h\r\n\r\n"}, + {raw: "HTTP/1.1 200 OK\r\nServer: s\r\n\r\n", asResponse: true}, + {raw: "HTTP/1.0 200 OK\r\nServer: s\r\n\r\n", asResponse: true}, + } { + t.Run(test.raw[:12], func(t *testing.T) { + var h HeaderV1 + if err := h.ParseBytes(test.asResponse, []byte(test.raw)); err != nil { + t.Fatalf("want parsed, got %v", err) + } + if !h.ParsingSuccess() { + t.Error("want a successful parse") + } + }) + } +} diff --git a/internet/pcap/capture.go b/internet/pcap/capture.go index f55abdb..5c8b25e 100644 --- a/internet/pcap/capture.go +++ b/internet/pcap/capture.go @@ -35,7 +35,7 @@ var ( ) type PacketBreakdown struct { - hdr httpraw.Header + hdr httpraw.HeaderV1 dmsg dns.Message vld lneto.Validator // SubfieldLimit will limit the number of captured subfields to the value it has. diff --git a/internet/pcap/capture_test.go b/internet/pcap/capture_test.go index 4ee6bbd..707de5f 100644 --- a/internet/pcap/capture_test.go +++ b/internet/pcap/capture_test.go @@ -23,7 +23,7 @@ import ( const httpProtocol = "HTTP/1.1" func makeHttpPayload(body string) ([]byte, error) { - var hdr httpraw.Header + var hdr httpraw.HeaderV1 hdr.SetProtocol(httpProtocol) hdr.SetStatus("200", "OK") hdr.Set("Cookie", "ABC=123") diff --git a/ipv4/definitions.go b/ipv4/definitions.go index 96c66e0..f42924c 100644 --- a/ipv4/definitions.go +++ b/ipv4/definitions.go @@ -18,6 +18,12 @@ func IsMulticast(addr [4]byte) bool { return addr[0]&0xf0 == 0xe0 } +// UnspecifiedAddr returns the unspecified address: 0.0.0.0 +func UnspecifiedAddr() [4]byte { return [4]byte{0, 0, 0, 0} } + +// BroadcastAddr returns the broadcast address: 255.255.255.255 +func BroadcastAddr() [4]byte { return [4]byte{255, 255, 255, 255} } + // IsBroadcast reports whether addr is the limited broadcast address // 255.255.255.255 used to address all hosts on the local network segment // as defined in [RFC919]. It does not detect directed (subnet) broadcast @@ -26,7 +32,7 @@ func IsMulticast(addr [4]byte) bool { // // [RFC919]: https://datatracker.ietf.org/doc/html/rfc919 func IsBroadcast(addr [4]byte) bool { - return addr == [4]byte{255, 255, 255, 255} + return addr == BroadcastAddr() } // IsLinkLocal reports whether addr is within the IPv4 link-local prefix @@ -113,3 +119,12 @@ func AppendFormatAddr(dst []byte, addr [4]byte) []byte { } return dst } + +// String returns the dotted-decimal text representation of an IPv4 address. +func String(addr [4]byte) string { + // See net/netip's (Addr).string4 pattern. + var buf [maxAddrStringLen]byte + return string(AppendFormatAddr(buf[:0], addr)) +} + +const maxAddrStringLen = len("255.255.255.255") diff --git a/ipv4/definitions_test.go b/ipv4/definitions_test.go index dc6f5af..df90a12 100644 --- a/ipv4/definitions_test.go +++ b/ipv4/definitions_test.go @@ -10,10 +10,10 @@ func TestAppendFormatAddr(t *testing.T) { addr [4]byte want string }{ - {addr: [4]byte{0, 0, 0, 0}, want: "0.0.0.0"}, + {addr: UnspecifiedAddr(), want: "0.0.0.0"}, + {addr: BroadcastAddr(), want: "255.255.255.255"}, {addr: [4]byte{127, 0, 0, 1}, want: "127.0.0.1"}, {addr: [4]byte{192, 168, 1, 1}, want: "192.168.1.1"}, - {addr: [4]byte{255, 255, 255, 255}, want: "255.255.255.255"}, {addr: [4]byte{10, 0, 0, 1}, want: "10.0.0.1"}, {addr: [4]byte{1, 2, 3, 4}, want: "1.2.3.4"}, {addr: [4]byte{100, 99, 9, 0}, want: "100.99.9.0"}, diff --git a/x/xnet/xnet_arp_test.go b/x/xnet/xnet_arp_test.go index 56ddd0e..7dc8aa5 100644 --- a/x/xnet/xnet_arp_test.go +++ b/x/xnet/xnet_arp_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/soypat/lneto/ethernet" + "github.com/soypat/lneto/ipv4" ) func TestARPLocal(t *testing.T) { @@ -20,7 +21,7 @@ func TestARPLocal(t *testing.T) { addr2 := netip.AddrPortFrom(netip.AddrFrom4(s2.Addr4()), 80) // listener, server. err := s1.AssimilateDHCPResults(&DHCPResults{ Router: netip.AddrFrom4([4]byte{10, 0, 0, 255}), - BroadcastAddr: netip.AddrFrom4([4]byte{255, 255, 255, 255}), + BroadcastAddr: netip.AddrFrom4(ipv4.BroadcastAddr()), AssignedAddr4: s1.Addr4(), Subnet: netip.PrefixFrom(netip.AddrFrom4(s2.Addr4()), 24), // Subnet containing s2 will force an ARP on s1. TRenewal: 1000, diff --git a/x/xnet/xnet_fuzz_test.go b/x/xnet/xnet_fuzz_test.go index 9db5503..671da33 100644 --- a/x/xnet/xnet_fuzz_test.go +++ b/x/xnet/xnet_fuzz_test.go @@ -21,7 +21,7 @@ func FuzzStackPacketHTTP(f *testing.F) { const seed = 1 var buf [ethernet.MaxFrameLength]byte s1, s2, c1, c2 := newTCPStacks(f, seed, MTU) - var hdr httpraw.Header + var hdr httpraw.HeaderV1 err := s1.ListenTCP4(c1, 80) if err != nil { f.Fatal(err) @@ -129,7 +129,7 @@ func FuzzStackPacketHTTP(f *testing.F) { if n1 == 0 && n2 == 0 { if !closed { if c1.BufferedInput() > 0 { - var hdr httpraw.Header + var hdr httpraw.HeaderV1 n, _ := c1.Read(buf[:]) hdr.ReadFromBytes(buf[:n]) hdr.TryParse(false)