explicit header key/value alloc and add ExchangeConfig

http/httpraw/header.go

@@ -65,10 +65,12 @@ type Header struct {
 // Flags returns [Flags] to signal status code has been set, Connection:Close or other useful signals provided by flags.
 func (h *Header) Flags() Flags { return h.flags }
 
-// EnableBufferGrowth disables buffer growth during parsing if b is false. Is enabled by default.
-// Disabling buffer growth prevents allocations but methods may throw errors on insufficient memory.
-func (h *Header) EnableBufferGrowth(b bool) {
-	if !b {
+// ConfigBufferGrowth configures the memory the header may use. Setting
+// outlives [Header.Reset]. Call before parsing/reading.
+//
+// enableBufferGrowth enables growing both the header buffer and the header key/value pair slice.
+func (h *Header) ConfigBufferGrowth(enableBufferGrowth bool) {
+	if !enableBufferGrowth {
 		h.flags |= flagNoBufferGrow
 	} else {
 		h.flags &^= flagNoBufferGrow
@@ -77,7 +79,7 @@ func (h *Header) EnableBufferGrowth(b bool) {
 
 // ParseBytes copies the bytes into buffer and parses the HTTP header. It fails if HTTP header data is incomplete.
 func (h *Header) ParseBytes(asResponse bool, b []byte) error {
-	h.Reset(nil)
+	h.Reset(nil, 0)
 	h.hbuf.readFromBytes(b)
 	return h.parse(asResponse)
 }
@@ -86,7 +88,7 @@ func (h *Header) ParseBytes(asResponse bool, b []byte) error {
 // It fails if HTTP data is incomplete.
 func (h *Header) Parse(asResponse bool) error {
 	debuglog("http:parse:reset")
-	h.Reset(h.hbuf.buf)
+	h.Reset(h.hbuf.buf, 0)
 	debuglog("http:parse:start")
 	return h.parse(asResponse)
 }
@@ -119,7 +121,7 @@ func (h *Header) TryParse(asResponse bool) (needMoreData bool, err error) {
 			return err == ErrNeedMoreData, err
 		}
 	}
-	err = h.parseNextHeaders()
+	err = h.parseNextHeaders(h.flags)
 	return err == ErrNeedMoreData, err
 }
 
@@ -133,7 +135,7 @@ func (h *Header) ParsingSuccess() bool {
 // If read is successful (read length>0) and reader returns [io.EOF] then ReadFromLimited will return a nil error.
 func (h *Header) ReadFromLimited(r io.Reader, maxBytesToRead int) (int, error) {
 	if maxBytesToRead <= 0 {
-		return 0, errSmallBuffer
+		return 0, ErrSmallHeaderBuffer
 	} else if h.flags.HasAny(flagMangledBuffer) {
 		return 0, errMangledBuffer
 	} else if h.flags.HasAny(flagReaderEOF) {
@@ -142,14 +144,14 @@ func (h *Header) ReadFromLimited(r io.Reader, maxBytesToRead int) (int, error) {
 	free := h.BufferFree()
 	if free < maxBytesToRead {
 		if h.flags.HasAny(flagNoBufferGrow) {
-			return 0, errSmallBuffer
+			return 0, ErrSmallHeaderBuffer
 		}
 		h.hbuf.buf = slices.Grow(h.hbuf.buf, maxBytesToRead)
 	}
 	blen := len(h.hbuf.buf)
 	b := h.hbuf.buf[blen:min(blen+maxBytesToRead, cap(h.hbuf.buf))]
 	if len(b) == 0 {
-		return 0, errSmallBuffer
+		return 0, ErrSmallHeaderBuffer
 	}
 	n, err := r.Read(b)
 	if err != nil && err == io.EOF {
@@ -166,12 +168,12 @@ func (h *Header) ReadFromLimited(r io.Reader, maxBytesToRead int) (int, error) {
 // Used to accumulate HTTP header for later parsing with [Header.TryParse].
 func (h *Header) ReadFromBytes(b []byte) (int, error) {
 	if len(b) == 0 {
-		return 0, errSmallBuffer
+		return 0, ErrSmallHeaderBuffer
 	}
 	free := h.BufferFree()
 	if free < len(b) {
 		if h.flags.HasAny(flagNoBufferGrow) {
-			return 0, errSmallBuffer
+			return 0, ErrSmallHeaderBuffer
 		}
 		h.hbuf.buf = slices.Grow(h.hbuf.buf, len(b))
 	}
@@ -253,13 +255,13 @@ func (hb *headerBuf) forEach(cb func(key, value []byte) error) error {
 //	h.Reset(prealloc[:0]); h.ParseBytes(httpHeader) // Tell header to use a pre-allocated buffer capacity.
 //	h.Reset(httpHeader); h.Parse() // Parse bytes in place with no copying.
 //	h.Reset(nil) // Reuse buffer previously set in a call to Reset.
-func (h *Header) Reset(buf []byte) {
-	if h.flags.HasAny(flagNoBufferGrow) && cap(buf) < 32 {
-		panic("small buffer and flagNoBufferGrow set")
-	}
+func (h *Header) Reset(buf []byte, numHeaderCapacity int) {
 	const persistentFlags = flagNoBufferGrow
 	debuglog("http:reset:hbuf")
-	h.hbuf.reset(buf)
+	h.hbuf.reset(buf, numHeaderCapacity)
+	if h.flags.HasAny(flagNoBufferGrow) && cap(h.hbuf.buf) < 32 {
+		panic("small buffer and flagNoBufferGrow set")
+	}
 	*h = Header{
 		hbuf:  h.hbuf,
 		flags: h.flags & persistentFlags,

http/httpraw/header_test.go

@@ -2,6 +2,7 @@ package httpraw
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -10,6 +11,8 @@ import (
 	"time"
 )
 
+const numHeaderCapacity = 16
+
 func TestHeaderParseRequest(t *testing.T) {
 	const (
 		wantMethod  = "GET"
@@ -387,7 +390,7 @@ func TestCopyDecodedPercentURLInPlace(t *testing.T) {
 
 func TestHeaderSetOverwrite(t *testing.T) {
 	var h Header
-	h.Reset(nil)
+	h.Reset(nil, numHeaderCapacity)
 	h.SetMethod("GET")
 	h.SetRequestTarget("/")
 	h.SetProtocol("HTTP/1.1")
@@ -410,7 +413,7 @@ func TestHeaderSetOverwrite(t *testing.T) {
 
 func TestHeaderSetBytesEmptyValue(t *testing.T) {
 	var h Header
-	h.Reset(nil)
+	h.Reset(nil, numHeaderCapacity)
 	h.SetBytes("X-Empty", nil)
 	if got := h.Get("X-Empty"); len(got) != 0 {
 		t.Errorf("want empty value, got %q", got)
@@ -465,7 +468,7 @@ func TestHeader_SplitBeforeColonStillParses(t *testing.T) {
 	const part2 = ": example.com\r\n\r\n"
 
 	var h Header
-	h.Reset(nil)
+	h.Reset(nil, numHeaderCapacity)
 	if _, err := h.ReadFromBytes([]byte(part1)); err != nil {
 		t.Fatal(err)
 	}
@@ -498,7 +501,7 @@ func TestHeader_AppendHeaderExactCapNoPanic(t *testing.T) {
 	const key, value = "K", "V"
 	buf := make([]byte, 0, len(key)+len(value)) // exact cap, no slack.
 	var h Header
-	h.Reset(buf)
+	h.Reset(buf, numHeaderCapacity)
 	defer func() {
 		if r := recover(); r != nil {
 			t.Fatalf("appendHeader panicked on exact-cap buffer: %v", r)
@@ -515,8 +518,8 @@ func TestHeader_AppendHeaderExactCapNoPanic(t *testing.T) {
 func TestHeader_AddFullBufferNoPanic(t *testing.T) {
 	buf := make([]byte, 0, 40) // Small cap; enough for Reset (len 0) but not the field below.
 	var h Header
-	h.Reset(buf)
-	h.EnableBufferGrowth(false)
+	h.Reset(buf, numHeaderCapacity)
+	h.ConfigBufferGrowth(false)
 	h.SetMethod("GET")
 	h.SetRequestTarget("/")
 	h.SetProtocol("HTTP/1.1")
@@ -550,7 +553,7 @@ func TestHeader_SetInt(t *testing.T) {
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			var h Header
-			h.Reset(nil)
+			h.Reset(nil, numHeaderCapacity)
 			h.SetInt("Content-Length", tc.value, tc.base)
 			if got := string(h.Get("Content-Length")); got != tc.want {
 				t.Fatalf("want %q, got %q", tc.want, got)
@@ -562,7 +565,7 @@ func TestHeader_SetInt(t *testing.T) {
 // SetInt on an existing key must reuse the slot in place (single field, latest value).
 func TestHeader_SetIntOverwrite(t *testing.T) {
 	var h Header
-	h.Reset(nil)
+	h.Reset(nil, numHeaderCapacity)
 	h.SetMethod("GET")
 	h.SetRequestTarget("/")
 	h.SetProtocol("HTTP/1.1")
@@ -586,8 +589,8 @@ func TestHeader_SetIntOverwrite(t *testing.T) {
 func TestHeader_SetIntNoAlloc(t *testing.T) {
 	buf := make([]byte, 0, 256)
 	var h Header
-	h.Reset(buf)
-	h.EnableBufferGrowth(false)
+	h.Reset(buf, numHeaderCapacity)
+	h.ConfigBufferGrowth(false)
 	h.Add("Content-Length", "0000000000000000000000") // pre-size a reusable slot.
 	allocs := testing.AllocsPerRun(100, func() {
 		h.SetInt("Content-Length", 1234567890, 10)
@@ -599,3 +602,43 @@ func TestHeader_SetIntNoAlloc(t *testing.T) {
 		t.Fatalf("want %q, got %q", "1234567890", got)
 	}
 }
+
+// A browser sends upwards of twenty header fields and an API client with a few
+// custom fields is not far behind. The field table must be sized from the
+// buffer the caller handed over, not fixed at a count that a real request
+// exceeds.
+func TestHeader_FieldTableSizedFromBuffer(t *testing.T) {
+	const wantVal = "the-canary-value"
+	raw := "GET / HTTP/1.1\r\nHost: lneto.test\r\n"
+	for i := 0; i < 40; i++ {
+		raw += "X-Field-" + strconv.Itoa(i) + ": value-of-a-realistic-length-here\r\n"
+	}
+	raw += "X-Canary: " + wantVal + "\r\n\r\n"
+
+	var h Header
+	h.Reset(make([]byte, 0, 8192), numHeaderCapacity) // Room for the block with plenty to spare.
+	err := h.ParseBytes(false, []byte(raw))
+	if err != nil {
+		t.Fatalf("parsing a 42 field request into an 8kB buffer: %s", err)
+	}
+	if got := string(h.Get("X-Canary")); got != wantVal {
+		t.Fatalf("want X-Canary %q, got %q", wantVal, got)
+	}
+}
+
+// A buffer too small for the fields it is handed must be refused with an error
+// the caller can act on, so a server answers 431 instead of dropping the peer.
+func TestHeader_FieldTableFullIsReported(t *testing.T) {
+	raw := "GET / HTTP/1.1\r\n"
+	for i := 0; i < 64; i++ {
+		raw += "H" + strconv.Itoa(i) + ":v\r\n" // As short as a field gets.
+	}
+	raw += "\r\n"
+	var h Header
+	h.Reset(make([]byte, 0, 512), numHeaderCapacity)
+	h.ConfigBufferGrowth(false)
+	err := h.ParseBytes(false, []byte(raw))
+	if !errors.Is(err, ErrHeaderTooMany) {
+		t.Fatalf("want ErrHeaderFieldsTooLarge, got %v", err)
+	}
+}

http/httpraw/parse.go

@@ -14,12 +14,17 @@ var (
 	errNoProto = errors.New("missing protocol, HTTP/0.9 unsupported")
 	// ErrNeedMoreData signals a parser was handed an incomplete buffer: append
 	// more data to it and call again.
-	ErrNeedMoreData = errors.New("need more data: cannot find trailing lf/delimiter")
-	errNoBoundary   = errors.New("httpraw: multipart boundary not set")
-	errUnparsed     = errors.New("need to finish parsing")
-	errInvalidName  = errors.New("invalid header name")
-	errSmallBuffer  = errors.New("small read buffer. Increase ReadBufferSize")
-	errOOM          = errors.New("httpraw: buffer out of memory")
+	ErrNeedMoreData      = errors.New("need more data: cannot find trailing lf/delimiter")
+	errNoBoundary        = errors.New("httpraw: multipart boundary not set")
+	errUnparsed          = errors.New("need to finish parsing")
+	errInvalidName       = errors.New("invalid header name")
+	ErrSmallHeaderBuffer = errors.New("httpraw: Header buffer exhausted, increase size")
+	errOOM               = errors.New("httpraw: Header incomplete due to OOM")
+	// ErrHeaderTooMany signals a header block carrying more fields than
+	// the buffer it is parsed into has room for, see [Header.Reset]. A server
+	// answers it with 431, RFC 6585 5: no larger buffer is coming, so reading
+	// the rest of the block would only spend memory on a request already lost.
+	ErrHeaderTooMany = errors.New("httpraw: more header fields than buffer holds")
 	// Header.Set and Header.Add mangles the buffer.
 	// Call them after retrieving the Body. Do not call them before parsing the header (why would you even do that?).
 	errMangledBuffer    = errors.New("httpraw: mangled buffer")
@@ -52,17 +57,21 @@ type headerBuf struct {
 	headers []argsKV
 }
 
-// reset sets the buffer data and discards all parsed data.
-func (h *headerBuf) reset(buf []byte) {
+// reset sets the buffer data and discards all parsed data. The field table is
+// grown to match the new buffer's capacity and never shrinks, so a header
+// reused across requests settles on its largest buffer and stops allocating.
+func (h *headerBuf) reset(buf []byte, numHeaderCapacity int) {
 	if buf == nil {
 		buf = h.buf[:0] // Reuse buffer but discard raw data on nil input.
 	}
-	if cap(h.headers) == 0 {
-		h.headers = make([]argsKV, 16)
+	if numHeaderCapacity != 0 {
+		internal.SliceReuse(&h.headers, numHeaderCapacity)
+	} else {
+		h.headers = h.headers[:0]
 	}
 	*h = headerBuf{
 		buf:     buf,
-		headers: h.headers[:0],
+		headers: h.headers,
 	}
 }
 
@@ -100,7 +109,7 @@ func (h *Header) parse(asResponse bool) (err error) {
 		return err
 	}
 	debuglog("http:firstline:done")
-	err = h.parseNextHeaders()
+	err = h.parseNextHeaders(h.flags)
 	debuglog("http:headers:done")
 	return err
 }
@@ -117,9 +126,9 @@ func (h *Header) parseFirstLine(asResponse bool) (err error) {
 	return err
 }
 
-func (h *Header) parseNextHeaders() error {
+func (h *Header) parseNextHeaders(flags Flags) error {
 	var ss scannerState
-	h.hbuf.parseNextHeaders(&ss)
+	h.hbuf.parseNextHeaders(&ss, flags)
 	if ss.err != nil {
 		h.flags |= flagConnClose
 		return ss.err
@@ -134,13 +143,13 @@ func (hb *headerBuf) readFromBytes(b []byte) {
 
 func (hb *headerBuf) free() int { return cap(hb.buf) - len(hb.buf) }
 
-func (hb *headerBuf) parseNextHeaders(ss *scannerState) {
+func (hb *headerBuf) parseNextHeaders(ss *scannerState, flags Flags) {
 	debuglog("http:nexthdr:loop")
 	for kv := hb.next(ss); kv.isValid(); kv = hb.next(ss) {
-		if len(hb.headers) == cap(hb.headers) {
-			// Refuse to grow the headers slice: caller must pre-allocate
-			// sufficient capacity via reset or use a larger initial size.
-			ss.err = errOOM
+		if len(hb.headers) == cap(hb.headers) && flags.HasAny(flagNoBufferGrow) {
+			// Refuse to grow the headers slice: the caller granted this much
+			// memory and no more, see [Header.Reset].
+			ss.err = ErrHeaderTooMany
 			return
 		}
 		hb.headers = append(hb.headers, kv)

http/httpraw/parse_test.go

@@ -10,7 +10,7 @@ func TestTryParse_IncrementalRequest(t *testing.T) {
 	// Full HTTP request split across multiple ReadFromBytes calls.
 	full := "GET /index.html HTTP/1.1\r\nHost: example.com\r\nContent-Type: text/html\r\n\r\nbody here"
 	var hdr Header
-	hdr.Reset(make([]byte, 0, 256))
+	hdr.Reset(make([]byte, 0, 256), numHeaderCapacity)
 
 	// Feed data in small chunks to exercise incremental parsing.
 	chunks := splitInto(full, 10)
@@ -85,7 +85,7 @@ func TestTryParse_IncrementalRequest(t *testing.T) {
 func TestTryParse_IncrementalResponse(t *testing.T) {
 	full := "HTTP/1.1 200 OK\r\nContent-Length: 5\r\nServer: lneto\r\n\r\nhello"
 	var hdr Header
-	hdr.Reset(make([]byte, 0, 256))
+	hdr.Reset(make([]byte, 0, 256), numHeaderCapacity)
 
 	chunks := splitInto(full, 8)
 	var done bool
@@ -137,7 +137,7 @@ func TestReadFromLimited(t *testing.T) {
 	r := strings.NewReader(data)
 
 	var hdr Header
-	hdr.Reset(make([]byte, 0, 256))
+	hdr.Reset(make([]byte, 0, 256), numHeaderCapacity)
 
 	// Read in one shot.
 	n, err := hdr.ReadFromLimited(r, 256)
@@ -163,7 +163,7 @@ func TestReadFromLimited(t *testing.T) {
 
 func TestReadFromLimited_MaxBytes(t *testing.T) {
 	var hdr Header
-	hdr.Reset(make([]byte, 0, 256))
+	hdr.Reset(make([]byte, 0, 256), numHeaderCapacity)
 
 	// Zero maxBytesToRead should error.
 	_, err := hdr.ReadFromLimited(strings.NewReader("data"), 0)
@@ -174,7 +174,7 @@ func TestReadFromLimited_MaxBytes(t *testing.T) {
 
 func TestReadFromBytes_Empty(t *testing.T) {
 	var hdr Header
-	hdr.Reset(make([]byte, 0, 256))
+	hdr.Reset(make([]byte, 0, 256), numHeaderCapacity)
 
 	_, err := hdr.ReadFromBytes(nil)
 	if err == nil {
@@ -184,7 +184,7 @@ func TestReadFromBytes_Empty(t *testing.T) {
 
 func TestBufferFreeAndCapacity(t *testing.T) {
 	var hdr Header
-	hdr.Reset(make([]byte, 0, 100))
+	hdr.Reset(make([]byte, 0, 100), numHeaderCapacity)
 
 	if hdr.BufferCapacity() != 100 {
 		t.Errorf("capacity = %d; want 100", hdr.BufferCapacity())
@@ -202,9 +202,8 @@ func TestBufferFreeAndCapacity(t *testing.T) {
 func TestEnableBufferGrowth(t *testing.T) {
 	var hdr Header
 	buf := make([]byte, 0, 64)
-	hdr.Reset(buf)
-	hdr.EnableBufferGrowth(false)
-
+	hdr.Reset(buf, numHeaderCapacity)
+	hdr.ConfigBufferGrowth(false)
 	// With growth disabled, reading more than capacity should fail.
 	big := make([]byte, 128)
 	for i := range big {
@@ -389,7 +388,7 @@ func TestHeader_MultilineValue(t *testing.T) {
 
 func TestHeader_ResponseRoundTrip(t *testing.T) {
 	var hdr Header
-	hdr.Reset(make([]byte, 0, 256))
+	hdr.Reset(make([]byte, 0, 256), numHeaderCapacity)
 	hdr.SetProtocol("HTTP/1.1")
 	hdr.SetStatus("404", "Not Found")
 	hdr.Add("Content-Type", "text/plain")
@@ -429,7 +428,7 @@ func TestHeader_ResponseRoundTrip(t *testing.T) {
 
 func TestHeader_RequestRoundTrip(t *testing.T) {
 	var hdr Header
-	hdr.Reset(make([]byte, 0, 256))
+	hdr.Reset(make([]byte, 0, 256), numHeaderCapacity)
 	hdr.SetProtocol("HTTP/1.1")
 	hdr.SetMethod("POST")
 	hdr.SetRequestTarget("/api/data")