ai insists with backoffs

http/httpraw/multipart.go

@@ -10,20 +10,25 @@ import (
 // value has no length: it ends where the next delimiter begins.
 //
 // Multipart stores none of the body, leaving the caller to decide what to keep,
-// what to skip and when a part has grown too large:
+// what to skip and when a part has grown too large. Both methods report how much
+// of buf they consumed, which the caller compacts away before reading more in:
 //
-//	m := httpraw.Multipart{Boundary: httpraw.MultipartBoundary(contentType)}
+//	var m httpraw.Multipart
+//	m.SetContentType(contentType)
 //	var hdr httpraw.MultipartHeader
 //	for {
-//		rest, err := m.NextHeader(&hdr, buf)
+//		parsed, err := m.NextHeader(&hdr, buf[:buflen])
 //		if err != nil {
-//			break // io.EOF, or ErrNeedMoreData: read more into buf and retry.
+//			break // io.EOF at the closing delimiter, body done.
+//		} else if parsed == 0 {
+//			// Header block incomplete: read more into buf[buflen:] and retry.
+//			continue
 //		}
+//		buflen = copy(buf, buf[parsed:buflen])
 //		for {
-//			body, next, done := m.NextBody(rest)
-//			// Consume body for hdr.Name, then compact next to the front of buf
-//			// and read more.
-//			rest = next
+//			bodyLen, restOff, done := m.NextBody(buf[:buflen])
+//			// Consume buf[:bodyLen] for hdr.Name, then compact and read more.
+//			buflen = copy(buf, buf[restOff:buflen])
 //			if done {
 //				break
 //			}
@@ -36,19 +41,29 @@ type Multipart struct {
 }
 
 // MultipartHeader is a part's header block and the Content-Disposition
-// parameters that identify it. All fields alias the buffer they were parsed
-// from and stay valid as long as it does.
+// parameters that identify it.
 type MultipartHeader struct {
-	// PartView is the part's raw header block, ending in its final CRLF.
+	// PartView is the part's raw header block, ending in its final CRLF. It
+	// aliases the buffer it was parsed from, so it is only valid until that
+	// buffer is compacted or read into again.
 	PartView []byte
 	// Name is the name parameter of a part's Content-Disposition field,
 	// i.e: "photo" for `form-data; name="photo"; filename="beach.png"`.
+	// Copied out of the buffer, so it outlives it, and reused between parts.
 	Name []byte
 	// Filename is the filename parameter of a part's Content-Disposition
-	// field, nil when the part is not a file upload.
+	// field, empty when the part is not a file upload. Copied like Name.
 	Filename []byte
 }
 
+// Reset clears the header for the next part, keeping the buffers Name and
+// Filename were copied into so a reused header stops allocating.
+func (hdr *MultipartHeader) Reset() {
+	hdr.PartView = nil
+	hdr.Name = hdr.Name[:0]
+	hdr.Filename = hdr.Filename[:0]
+}
+
 // SetContentType sets [Multipart.Boundary] from the boundary parameter of a
 // Content-Type field value, i.e: "abc123" for
 // "multipart/form-data; boundary=abc123". The leading "--" the delimiter carries
@@ -63,48 +78,13 @@ func (m *Multipart) SetContentType(contentType []byte) error {
 	return nil
 }
 
-// NextHeader splits the leading part's header block off a multipart body into
-// dst, returning the body bytes that follow it. Returns [ErrNeedMoreData] while
-// data holds no complete delimiter and header block, and [io.EOF] once the
-// closing delimiter is reached. dst is left zeroed on error.
-func (m *Multipart) NextHeader(dst *MultipartHeader, data []byte) (rest []byte, err error) {
-	*dst = MultipartHeader{}
-	if len(m.Boundary) == 0 {
-		return nil, errNoBoundary
-	}
-	idx := m.indexDelimiter(data)
-	if idx < 0 {
-		return nil, ErrNeedMoreData
-	}
-	after := idx + len("--") + len(m.Boundary)
-	if after+2 > len(data) {
-		return nil, ErrNeedMoreData // Cannot tell a closing delimiter yet.
-	} else if data[after] == '-' && data[after+1] == '-' {
-		return nil, io.EOF
-	}
-	// Delimiter is followed by CRLF, then the part's header block.
-	if data[after] == '\r' {
-		after++
-	}
-	if after >= len(data) {
-		return nil, ErrNeedMoreData
-	} else if data[after] != '\n' {
-		return nil, errBadDelimiter
-	}
-	after++
-	end := bytes.Index(data[after:], []byte("\r\n\r\n"))
-	if end < 0 {
-		return nil, ErrNeedMoreData
-	}
-	dst.PartView = data[after : after+end+2]
-	disposition := partField(dst.PartView)
-	dst.Name = append(dst.Name[:0], ContentParam(disposition, "name")...)
-	dst.Filename = append(dst.Filename[:0], ContentParam(disposition, "filename")...)
-	return data[after+end+4:], nil
-}
-
-func (m *Multipart) NextHeaderInt(dst *MultipartHeader, data []byte) (parsedLen int, err error) {
-	*dst = MultipartHeader{}
+// NextHeader parses the leading part's header block off a multipart body into
+// dst, returning how many bytes of data it consumed: the part's content begins
+// at data[parsedLen]. A zero parsedLen and no error means data holds no complete
+// delimiter and header block yet, so the caller reads more in and retries.
+// Returns [io.EOF] once the closing delimiter is reached. dst is reset on error.
+func (m *Multipart) NextHeader(dst *MultipartHeader, data []byte) (parsedLen int, err error) {
+	dst.Reset()
 	if len(m.Boundary) == 0 {
 		return 0, errNoBoundary
 	}
@@ -139,35 +119,22 @@ func (m *Multipart) NextHeaderInt(dst *MultipartHeader, data []byte) (parsedLen
 	return after + end + 4, nil
 }
 
-func (m *Multipart) NextBodyInt(data []byte) (bodyLen int, done bool) {
+// NextBody reports how much of data is part content, data[:bodyLen], and where
+// what is left begins, data[restOff:], which the caller compacts to the front of
+// its buffer before reading more in. done reports the part ended, in which case
+// data[restOff:] begins the next part's delimiter; otherwise the bytes past
+// bodyLen are a tail held back because it could still turn into a delimiter.
+func (m *Multipart) NextBody(data []byte) (bodyLen, restOff int, done bool) {
 	idx := m.indexPartEnd(data)
 	if idx >= 0 {
-		return idx, true
-		// return data[:idx], data[idx+len("\r\n"):], true
+		return idx, idx + len("\r\n"), true
 	}
 	// Longest prefix of "\r\n--"+boundary that could still be completed.
 	hold := len("\r\n--") + len(m.Boundary) - 1
 	if hold > len(data) {
 		hold = len(data)
 	}
-	return len(data) - hold, false
-}
-
-// NextBody returns the part bytes available in data, holding back any tail
-// that could be the start of a delimiter. done reports the part ended, in which
-// case rest begins the next part's delimiter; otherwise rest is the held back
-// tail, which the caller compacts before reading more data into the buffer.
-func (m *Multipart) NextBody(data []byte) (body, rest []byte, done bool) {
-	idx := m.indexPartEnd(data)
-	if idx >= 0 {
-		return data[:idx], data[idx+len("\r\n"):], true
-	}
-	// Longest prefix of "\r\n--"+boundary that could still be completed.
-	hold := len("\r\n--") + len(m.Boundary) - 1
-	if hold > len(data) {
-		hold = len(data)
-	}
-	return data[:len(data)-hold], data[len(data)-hold:], false
+	return len(data) - hold, len(data) - hold, false
 }
 
 // indexDelimiter returns the offset of the leading "--"+Boundary in data.

http/httpraw/multipart_test.go

@@ -101,7 +101,7 @@ func TestContentParam(t *testing.T) {
 func TestNextPartHeader(t *testing.T) {
 	m := Multipart{Boundary: []byte(multiBoundary)}
 	var hdr MultipartHeader
-	rest, err := m.NextHeader(&hdr, []byte(multiBody))
+	parsed, err := m.NextHeader(&hdr, []byte(multiBody))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -112,11 +112,28 @@ func TestNextPartHeader(t *testing.T) {
 	if string(hdr.Name) != "caption" {
 		t.Errorf("want name %q, got %q", "caption", hdr.Name)
 	}
-	if hdr.Filename != nil {
-		t.Errorf("want nil filename for a non file part, got %q", hdr.Filename)
+	if len(hdr.Filename) != 0 {
+		t.Errorf("want no filename for a non file part, got %q", hdr.Filename)
 	}
-	if !strings.HasPrefix(string(rest), "hi there\r\n") {
-		t.Errorf("want rest at part body, got %q", rest)
+	if !strings.HasPrefix(multiBody[parsed:], "hi there\r\n") {
+		t.Errorf("want rest at part body, got %q", multiBody[parsed:])
+	}
+}
+
+// Names and filenames must outlive the buffer they were parsed from, so a
+// caller may compact it and read more without losing the part it is reading.
+func TestNextPartHeaderOutlivesBuffer(t *testing.T) {
+	m := Multipart{Boundary: []byte(multiBoundary)}
+	data := []byte(multiBody)
+	var hdr MultipartHeader
+	if _, err := m.NextHeader(&hdr, data); err != nil {
+		t.Fatal(err)
+	}
+	for i := range data {
+		data[i] = 'x' // Buffer reused for the next read.
+	}
+	if string(hdr.Name) != "caption" {
+		t.Errorf("want name %q to survive the buffer, got %q", "caption", hdr.Name)
 	}
 }
 
@@ -130,8 +147,9 @@ func TestNextPartHeaderNeedMore(t *testing.T) {
 		"------abc123\r\nContent-Disposition: form-", // Header block unterminated.
 	} {
 		var hdr MultipartHeader
-		if _, err := m.NextHeader(&hdr, []byte(data)); err != ErrNeedMoreData {
-			t.Errorf("%q: want ErrNeedMoreData, got %v", data, err)
+		parsed, err := m.NextHeader(&hdr, []byte(data))
+		if parsed != 0 || err != nil {
+			t.Errorf("%q: want (0, nil) asking for more data, got (%d, %v)", data, parsed, err)
 		}
 	}
 }
@@ -158,45 +176,48 @@ func TestNextPartHeaderEnd(t *testing.T) {
 func TestNextPartBody(t *testing.T) {
 	m := Multipart{Boundary: []byte(multiBoundary)}
 	var hdr MultipartHeader
-	rest, err := m.NextHeader(&hdr, []byte(multiBody))
+	parsed, err := m.NextHeader(&hdr, []byte(multiBody))
 	if err != nil {
 		t.Fatal(err)
 	}
-	body, rest, done := m.NextBody(rest)
+	rest := multiBody[parsed:]
+	bodyLen, restOff, done := m.NextBody([]byte(rest))
 	if !done {
 		t.Fatal("want the part to end within the buffer")
 	}
-	if string(body) != "hi there" {
-		t.Errorf("want body %q, got %q", "hi there", body)
+	if rest[:bodyLen] != "hi there" {
+		t.Errorf("want body %q, got %q", "hi there", rest[:bodyLen])
 	}
-	if !strings.HasPrefix(string(rest), "------abc123\r\n") {
-		t.Errorf("want rest at next delimiter, got %q", rest)
+	if !strings.HasPrefix(rest[restOff:], "------abc123\r\n") {
+		t.Errorf("want rest at next delimiter, got %q", rest[restOff:])
 	}
 }
 
 // A part whose bytes contain CRLFs and boundary-like text must survive intact.
 func TestNextPartBodyBinary(t *testing.T) {
 	m := Multipart{Boundary: []byte(multiBoundary)}
-	data := []byte(multiBody)
+	rest := []byte(multiBody)
 	var hdr MultipartHeader
-	rest, err := m.NextHeader(&hdr, data) // caption part.
+	parsed, err := m.NextHeader(&hdr, rest) // caption part.
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, rest, _ = m.NextBody(rest)
-	rest, err = m.NextHeader(&hdr, rest) // photo part.
+	_, restOff, _ := m.NextBody(rest[parsed:])
+	rest = rest[parsed+restOff:]
+	parsed, err = m.NextHeader(&hdr, rest) // photo part.
 	if err != nil {
 		t.Fatal(err)
 	}
-	body, rest, done := m.NextBody(rest)
+	rest = rest[parsed:]
+	bodyLen, restOff, done := m.NextBody(rest)
 	if !done {
 		t.Fatal("want the part to end within the buffer")
 	}
 	const want = "\x89PNG\r\n--not-the-boundary\r\n\x00\xff"
-	if string(body) != want {
-		t.Errorf("want body %q, got %q", want, body)
+	if string(rest[:bodyLen]) != want {
+		t.Errorf("want body %q, got %q", want, rest[:bodyLen])
 	}
-	if _, err = m.NextHeader(&hdr, rest); err != io.EOF {
+	if _, err = m.NextHeader(&hdr, rest[restOff:]); err != io.EOF {
 		t.Errorf("want io.EOF after last part, got %v", err)
 	}
 }
@@ -208,15 +229,16 @@ func TestNextPartBodySplitDelimiter(t *testing.T) {
 	const part = "hi there"
 	full := part + "\r\n------abc123\r\n"
 	for split := 1; split < len(full); split++ {
-		body, rest, done := m.NextBody([]byte(full[:split]))
+		data := full[:split]
+		bodyLen, restOff, done := m.NextBody([]byte(data))
 		if done {
 			continue // Whole delimiter already present, nothing to prove.
 		}
-		if len(body) > len(part) {
-			t.Fatalf("split %d: emitted %q, past the end of the part", split, body)
+		if bodyLen > len(part) {
+			t.Fatalf("split %d: emitted %q, past the end of the part", split, data[:bodyLen])
 		}
-		if string(body)+string(rest) != full[:split] {
-			t.Fatalf("split %d: body+rest %q%q does not reconstruct input", split, body, rest)
+		if data[:bodyLen]+data[restOff:] != data {
+			t.Fatalf("split %d: body+rest %q%q does not reconstruct input", split, data[:bodyLen], data[restOff:])
 		}
 	}
 }
@@ -225,12 +247,13 @@ func TestNextPartBodySplitDelimiter(t *testing.T) {
 func TestNextHeaderFilePart(t *testing.T) {
 	m := Multipart{Boundary: []byte(multiBoundary)}
 	var hdr MultipartHeader
-	rest, err := m.NextHeader(&hdr, []byte(multiBody)) // caption part.
+	parsed, err := m.NextHeader(&hdr, []byte(multiBody)) // caption part.
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, rest, _ = m.NextBody(rest)
-	if _, err = m.NextHeader(&hdr, rest); err != nil { // photo part.
+	rest := []byte(multiBody[parsed:])
+	_, restOff, _ := m.NextBody(rest)
+	if _, err = m.NextHeader(&hdr, rest[restOff:]); err != nil { // photo part.
 		t.Fatal(err)
 	}
 	if got := string(hdr.Name); got != "photo" {
@@ -245,7 +268,7 @@ func TestNextHeaderFilePart(t *testing.T) {
 }
 
 // A failed call must not leave the previous part's fields behind.
-func TestNextHeaderZeroesOnError(t *testing.T) {
+func TestNextHeaderResetsOnError(t *testing.T) {
 	m := Multipart{Boundary: []byte(multiBoundary)}
 	var hdr MultipartHeader
 	if _, err := m.NextHeader(&hdr, []byte(multiBody)); err != nil {
@@ -254,31 +277,49 @@ func TestNextHeaderZeroesOnError(t *testing.T) {
 	if _, err := m.NextHeader(&hdr, []byte("------abc123--\r\n")); err != io.EOF {
 		t.Fatalf("want io.EOF, got %v", err)
 	}
-	if hdr.PartView != nil || hdr.Name != nil || hdr.Filename != nil {
-		t.Errorf("want zeroed header on error, got %+v", hdr)
+	if hdr.PartView != nil || len(hdr.Name) != 0 || len(hdr.Filename) != 0 {
+		t.Errorf("want cleared header on error, got %+v", hdr)
 	}
 }
 
-// The whole loop, as a caller writes it.
+// A header reused across parts must stop allocating once its name and filename
+// buffers are big enough.
+func TestNextHeaderReuseNoAlloc(t *testing.T) {
+	m := Multipart{Boundary: []byte(multiBoundary)}
+	data := []byte(multiBody)
+	var hdr MultipartHeader
+	allocs := testing.AllocsPerRun(10, func() {
+		if _, err := m.NextHeader(&hdr, data); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocs != 0 {
+		t.Errorf("want a reused header to allocate 0 times, got %v", allocs)
+	}
+}
+
+// The whole loop, as a caller writes it over a buffer it compacts.
 func TestMultipartLoop(t *testing.T) {
 	m := Multipart{Boundary: []byte(multiBoundary)}
 	rest := []byte(multiBody)
 	var got []string
 	var hdr MultipartHeader
 	for {
-		next, err := m.NextHeader(&hdr, rest)
+		parsed, err := m.NextHeader(&hdr, rest)
 		if err == io.EOF {
 			break
 		} else if err != nil {
 			t.Fatal(err)
+		} else if parsed == 0 {
+			t.Fatal("header must complete within the buffer")
 		}
 		name := string(hdr.Name)
 		total := 0
-		rest = next
+		rest = rest[parsed:]
 		for {
-			body, next, done := m.NextBody(rest)
-			total += len(body)
-			rest = next
+			bodyLen, restOff, done := m.NextBody(rest)
+			total += bodyLen
+			rest = rest[restOff:]
 			if done {
 				break
 			}