more httpraw bug fixes (#160)

* more httpraw bug fixes

* add http-linux example and add httpraw.Header.SetInt

* finish http-linux example

* fix CI vet
This commit is contained in:
Pat Whittingslow
2026-07-18 12:11:37 -03:00
committed by Patricio Whittingslow
parent 347cef9ba5
commit 1a474154cb
6 changed files with 545 additions and 27 deletions
+1 -1
View File
@@ -22,7 +22,7 @@ vendor/
*.dylib
*.hex
*.wasm
/http-linux
# Profiling
*.pprof
+96
View File
@@ -0,0 +1,96 @@
//go:build !tinygo && linux
package main
import (
"net/netip"
"syscall"
)
// Conn wraps an accepted TCP connection from a raw Linux socket file descriptor.
// It implements io.Reader/io.Writer/io.Closer over syscall.Read/Write/Close.
type Conn struct {
fd int
remote netip.AddrPort
}
// Read reads bytes from the connection into b.
func (c *Conn) Read(b []byte) (int, error) {
if len(b) == 0 {
return 0, nil
}
n, err := syscall.Read(c.fd, b)
if err != nil {
return 0, err
}
if n == 0 {
return 0, syscall.ECONNRESET // Peer closed.
}
return n, nil
}
// Write writes b to the connection, looping until all bytes are sent.
func (c *Conn) Write(b []byte) (int, error) {
total := 0
for total < len(b) {
n, err := syscall.Write(c.fd, b[total:])
if err != nil {
return total, err
}
total += n
}
return total, nil
}
// Close closes the underlying file descriptor.
func (c *Conn) Close() error {
return syscall.Close(c.fd)
}
// RemoteAddr returns the peer address of the connection.
func (c *Conn) RemoteAddr() netip.AddrPort { return c.remote }
// Listener wraps a listening TCP socket bound to a local port.
type Listener struct {
fd int
}
// Listen creates a listening TCP socket bound to port on all interfaces.
func Listen(port uint16) (*Listener, error) {
fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)
if err != nil {
return nil, err
}
// Allow quick rebind after restart.
if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
syscall.Close(fd)
return nil, err
}
addr := &syscall.SockaddrInet4{Port: int(port)}
if err = syscall.Bind(fd, addr); err != nil {
syscall.Close(fd)
return nil, err
}
if err = syscall.Listen(fd, syscall.SOMAXCONN); err != nil {
syscall.Close(fd)
return nil, err
}
return &Listener{fd: fd}, nil
}
// Accept blocks until an incoming connection arrives and returns it as a Conn.
func (l *Listener) Accept(conn *Conn) error {
nfd, sa, err := syscall.Accept(l.fd)
if err != nil {
return err
}
conn.fd = nfd
if sa4, ok := sa.(*syscall.SockaddrInet4); ok {
conn.remote = netip.AddrPortFrom(netip.AddrFrom4(sa4.Addr), uint16(sa4.Port))
}
return nil
}
// Close closes the listening socket.
func (l *Listener) Close() error { return syscall.Close(l.fd) }
+107
View File
@@ -0,0 +1,107 @@
//go:build !tinygo && linux
package main
import (
"os"
"strconv"
"sync/atomic"
"time"
"github.com/soypat/lneto/http/httpraw"
)
const listenPort = 8080
func main() {
if err := run(); err != nil {
println("Error: ", err)
os.Exit(1)
}
println("DONE")
}
func run() error {
ln, err := Listen(listenPort)
if err != nil {
return err
}
defer ln.Close()
println("listening on port", listenPort)
conn := new(Conn)
for {
err := ln.Accept(conn)
if err != nil {
return err
}
visits.Add(1)
if err := handle(conn); err != nil {
println("handle:", conn.RemoteAddr().String(), err.Error())
}
conn.Close()
}
}
const (
htmlHead = `<html><body bgcolor="#000080" text="#00FF00"><center>` +
`<marquee><font face="Comic Sans MS" size="5" color="#FFFF00">` +
`*** WELCOME TO MY HOMEPAGE ***</font></marquee>` +
`<h1><blink>YOU ARE VISITOR #`
htmlTail = `!</blink></h1>` +
`<font color="#FF00FF">Sign my guestbook!</font>` +
`<br><hr>Best viewed in Netscape Navigator</center></body></html>`
)
const maxHTTPHeader = 1024
var (
hdr httpraw.Header
httpbuf [maxHTTPHeader]byte
htmlbuf [512]byte
visits atomic.Uint64
)
func handle(conn *Conn) error {
hdr.Reset(httpbuf[:0])
hdr.EnableBufferGrowth(false) // Limit memory to buffer capacity.
const incomingIsResponse = false // We get HTTP requests from clients.
deadline := time.Now().Add(50 * time.Millisecond)
for time.Until(deadline) > 0 {
if _, err := hdr.ReadFromLimited(conn, maxHTTPHeader); err != nil {
return err
}
needmoredata, err := hdr.TryParse(incomingIsResponse)
if err != nil {
return err
}
if needmoredata {
continue
}
break
}
println("\n\n================\n\n", hdr.String())
if time.Since(deadline) > 0 {
print("DEADLINE EXCEED: ", hdr.BufferParsed(), "/", hdr.BufferReceived(), " bytes parsed/read\n")
return nil
}
// Prepare tacky HTML response.
n := copy(htmlbuf[:], htmlHead)
n += len(strconv.AppendUint(htmlbuf[n:n], visits.Load(), 10))
n += copy(htmlbuf[n:], htmlTail)
contentLen := n
hdr.Reset(httpbuf[:0])
hdr.SetProtocol("HTTP/1.1")
hdr.SetStatus("200", "OK")
hdr.Set("Content-Type", "text/html")
hdr.SetInt("Content-Length", int64(contentLen), 10)
// Here we do some buffer juggling. We use remaining space
// of HTTP Header buffer to write the response that will be written over the wire.
respbuf := httpbuf[hdr.BufferUsed():]
header, err := hdr.AppendResponse(respbuf[:0])
if err != nil {
return err
}
conn.Write(header)
_, err = conn.Write(htmlbuf[:contentLen])
return err
}
+41 -6
View File
@@ -164,6 +164,7 @@ func (h *Header) ReadFromBytes(b []byte) (int, error) {
}
// BufferReceived returns the amoung of bytes read during calls to Read* methods.
// Returns 0 if buffer is invalid/mangled.
func (h *Header) BufferReceived() int {
if h.flags.hasAny(flagMangledBuffer | flagOOMReached) {
return 0
@@ -181,12 +182,23 @@ func (h *Header) BufferParsed() int {
return h.hbuf.off
}
// BufferUsed returns the raw memory used.
//
// BufferUsed + BufferFree == BufferCapacity
func (h *Header) BufferUsed() int {
return len(h.hbuf.buf)
}
// BufferFree returns amount of bytes free in underlying buffer.
//
// BufferUsed + BufferFree == BufferCapacity
func (h *Header) BufferFree() int {
return h.hbuf.free()
}
// BufferCapacity returns the total capacity of the underlying buffer.
//
// BufferUsed + BufferFree == BufferCapacity
func (h *Header) BufferCapacity() int {
return cap(h.hbuf.buf)
}
@@ -222,7 +234,7 @@ func (hb *headerBuf) forEach(cb func(key, value []byte) error) error {
// h.Reset(httpHeader); h.Parse() // Parse bytes in place with no copying.
// h.Reset(nil) // Reuse buffer previously set in a call to Reset.
func (h *Header) Reset(buf []byte) {
if h.flags.hasAny(flagNoBufferGrow) && len(buf) < 32 {
if h.flags.hasAny(flagNoBufferGrow) && cap(buf) < 32 {
panic("small buffer and flagNoBufferGrow set")
}
const persistentFlags = flagNoBufferGrow
@@ -252,9 +264,36 @@ func (h *Header) SetBytes(key string, value []byte) {
h.Set(key, b2s(value))
}
// SetInt is equivalent to [Header.Set] but with an integer value i.e: Content-Length header key.
// base must be in the range 2..36 (as accepted by [strconv.AppendInt]); other bases are dropped.
// SetInt formats the value directly into the header buffer without heap allocation.
func (h *Header) SetInt(key string, value int64, base int) {
if base < 2 || base > 36 {
return // strconv.AppendInt only supports base 2..36.
}
useKv := h.takeReusableSlot(key)
if useKv == nil {
h.appendHeaderInt(key, value, base)
} else {
useKv.value = h.reuseOrAppendInt(useKv.value, value, base)
}
}
// Set sets a key-value pair in the HTTP header.
// Calling Set mangles the buffer.
func (h *Header) Set(key, value string) {
useKv := h.takeReusableSlot(key)
if useKv == nil {
h.appendHeader(key, value)
} else {
useKv.value = h.reuseOrAppend(useKv.value, value)
}
}
// takeReusableSlot returns the valid key-value entry for key with the largest
// value buffer (best candidate for in-place reuse) and invalidates any other
// entries sharing the key. Returns nil if the key is not present.
func (h *Header) takeReusableSlot(key string) *argsKV {
hb := &h.hbuf
var useKv *argsKV
for i := 0; i < len(hb.headers); i++ {
@@ -271,11 +310,7 @@ func (h *Header) Set(key, value string) {
}
}
}
if useKv == nil {
h.appendHeader(key, value)
} else {
useKv.value = h.reuseOrAppend(useKv.value, value)
}
return useKv
}
// Get gets the first value of a key found in the headers. Use [Header.ForEach] to find multiple values corresponding to same key.
+183
View File
@@ -240,3 +240,186 @@ func TestHeaderSetBytesEmptyValue(t *testing.T) {
t.Errorf("want empty value, got %q", got)
}
}
// buffer/offset past uint16 range silently corrupts or panics.
// A header block > 64KiB pushes a field's offset past 65535. tokint truncation
// makes Get return the wrong window (silent corruption) or panic on slice bounds.
func TestHeader_LargeBufferOverflow(t *testing.T) {
const wantVal = "the-canary-value"
// Pad with a big header so the canary field lands past offset 65535.
pad := strings.Repeat("x", 70000)
raw := "GET / HTTP/1.1\r\n" +
"X-Pad: " + pad + "\r\n" +
"X-Canary: " + wantVal + "\r\n" +
"\r\n"
var h Header
err := h.ParseBytes(false, []byte(raw))
if err != nil {
// Clean rejection of the oversized header is the intended behavior:
// no panic, no silent corruption.
return
}
// If it did parse, the value must be correct (never a truncated-offset window).
got := string(h.Get("X-Canary"))
if got != wantVal {
t.Fatalf("overflow corruption: want X-Canary %q, got %q", wantVal, got)
}
}
// a complete but malformed header line with no colon must be a hard error,
// not errNeedMore (which makes a streaming parser wait forever).
func TestHeader_ColonlessLineIsHardError(t *testing.T) {
raw := "GET / HTTP/1.1\r\nBadHeaderNoColon\r\n\r\n"
var h Header
err := h.ParseBytes(false, []byte(raw))
if err == nil {
t.Fatal("want error on colonless header line, got nil")
}
if err == errNeedMore {
t.Fatalf("colonless line reported as errNeedMore (parser would hang); want a hard error like errInvalidName")
}
}
// Regression for a TCP split landing BEFORE the colon (no newline yet) must
// stay "need more data" and parse once the rest arrives. This is the case the
// Colonless fix must NOT turn into a hard error.
func TestHeader_SplitBeforeColonStillParses(t *testing.T) {
const part1 = "GET / HTTP/1.1\r\nHost" // split mid-key, before colon+newline
const part2 = ": example.com\r\n\r\n"
var h Header
h.Reset(nil)
if _, err := h.ReadFromBytes([]byte(part1)); err != nil {
t.Fatal(err)
}
needMore, err := h.TryParse(false)
if err != nil && err != errNeedMore {
t.Fatalf("split before colon: want errNeedMore/nil, got %v", err)
}
if !needMore {
t.Fatal("want needMoreData=true after partial input")
}
if _, err := h.ReadFromBytes([]byte(part2)); err != nil {
t.Fatal(err)
}
needMore, err = h.TryParse(false)
if err != nil {
t.Fatalf("after full input: %v", err)
}
if needMore {
t.Fatal("want needMoreData=false after full input")
}
if got := string(h.Get("Host")); got != "example.com" {
t.Fatalf("want Host %q, got %q", "example.com", got)
}
}
// appendHeader must reserve the +1 byte that mustAppendSlice consumes on an
// empty buffer. Force cap == len(key)+len(value) to defeat allocator rounding.
func TestHeader_AppendHeaderExactCapNoPanic(t *testing.T) {
const key, value = "K", "V"
buf := make([]byte, 0, len(key)+len(value)) // exact cap, no slack.
var h Header
h.Reset(buf)
defer func() {
if r := recover(); r != nil {
t.Fatalf("appendHeader panicked on exact-cap buffer: %v", r)
}
}()
h.Add(key, value)
if got := string(h.Get(key)); got != value {
t.Fatalf("want %q, got %q", value, got)
}
}
// Add/Set on a full buffer with growth disabled must drop gracefully (flag OOM),
// never panic. Panicking is unacceptable for this package.
func TestHeader_AddFullBufferNoPanic(t *testing.T) {
buf := make([]byte, 0, 40) // Small cap; enough for Reset (len 0) but not the field below.
var h Header
h.Reset(buf)
h.EnableBufferGrowth(false)
h.SetMethod("GET")
h.SetRequestURI("/")
h.SetProtocol("HTTP/1.1")
defer func() {
if r := recover(); r != nil {
t.Fatalf("Add on full no-grow buffer panicked: %v", r)
}
}()
h.Add("X-Very-Long-Header-Key", "a-value-that-cannot-possibly-fit-in-the-buffer")
// Graceful drop: request marshalling reports OOM instead of emitting bad data.
if _, err := h.AppendRequest(nil); err == nil {
t.Fatal("want OOM error after dropped Add, got nil")
}
}
func TestHeader_SetInt(t *testing.T) {
for _, tc := range []struct {
name string
value int64
base int
want string
}{
{"decimal", 1234, 10, "1234"},
{"zero", 0, 10, "0"},
{"negative", -42, 10, "-42"},
{"maxint64", 9223372036854775807, 10, "9223372036854775807"},
{"minint64", -9223372036854775808, 10, "-9223372036854775808"},
{"hex", 255, 16, "ff"},
} {
t.Run(tc.name, func(t *testing.T) {
var h Header
h.Reset(nil)
h.SetInt("Content-Length", tc.value, tc.base)
if got := string(h.Get("Content-Length")); got != tc.want {
t.Fatalf("want %q, got %q", tc.want, got)
}
})
}
}
// SetInt on an existing key must reuse the slot in place (single field, latest value).
func TestHeader_SetIntOverwrite(t *testing.T) {
var h Header
h.Reset(nil)
h.SetMethod("GET")
h.SetRequestURI("/")
h.SetProtocol("HTTP/1.1")
h.SetInt("Content-Length", 100, 10)
h.SetInt("Content-Length", 5, 10) // shorter, must fit in old slot.
if got := string(h.Get("Content-Length")); got != "5" {
t.Fatalf("want Content-Length %q, got %q", "5", got)
}
req, err := h.AppendRequest(nil)
if err != nil {
t.Fatal(err)
}
if n := strings.Count(string(req), "Content-Length:"); n != 1 {
t.Errorf("want 1 Content-Length field, got %d:\n%s", n, req)
}
}
// SetInt must not heap-allocate: it must format directly into the header buffer.
func TestHeader_SetIntNoAlloc(t *testing.T) {
buf := make([]byte, 0, 256)
var h Header
h.Reset(buf)
h.EnableBufferGrowth(false)
h.Add("Content-Length", "0000000000000000000000") // pre-size a reusable slot.
allocs := testing.AllocsPerRun(100, func() {
h.SetInt("Content-Length", 1234567890, 10)
})
if allocs != 0 {
t.Fatalf("SetInt allocated %v times, want 0", allocs)
}
if got := string(h.Get("Content-Length")); got != "1234567890" {
t.Fatalf("want %q, got %q", "1234567890", got)
}
}
+117 -20
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"errors"
"slices"
"strconv"
"unsafe"
"github.com/soypat/lneto/internal"
@@ -27,8 +28,14 @@ var (
errNeedMethodURI = errors.New("need method/request URI to create request header")
errBadStatusCodeTxt = errors.New("invalid status code or text")
errCookiesParsed = errors.New("cookies already parsed, reset before parsing again")
errBufferTooLarge = errors.New("httpraw: buffer exceeds max size (offsets are uint16)")
)
// maxBufLen bounds the header buffer. Offsets/lengths are stored as uint16
// (tokint); a buffer past this would truncate/overflow those, silently
// returning the wrong bytes or panicking on a wrapped slice bound.
const maxBufLen = 0xffff
type headerBuf struct {
// buf[:len] holds entire HTTP header data, which may be normalized by [flags]. buf[off:len] holds data not yet processed during parsing.
buf []byte
@@ -92,6 +99,9 @@ func (h *Header) parse(asResponse bool) (err error) {
}
func (h *Header) parseFirstLine(asResponse bool) (err error) {
if len(h.hbuf.buf) > maxBufLen {
return errBufferTooLarge // Offsets would overflow uint16 tokint.
}
if asResponse {
h.statusCode, h.statusText, h.flags, err = h.hbuf.parseFirstLineResponse(h.flags)
} else {
@@ -302,32 +312,22 @@ func (h *Header) reuseOrAppend(tok headerSlice, value string) headerSlice {
func (h *Header) appendSlice(value string) headerSlice {
debuglog("http:appendslice:start")
free := h.hbuf.free()
if len(value) > free {
if h.flags.hasAny(flagNoBufferGrow) {
h.flags |= flagOOMReached
return headerSlice{}
}
debuglog("http:appendslice:grow-buf")
h.hbuf.buf = slices.Grow(h.hbuf.buf, len(value)+1) // Grow 1 beyond due to slice validity.
if !h.reserve(len(value)) {
return headerSlice{}
}
h.flags |= flagMangledBuffer
return h.hbuf.mustAppendSlice(value)
}
func (h *Header) appendHeader(key, value string) {
hb := &h.hbuf
free := hb.free()
buf := h.hbuf.buf
if len(key)+len(value) > free {
if h.flags.hasAny(flagNoBufferGrow) {
panic(errSmallBuffer)
}
debuglog("http:appendhdr:grow-buf")
hb.buf = slices.Grow(buf, len(key)+len(value))
// reserve accounts for the byte-0 reservation mustAppendSlice makes on an
// empty buffer, and drops (flagging OOM) rather than panicking when growth
// is disabled and space runs out.
if !h.reserve(len(key) + len(value)) {
return
}
h.flags |= flagMangledBuffer
hb := &h.hbuf
k := hb.mustAppendSlice(key)
v := hb.mustAppendSlice(value)
debuglog("http:appendhdr:grow-hdrs")
@@ -337,6 +337,100 @@ func (h *Header) appendHeader(key, value string) {
})
}
// appendHeaderInt is appendHeader's integer counterpart: it appends key and the
// formatted integer value as a new header field.
func (h *Header) appendHeaderInt(key string, value int64, base int) {
n := intLen(value, base)
if !h.reserve(len(key) + n) {
return // Drop and flag OOM; never panic.
}
h.flags |= flagMangledBuffer
hb := &h.hbuf
k := hb.mustAppendSlice(key)
v := hb.mustAppendInt(value, base)
hb.headers = append(hb.headers, argsKV{
key: k,
value: v,
})
}
// reserve ensures need free bytes are available in the buffer, growing it when
// permitted. It accounts for the byte-0 reservation on an empty buffer (see
// mustAppendSlice). It returns false and sets flagOOMReached when the space
// cannot be guaranteed: a tokint offset overflow, or a full buffer with
// flagNoBufferGrow set.
func (h *Header) reserve(need int) bool {
hb := &h.hbuf
if len(hb.buf) == 0 {
need++ // mustAppend* reserves byte 0 on an empty buffer.
}
if len(hb.buf)+need > maxBufLen {
h.flags |= flagOOMReached // Offsets would overflow uint16 tokint.
return false
}
if need > hb.free() {
if h.flags.hasAny(flagNoBufferGrow) {
h.flags |= flagOOMReached
return false
}
hb.buf = slices.Grow(hb.buf, need)
}
return true
}
// reuseOrAppendInt writes value into tok's slot in place when it fits, avoiding
// any buffer growth; otherwise it appends a fresh slot.
func (h *Header) reuseOrAppendInt(tok headerSlice, value int64, base int) headerSlice {
n := intLen(value, base)
if int(tok.len) >= n {
// Reuse: format directly over the existing slot. No free space needed
// since n <= tok.len and the slot already lives inside buf.
v := strconv.AppendInt(h.hbuf.buf[tok.start:tok.start], value, base)
tok.len = tokint(len(v))
h.flags |= flagMangledBuffer
return tok
}
return h.appendInt(value, base, n)
}
// appendInt reserves space (growing or flagging OOM) and appends value as a new slot.
func (h *Header) appendInt(value int64, base, n int) headerSlice {
if !h.reserve(n) {
return headerSlice{} // Drop and flag OOM; never panic.
}
h.flags |= flagMangledBuffer
return h.hbuf.mustAppendInt(value, base)
}
// mustAppendInt formats value into the buffer's free region and commits it.
// The caller must have reserved at least intLen(value, base) free bytes.
func (hb *headerBuf) mustAppendInt(value int64, base int) headerSlice {
L := len(hb.buf)
if L == 0 {
L++ // Valid key-values start after byte 0.
}
v := strconv.AppendInt(hb.buf[L:L], value, base)
hb.buf = hb.buf[:L+len(v)]
return hb.slice(hb.buf[L : L+len(v)])
}
// intLen returns the number of bytes strconv.AppendInt would emit for value in
// the given base (including a leading minus sign for negatives). Used to size
// the buffer and to test whether a value fits an existing slot without writing.
func intLen(value int64, base int) int {
n := 1
u := uint64(value)
if value < 0 {
n++ // Leading minus sign.
u = -u // Two's-complement magnitude; correct even for math.MinInt64.
}
for u >= uint64(base) {
u /= uint64(base)
n++
}
return n
}
func (hb *headerBuf) noKV() argsKV { return argsKV{} }
func (hb *headerBuf) next(ss *scannerState) argsKV {
@@ -373,8 +467,11 @@ func (hb *headerBuf) next(ss *scannerState) argsKV {
ss.err = errInvalidName
return hb.noKV()
} else if n < 0 {
// No colon found, probably missing data.
ss.err = errNeedMore
// A newline is present (x>=0 reached here) but the line has no
// colon: malformed, not incomplete. A split arriving before the
// colon has no newline yet and is caught by the x<0 branch above,
// so it still returns errNeedMore.
ss.err = errInvalidName
return hb.noKV()
}
}