diff --git a/.gitignore b/.gitignore index 321751d..29d9708 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ vendor/ *.dylib *.hex *.wasm - +/http-linux # Profiling *.pprof diff --git a/examples/http-linux/conn-linux.go b/examples/http-linux/conn-linux.go new file mode 100644 index 0000000..9da6143 --- /dev/null +++ b/examples/http-linux/conn-linux.go @@ -0,0 +1,96 @@ +//go:build !tinygo && linux + +package main + +import ( + "net/netip" + "syscall" +) + +// Conn wraps an accepted TCP connection from a raw Linux socket file descriptor. +// It implements io.Reader/io.Writer/io.Closer over syscall.Read/Write/Close. +type Conn struct { + fd int + remote netip.AddrPort +} + +// Read reads bytes from the connection into b. +func (c *Conn) Read(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + n, err := syscall.Read(c.fd, b) + if err != nil { + return 0, err + } + if n == 0 { + return 0, syscall.ECONNRESET // Peer closed. + } + return n, nil +} + +// Write writes b to the connection, looping until all bytes are sent. +func (c *Conn) Write(b []byte) (int, error) { + total := 0 + for total < len(b) { + n, err := syscall.Write(c.fd, b[total:]) + if err != nil { + return total, err + } + total += n + } + return total, nil +} + +// Close closes the underlying file descriptor. +func (c *Conn) Close() error { + return syscall.Close(c.fd) +} + +// RemoteAddr returns the peer address of the connection. +func (c *Conn) RemoteAddr() netip.AddrPort { return c.remote } + +// Listener wraps a listening TCP socket bound to a local port. +type Listener struct { + fd int +} + +// Listen creates a listening TCP socket bound to port on all interfaces. +func Listen(port uint16) (*Listener, error) { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != nil { + return nil, err + } + // Allow quick rebind after restart. + if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil { + syscall.Close(fd) + return nil, err + } + addr := &syscall.SockaddrInet4{Port: int(port)} + if err = syscall.Bind(fd, addr); err != nil { + syscall.Close(fd) + return nil, err + } + if err = syscall.Listen(fd, syscall.SOMAXCONN); err != nil { + syscall.Close(fd) + return nil, err + } + return &Listener{fd: fd}, nil +} + +// Accept blocks until an incoming connection arrives and returns it as a Conn. +func (l *Listener) Accept(conn *Conn) error { + nfd, sa, err := syscall.Accept(l.fd) + if err != nil { + return err + } + + conn.fd = nfd + if sa4, ok := sa.(*syscall.SockaddrInet4); ok { + conn.remote = netip.AddrPortFrom(netip.AddrFrom4(sa4.Addr), uint16(sa4.Port)) + } + return nil +} + +// Close closes the listening socket. +func (l *Listener) Close() error { return syscall.Close(l.fd) } diff --git a/examples/http-linux/main-httplinux.go b/examples/http-linux/main-httplinux.go new file mode 100644 index 0000000..b243fea --- /dev/null +++ b/examples/http-linux/main-httplinux.go @@ -0,0 +1,107 @@ +//go:build !tinygo && linux + +package main + +import ( + "os" + "strconv" + "sync/atomic" + "time" + + "github.com/soypat/lneto/http/httpraw" +) + +const listenPort = 8080 + +func main() { + if err := run(); err != nil { + println("Error: ", err) + os.Exit(1) + } + println("DONE") +} + +func run() error { + ln, err := Listen(listenPort) + if err != nil { + return err + } + defer ln.Close() + println("listening on port", listenPort) + conn := new(Conn) + for { + err := ln.Accept(conn) + if err != nil { + return err + } + visits.Add(1) + if err := handle(conn); err != nil { + println("handle:", conn.RemoteAddr().String(), err.Error()) + } + conn.Close() + } +} + +const ( + htmlHead = `
` + + `` + + `*** WELCOME TO MY HOMEPAGE ***` + + `

YOU ARE VISITOR #` + htmlTail = `!

` + + `Sign my guestbook!` + + `
Best viewed in Netscape Navigator
` +) + +const maxHTTPHeader = 1024 + +var ( + hdr httpraw.Header + httpbuf [maxHTTPHeader]byte + htmlbuf [512]byte + visits atomic.Uint64 +) + +func handle(conn *Conn) error { + hdr.Reset(httpbuf[:0]) + hdr.EnableBufferGrowth(false) // Limit memory to buffer capacity. + const incomingIsResponse = false // We get HTTP requests from clients. + deadline := time.Now().Add(50 * time.Millisecond) + for time.Until(deadline) > 0 { + if _, err := hdr.ReadFromLimited(conn, maxHTTPHeader); err != nil { + return err + } + needmoredata, err := hdr.TryParse(incomingIsResponse) + if err != nil { + return err + } + if needmoredata { + continue + } + break + } + println("\n\n================\n\n", hdr.String()) + if time.Since(deadline) > 0 { + print("DEADLINE EXCEED: ", hdr.BufferParsed(), "/", hdr.BufferReceived(), " bytes parsed/read\n") + return nil + } + // Prepare tacky HTML response. + n := copy(htmlbuf[:], htmlHead) + n += len(strconv.AppendUint(htmlbuf[n:n], visits.Load(), 10)) + n += copy(htmlbuf[n:], htmlTail) + contentLen := n + hdr.Reset(httpbuf[:0]) + hdr.SetProtocol("HTTP/1.1") + hdr.SetStatus("200", "OK") + hdr.Set("Content-Type", "text/html") + hdr.SetInt("Content-Length", int64(contentLen), 10) + // Here we do some buffer juggling. We use remaining space + // of HTTP Header buffer to write the response that will be written over the wire. + respbuf := httpbuf[hdr.BufferUsed():] + header, err := hdr.AppendResponse(respbuf[:0]) + if err != nil { + return err + } + conn.Write(header) + _, err = conn.Write(htmlbuf[:contentLen]) + return err +} diff --git a/http/httpraw/header.go b/http/httpraw/header.go index ada9046..d92ba68 100644 --- a/http/httpraw/header.go +++ b/http/httpraw/header.go @@ -164,6 +164,7 @@ func (h *Header) ReadFromBytes(b []byte) (int, error) { } // BufferReceived returns the amoung of bytes read during calls to Read* methods. +// Returns 0 if buffer is invalid/mangled. func (h *Header) BufferReceived() int { if h.flags.hasAny(flagMangledBuffer | flagOOMReached) { return 0 @@ -181,12 +182,23 @@ func (h *Header) BufferParsed() int { return h.hbuf.off } +// BufferUsed returns the raw memory used. +// +// BufferUsed + BufferFree == BufferCapacity +func (h *Header) BufferUsed() int { + return len(h.hbuf.buf) +} + // BufferFree returns amount of bytes free in underlying buffer. +// +// BufferUsed + BufferFree == BufferCapacity func (h *Header) BufferFree() int { return h.hbuf.free() } // BufferCapacity returns the total capacity of the underlying buffer. +// +// BufferUsed + BufferFree == BufferCapacity func (h *Header) BufferCapacity() int { return cap(h.hbuf.buf) } @@ -222,7 +234,7 @@ func (hb *headerBuf) forEach(cb func(key, value []byte) error) error { // h.Reset(httpHeader); h.Parse() // Parse bytes in place with no copying. // h.Reset(nil) // Reuse buffer previously set in a call to Reset. func (h *Header) Reset(buf []byte) { - if h.flags.hasAny(flagNoBufferGrow) && len(buf) < 32 { + if h.flags.hasAny(flagNoBufferGrow) && cap(buf) < 32 { panic("small buffer and flagNoBufferGrow set") } const persistentFlags = flagNoBufferGrow @@ -252,9 +264,36 @@ func (h *Header) SetBytes(key string, value []byte) { h.Set(key, b2s(value)) } +// SetInt is equivalent to [Header.Set] but with an integer value i.e: Content-Length header key. +// base must be in the range 2..36 (as accepted by [strconv.AppendInt]); other bases are dropped. +// SetInt formats the value directly into the header buffer without heap allocation. +func (h *Header) SetInt(key string, value int64, base int) { + if base < 2 || base > 36 { + return // strconv.AppendInt only supports base 2..36. + } + useKv := h.takeReusableSlot(key) + if useKv == nil { + h.appendHeaderInt(key, value, base) + } else { + useKv.value = h.reuseOrAppendInt(useKv.value, value, base) + } +} + // Set sets a key-value pair in the HTTP header. // Calling Set mangles the buffer. func (h *Header) Set(key, value string) { + useKv := h.takeReusableSlot(key) + if useKv == nil { + h.appendHeader(key, value) + } else { + useKv.value = h.reuseOrAppend(useKv.value, value) + } +} + +// takeReusableSlot returns the valid key-value entry for key with the largest +// value buffer (best candidate for in-place reuse) and invalidates any other +// entries sharing the key. Returns nil if the key is not present. +func (h *Header) takeReusableSlot(key string) *argsKV { hb := &h.hbuf var useKv *argsKV for i := 0; i < len(hb.headers); i++ { @@ -271,11 +310,7 @@ func (h *Header) Set(key, value string) { } } } - if useKv == nil { - h.appendHeader(key, value) - } else { - useKv.value = h.reuseOrAppend(useKv.value, value) - } + return useKv } // Get gets the first value of a key found in the headers. Use [Header.ForEach] to find multiple values corresponding to same key. diff --git a/http/httpraw/header_test.go b/http/httpraw/header_test.go index 4228225..2d94955 100644 --- a/http/httpraw/header_test.go +++ b/http/httpraw/header_test.go @@ -240,3 +240,186 @@ func TestHeaderSetBytesEmptyValue(t *testing.T) { t.Errorf("want empty value, got %q", got) } } + +// buffer/offset past uint16 range silently corrupts or panics. +// A header block > 64KiB pushes a field's offset past 65535. tokint truncation +// makes Get return the wrong window (silent corruption) or panic on slice bounds. +func TestHeader_LargeBufferOverflow(t *testing.T) { + const wantVal = "the-canary-value" + // Pad with a big header so the canary field lands past offset 65535. + pad := strings.Repeat("x", 70000) + raw := "GET / HTTP/1.1\r\n" + + "X-Pad: " + pad + "\r\n" + + "X-Canary: " + wantVal + "\r\n" + + "\r\n" + + var h Header + err := h.ParseBytes(false, []byte(raw)) + if err != nil { + // Clean rejection of the oversized header is the intended behavior: + // no panic, no silent corruption. + return + } + // If it did parse, the value must be correct (never a truncated-offset window). + got := string(h.Get("X-Canary")) + if got != wantVal { + t.Fatalf("overflow corruption: want X-Canary %q, got %q", wantVal, got) + } +} + +// a complete but malformed header line with no colon must be a hard error, +// not errNeedMore (which makes a streaming parser wait forever). +func TestHeader_ColonlessLineIsHardError(t *testing.T) { + raw := "GET / HTTP/1.1\r\nBadHeaderNoColon\r\n\r\n" + var h Header + err := h.ParseBytes(false, []byte(raw)) + if err == nil { + t.Fatal("want error on colonless header line, got nil") + } + if err == errNeedMore { + t.Fatalf("colonless line reported as errNeedMore (parser would hang); want a hard error like errInvalidName") + } +} + +// Regression for a TCP split landing BEFORE the colon (no newline yet) must +// stay "need more data" and parse once the rest arrives. This is the case the +// Colonless fix must NOT turn into a hard error. +func TestHeader_SplitBeforeColonStillParses(t *testing.T) { + const part1 = "GET / HTTP/1.1\r\nHost" // split mid-key, before colon+newline + const part2 = ": example.com\r\n\r\n" + + var h Header + h.Reset(nil) + if _, err := h.ReadFromBytes([]byte(part1)); err != nil { + t.Fatal(err) + } + needMore, err := h.TryParse(false) + if err != nil && err != errNeedMore { + t.Fatalf("split before colon: want errNeedMore/nil, got %v", err) + } + if !needMore { + t.Fatal("want needMoreData=true after partial input") + } + + if _, err := h.ReadFromBytes([]byte(part2)); err != nil { + t.Fatal(err) + } + needMore, err = h.TryParse(false) + if err != nil { + t.Fatalf("after full input: %v", err) + } + if needMore { + t.Fatal("want needMoreData=false after full input") + } + if got := string(h.Get("Host")); got != "example.com" { + t.Fatalf("want Host %q, got %q", "example.com", got) + } +} + +// appendHeader must reserve the +1 byte that mustAppendSlice consumes on an +// empty buffer. Force cap == len(key)+len(value) to defeat allocator rounding. +func TestHeader_AppendHeaderExactCapNoPanic(t *testing.T) { + const key, value = "K", "V" + buf := make([]byte, 0, len(key)+len(value)) // exact cap, no slack. + var h Header + h.Reset(buf) + defer func() { + if r := recover(); r != nil { + t.Fatalf("appendHeader panicked on exact-cap buffer: %v", r) + } + }() + h.Add(key, value) + if got := string(h.Get(key)); got != value { + t.Fatalf("want %q, got %q", value, got) + } +} + +// Add/Set on a full buffer with growth disabled must drop gracefully (flag OOM), +// never panic. Panicking is unacceptable for this package. +func TestHeader_AddFullBufferNoPanic(t *testing.T) { + buf := make([]byte, 0, 40) // Small cap; enough for Reset (len 0) but not the field below. + var h Header + h.Reset(buf) + h.EnableBufferGrowth(false) + h.SetMethod("GET") + h.SetRequestURI("/") + h.SetProtocol("HTTP/1.1") + + defer func() { + if r := recover(); r != nil { + t.Fatalf("Add on full no-grow buffer panicked: %v", r) + } + }() + h.Add("X-Very-Long-Header-Key", "a-value-that-cannot-possibly-fit-in-the-buffer") + + // Graceful drop: request marshalling reports OOM instead of emitting bad data. + if _, err := h.AppendRequest(nil); err == nil { + t.Fatal("want OOM error after dropped Add, got nil") + } +} + +func TestHeader_SetInt(t *testing.T) { + for _, tc := range []struct { + name string + value int64 + base int + want string + }{ + {"decimal", 1234, 10, "1234"}, + {"zero", 0, 10, "0"}, + {"negative", -42, 10, "-42"}, + {"maxint64", 9223372036854775807, 10, "9223372036854775807"}, + {"minint64", -9223372036854775808, 10, "-9223372036854775808"}, + {"hex", 255, 16, "ff"}, + } { + t.Run(tc.name, func(t *testing.T) { + var h Header + h.Reset(nil) + h.SetInt("Content-Length", tc.value, tc.base) + if got := string(h.Get("Content-Length")); got != tc.want { + t.Fatalf("want %q, got %q", tc.want, got) + } + }) + } +} + +// SetInt on an existing key must reuse the slot in place (single field, latest value). +func TestHeader_SetIntOverwrite(t *testing.T) { + var h Header + h.Reset(nil) + h.SetMethod("GET") + h.SetRequestURI("/") + h.SetProtocol("HTTP/1.1") + + h.SetInt("Content-Length", 100, 10) + h.SetInt("Content-Length", 5, 10) // shorter, must fit in old slot. + + if got := string(h.Get("Content-Length")); got != "5" { + t.Fatalf("want Content-Length %q, got %q", "5", got) + } + req, err := h.AppendRequest(nil) + if err != nil { + t.Fatal(err) + } + if n := strings.Count(string(req), "Content-Length:"); n != 1 { + t.Errorf("want 1 Content-Length field, got %d:\n%s", n, req) + } +} + +// SetInt must not heap-allocate: it must format directly into the header buffer. +func TestHeader_SetIntNoAlloc(t *testing.T) { + buf := make([]byte, 0, 256) + var h Header + h.Reset(buf) + h.EnableBufferGrowth(false) + h.Add("Content-Length", "0000000000000000000000") // pre-size a reusable slot. + allocs := testing.AllocsPerRun(100, func() { + h.SetInt("Content-Length", 1234567890, 10) + }) + if allocs != 0 { + t.Fatalf("SetInt allocated %v times, want 0", allocs) + } + if got := string(h.Get("Content-Length")); got != "1234567890" { + t.Fatalf("want %q, got %q", "1234567890", got) + } +} diff --git a/http/httpraw/parse.go b/http/httpraw/parse.go index 0b0bd26..82f4027 100644 --- a/http/httpraw/parse.go +++ b/http/httpraw/parse.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "slices" + "strconv" "unsafe" "github.com/soypat/lneto/internal" @@ -27,8 +28,14 @@ var ( errNeedMethodURI = errors.New("need method/request URI to create request header") errBadStatusCodeTxt = errors.New("invalid status code or text") errCookiesParsed = errors.New("cookies already parsed, reset before parsing again") + errBufferTooLarge = errors.New("httpraw: buffer exceeds max size (offsets are uint16)") ) +// maxBufLen bounds the header buffer. Offsets/lengths are stored as uint16 +// (tokint); a buffer past this would truncate/overflow those, silently +// returning the wrong bytes or panicking on a wrapped slice bound. +const maxBufLen = 0xffff + type headerBuf struct { // buf[:len] holds entire HTTP header data, which may be normalized by [flags]. buf[off:len] holds data not yet processed during parsing. buf []byte @@ -92,6 +99,9 @@ func (h *Header) parse(asResponse bool) (err error) { } func (h *Header) parseFirstLine(asResponse bool) (err error) { + if len(h.hbuf.buf) > maxBufLen { + return errBufferTooLarge // Offsets would overflow uint16 tokint. + } if asResponse { h.statusCode, h.statusText, h.flags, err = h.hbuf.parseFirstLineResponse(h.flags) } else { @@ -302,32 +312,22 @@ func (h *Header) reuseOrAppend(tok headerSlice, value string) headerSlice { func (h *Header) appendSlice(value string) headerSlice { debuglog("http:appendslice:start") - free := h.hbuf.free() - if len(value) > free { - if h.flags.hasAny(flagNoBufferGrow) { - h.flags |= flagOOMReached - return headerSlice{} - } - debuglog("http:appendslice:grow-buf") - h.hbuf.buf = slices.Grow(h.hbuf.buf, len(value)+1) // Grow 1 beyond due to slice validity. + if !h.reserve(len(value)) { + return headerSlice{} } h.flags |= flagMangledBuffer return h.hbuf.mustAppendSlice(value) } func (h *Header) appendHeader(key, value string) { - hb := &h.hbuf - free := hb.free() - buf := h.hbuf.buf - - if len(key)+len(value) > free { - if h.flags.hasAny(flagNoBufferGrow) { - panic(errSmallBuffer) - } - debuglog("http:appendhdr:grow-buf") - hb.buf = slices.Grow(buf, len(key)+len(value)) + // reserve accounts for the byte-0 reservation mustAppendSlice makes on an + // empty buffer, and drops (flagging OOM) rather than panicking when growth + // is disabled and space runs out. + if !h.reserve(len(key) + len(value)) { + return } h.flags |= flagMangledBuffer + hb := &h.hbuf k := hb.mustAppendSlice(key) v := hb.mustAppendSlice(value) debuglog("http:appendhdr:grow-hdrs") @@ -337,6 +337,100 @@ func (h *Header) appendHeader(key, value string) { }) } +// appendHeaderInt is appendHeader's integer counterpart: it appends key and the +// formatted integer value as a new header field. +func (h *Header) appendHeaderInt(key string, value int64, base int) { + n := intLen(value, base) + if !h.reserve(len(key) + n) { + return // Drop and flag OOM; never panic. + } + h.flags |= flagMangledBuffer + hb := &h.hbuf + k := hb.mustAppendSlice(key) + v := hb.mustAppendInt(value, base) + hb.headers = append(hb.headers, argsKV{ + key: k, + value: v, + }) +} + +// reserve ensures need free bytes are available in the buffer, growing it when +// permitted. It accounts for the byte-0 reservation on an empty buffer (see +// mustAppendSlice). It returns false and sets flagOOMReached when the space +// cannot be guaranteed: a tokint offset overflow, or a full buffer with +// flagNoBufferGrow set. +func (h *Header) reserve(need int) bool { + hb := &h.hbuf + if len(hb.buf) == 0 { + need++ // mustAppend* reserves byte 0 on an empty buffer. + } + if len(hb.buf)+need > maxBufLen { + h.flags |= flagOOMReached // Offsets would overflow uint16 tokint. + return false + } + if need > hb.free() { + if h.flags.hasAny(flagNoBufferGrow) { + h.flags |= flagOOMReached + return false + } + hb.buf = slices.Grow(hb.buf, need) + } + return true +} + +// reuseOrAppendInt writes value into tok's slot in place when it fits, avoiding +// any buffer growth; otherwise it appends a fresh slot. +func (h *Header) reuseOrAppendInt(tok headerSlice, value int64, base int) headerSlice { + n := intLen(value, base) + if int(tok.len) >= n { + // Reuse: format directly over the existing slot. No free space needed + // since n <= tok.len and the slot already lives inside buf. + v := strconv.AppendInt(h.hbuf.buf[tok.start:tok.start], value, base) + tok.len = tokint(len(v)) + h.flags |= flagMangledBuffer + return tok + } + return h.appendInt(value, base, n) +} + +// appendInt reserves space (growing or flagging OOM) and appends value as a new slot. +func (h *Header) appendInt(value int64, base, n int) headerSlice { + if !h.reserve(n) { + return headerSlice{} // Drop and flag OOM; never panic. + } + h.flags |= flagMangledBuffer + return h.hbuf.mustAppendInt(value, base) +} + +// mustAppendInt formats value into the buffer's free region and commits it. +// The caller must have reserved at least intLen(value, base) free bytes. +func (hb *headerBuf) mustAppendInt(value int64, base int) headerSlice { + L := len(hb.buf) + if L == 0 { + L++ // Valid key-values start after byte 0. + } + v := strconv.AppendInt(hb.buf[L:L], value, base) + hb.buf = hb.buf[:L+len(v)] + return hb.slice(hb.buf[L : L+len(v)]) +} + +// intLen returns the number of bytes strconv.AppendInt would emit for value in +// the given base (including a leading minus sign for negatives). Used to size +// the buffer and to test whether a value fits an existing slot without writing. +func intLen(value int64, base int) int { + n := 1 + u := uint64(value) + if value < 0 { + n++ // Leading minus sign. + u = -u // Two's-complement magnitude; correct even for math.MinInt64. + } + for u >= uint64(base) { + u /= uint64(base) + n++ + } + return n +} + func (hb *headerBuf) noKV() argsKV { return argsKV{} } func (hb *headerBuf) next(ss *scannerState) argsKV { @@ -373,8 +467,11 @@ func (hb *headerBuf) next(ss *scannerState) argsKV { ss.err = errInvalidName return hb.noKV() } else if n < 0 { - // No colon found, probably missing data. - ss.err = errNeedMore + // A newline is present (x>=0 reached here) but the line has no + // colon: malformed, not incomplete. A split arriving before the + // colon has no newline yet and is caught by the x<0 branch above, + // so it still returns errNeedMore. + ss.err = errInvalidName return hb.noKV() } }