diff --git a/README.md b/README.md index e46721e..fe65c02 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,14 @@ go mod download github.com/soypat/lneto@latest - [`xcurl`](./examples/xcurl) Contains example of a application that uses lneto and can attach to a linux tap/bridge interface or a [httptap](./examples/httptap)(with -ihttp flag) to work. When using httptap can be run as non-root user to be debugged comfortably. - Example: `go run ./examples/xcurl -host google.com -ihttp` +### LLM Policy / AI Policy +LLMs are a tool. As such they should be used carefully. The policy for this project is [covered in Oxide's RFD576](https://rfd.shared.oxide.computer/rfd/0576). + +Examples of LLM contribution in lneto: +- https://github.com/soypat/lneto/pull/19 and https://github.com/soypat/lneto/pull/18: Egon told me he was using an LLM to find bugs in lneto before submitting these PRs. +- https://github.com/soypat/lneto/commit/6b06cb1237071a0c22f86821db5aaa6135996e98: Writing tests to capture functionality in lneto that has been tested and works. +- https://github.com/soypat/lneto/commit/7042a653a6e46999314c2e36dd5c868bcbc68908: adding mutex locking on tcp.Conn + ### Quick run xcurl Run xcurl over httptap interface. Requires running two programs in separate shell/consoles in linux: ```sh diff --git a/http/httpraw/parse.go b/http/httpraw/parse.go index 8e7817f..92b46e0 100644 --- a/http/httpraw/parse.go +++ b/http/httpraw/parse.go @@ -159,10 +159,12 @@ func (hb *headerBuf) scanUntilByte(c byte) []byte { func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto headerSlice, flags flags, err error) { debuglog("http:req:scan") hb.off = 0 // Parsing first line resets offset. - var b []byte hb.skipLeadingCRLF() - b = hb.scanLine() flags = initFlags + if bytes.IndexByte(hb.offBuf(), '\n') < 0 { + return method, uri, proto, flags, errNeedMore // Incomplete line. + } + b := hb.scanLine() if len(b) < 5 { return method, uri, proto, flags, errNeedMore } @@ -173,18 +175,17 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto if reqURIEnd > 0 { reqURIEnd += methodEnd + 1 uri = hb.slice(b[methodEnd+1 : reqURIEnd]) - if b2s(b[methodEnd+1:reqURIEnd]) != strHTTP11 { + proto = hb.slice(b[reqURIEnd+1:]) // Skip space before protocol. + if b2s(b[reqURIEnd+1:]) != strHTTP11 { flags |= flagNoHTTP11 } } else if reqURIEnd == 0 { return method, uri, proto, flags, errEmptyURI } else { // No version provided. - reqURIEnd = methodEnd + 1 flags |= flagNoHTTP11 - uri = hb.slice(b[methodEnd+1 : reqURIEnd]) + uri = hb.slice(b[methodEnd+1:]) } - proto = hb.slice(b[reqURIEnd:]) method = hb.slice(b[:methodEnd]) return method, uri, proto, flags, nil } @@ -192,18 +193,32 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto func (hb *headerBuf) parseFirstLineResponse(initFlags flags) (statusCode, statusText headerSlice, flags flags, err error) { debuglog("http:resp:scan") hb.off = 0 // Parsing first line resets offset. - var b []byte hb.skipLeadingCRLF() - b = hb.scanLine() flags = initFlags + if bytes.IndexByte(hb.offBuf(), '\n') < 0 { + return statusCode, statusText, flags, errNeedMore // Incomplete line. + } + b := hb.scanLine() if len(b) < 5 { return statusCode, statusText, flags, errNeedMore } debuglog("http:resp:parse") - statusCodeEnd := max(0, bytes.IndexByte(b, ' ')) - code := b[:statusCodeEnd] - text := b[statusCodeEnd:] + // Parse protocol (e.g. "HTTP/1.1"), then status code, then status text. + protoEnd := bytes.IndexByte(b, ' ') + if protoEnd < 0 { + return statusCode, statusText, flags, errNeedMore + } + if b2s(b[:protoEnd]) != strHTTP11 { + flags |= flagNoHTTP11 + } + b = b[protoEnd+1:] // Advance past protocol and space. + + codeEnd := bytes.IndexByte(b, ' ') + if codeEnd < 0 { + codeEnd = len(b) // Status text is optional. + } + code := b[:codeEnd] if len(code) > 3 { return statusCode, statusText, flags, errLongStatusCode } @@ -214,7 +229,9 @@ func (hb *headerBuf) parseFirstLineResponse(initFlags flags) (statusCode, status } } statusCode = hb.slice(code) - statusText = hb.slice(text) + if codeEnd < len(b) { + statusText = hb.slice(b[codeEnd+1:]) // Skip space before text. + } debuglog("http:resp:done") return statusCode, statusText, flags, nil } @@ -398,7 +415,8 @@ func (hb *headerBuf) next(ss *scannerState) argsKV { // ConnectionClose returns true if 'Connection: close' header is set or if a invalid header was found. func (h *Header) ConnectionClose() bool { closed := h.flags.hasAny(flagConnClose) || - (h.flags.hasAny(flagNoHTTP11) && !h.hasHeaderValue("Connection", "keep-alive")) + h.hasHeaderValue(headerConnection, strClose) || + (h.flags.hasAny(flagNoHTTP11) && !h.hasHeaderValue(headerConnection, "keep-alive")) if closed { h.flags |= flagConnClose } diff --git a/http/httpraw/parse_test.go b/http/httpraw/parse_test.go new file mode 100644 index 0000000..fd66d9e --- /dev/null +++ b/http/httpraw/parse_test.go @@ -0,0 +1,544 @@ +package httpraw + +import ( + "bytes" + "strings" + "testing" +) + +func TestTryParse_IncrementalRequest(t *testing.T) { + // Full HTTP request split across multiple ReadFromBytes calls. + full := "GET /index.html HTTP/1.1\r\nHost: example.com\r\nContent-Type: text/html\r\n\r\nbody here" + var hdr Header + hdr.Reset(make([]byte, 0, 256)) + + // Feed data in small chunks to exercise incremental parsing. + chunks := splitInto(full, 10) + var done bool + var doneIdx int + for i, chunk := range chunks { + n, err := hdr.ReadFromBytes([]byte(chunk)) + if err != nil { + t.Fatalf("ReadFromBytes: %v", err) + } + if n != len(chunk) { + t.Fatalf("expected %d bytes read, got %d", len(chunk), n) + } + + var needMore bool + needMore, err = hdr.TryParse(false) + if err != nil && needMore { + continue // need more data + } + if err != nil { + t.Fatalf("TryParse: %v", err) + } + done = !needMore + doneIdx = i + break + } + + if !done { + t.Fatal("header parsing did not complete") + } + // Feed remaining chunks so body is complete. + for _, chunk := range chunks[doneIdx+1:] { + hdr.ReadFromBytes([]byte(chunk)) + } + if !hdr.ParsingSuccess() { + t.Fatal("ParsingSuccess should be true") + } + + if string(hdr.Method()) != "GET" { + t.Errorf("method = %q; want GET", hdr.Method()) + } + if string(hdr.RequestURI()) != "/index.html" { + t.Errorf("URI = %q; want /index.html", hdr.RequestURI()) + } + + // Verify headers via ForEach. + headers := make(map[string]string) + err := hdr.ForEach(func(key, value []byte) error { + headers[string(key)] = string(value) + return nil + }) + if err != nil { + t.Fatal(err) + } + if headers["Host"] != "example.com" { + t.Errorf("Host = %q; want example.com", headers["Host"]) + } + if headers["Content-Type"] != "text/html" { + t.Errorf("Content-Type = %q; want text/html", headers["Content-Type"]) + } + + // Verify body is accessible. + body, err := hdr.Body() + if err != nil { + t.Fatal(err) + } + if string(body) != "body here" { + t.Errorf("body = %q; want %q", body, "body here") + } +} + +func TestTryParse_IncrementalResponse(t *testing.T) { + full := "HTTP/1.1 200 OK\r\nContent-Length: 5\r\nServer: lneto\r\n\r\nhello" + var hdr Header + hdr.Reset(make([]byte, 0, 256)) + + chunks := splitInto(full, 8) + var done bool + var doneIdx int + for i, chunk := range chunks { + hdr.ReadFromBytes([]byte(chunk)) + needMore, err := hdr.TryParse(true) + if err != nil && needMore { + continue + } + if err != nil { + t.Fatalf("TryParse response: %v", err) + } + done = !needMore + doneIdx = i + break + } + + if !done { + t.Fatal("response parsing did not complete") + } + for _, chunk := range chunks[doneIdx+1:] { + hdr.ReadFromBytes([]byte(chunk)) + } + + code, text := hdr.Status() + if string(code) != "200" { + t.Errorf("status code = %q; want 200", code) + } + if !bytes.Contains(text, []byte("OK")) { + t.Errorf("status text = %q; want to contain OK", text) + } + + if string(hdr.Get("Server")) != "lneto" { + t.Errorf("Server header = %q; want lneto", hdr.Get("Server")) + } + + body, err := hdr.Body() + if err != nil { + t.Fatal(err) + } + if string(body) != "hello" { + t.Errorf("body = %q; want hello", body) + } +} + +func TestReadFromLimited(t *testing.T) { + data := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" + r := strings.NewReader(data) + + var hdr Header + hdr.Reset(make([]byte, 0, 256)) + + // Read in one shot. + n, err := hdr.ReadFromLimited(r, 256) + if err != nil { + t.Fatal(err) + } + if n != len(data) { + t.Fatalf("expected %d bytes, got %d", len(data), n) + } + + if hdr.BufferReceived() != len(data) { + t.Errorf("BufferReceived = %d; want %d", hdr.BufferReceived(), len(data)) + } + + err = hdr.Parse(false) + if err != nil { + t.Fatal(err) + } + if string(hdr.Method()) != "GET" { + t.Errorf("method = %q; want GET", hdr.Method()) + } +} + +func TestReadFromLimited_MaxBytes(t *testing.T) { + var hdr Header + hdr.Reset(make([]byte, 0, 256)) + + // Zero maxBytesToRead should error. + _, err := hdr.ReadFromLimited(strings.NewReader("data"), 0) + if err == nil { + t.Fatal("expected error for maxBytesToRead=0") + } +} + +func TestReadFromBytes_Empty(t *testing.T) { + var hdr Header + hdr.Reset(make([]byte, 0, 256)) + + _, err := hdr.ReadFromBytes(nil) + if err == nil { + t.Fatal("expected error for empty bytes") + } +} + +func TestBufferFreeAndCapacity(t *testing.T) { + var hdr Header + hdr.Reset(make([]byte, 0, 100)) + + if hdr.BufferCapacity() != 100 { + t.Errorf("capacity = %d; want 100", hdr.BufferCapacity()) + } + if hdr.BufferFree() != 100 { + t.Errorf("free = %d; want 100", hdr.BufferFree()) + } + + hdr.ReadFromBytes([]byte("1234567890")) + if hdr.BufferFree() != 90 { + t.Errorf("free after read = %d; want 90", hdr.BufferFree()) + } +} + +func TestEnableBufferGrowth(t *testing.T) { + var hdr Header + buf := make([]byte, 0, 64) + hdr.Reset(buf) + hdr.EnableBufferGrowth(false) + + // With growth disabled, reading more than capacity should fail. + big := make([]byte, 128) + for i := range big { + big[i] = 'A' + } + _, err := hdr.ReadFromBytes(big) + if err == nil { + t.Fatal("expected error when buffer growth disabled and data exceeds capacity") + } +} + +func TestHeader_Add(t *testing.T) { + full := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" + var hdr Header + err := hdr.ParseBytes(false, []byte(full)) + if err != nil { + t.Fatal(err) + } + + hdr.Add("X-Custom", "value1") + hdr.Add("X-Custom", "value2") + + // ForEach should find both. + var values []string + hdr.ForEach(func(key, value []byte) error { + if string(key) == "X-Custom" { + values = append(values, string(value)) + } + return nil + }) + if len(values) != 2 { + t.Fatalf("expected 2 X-Custom headers, got %d", len(values)) + } + if values[0] != "value1" || values[1] != "value2" { + t.Errorf("values = %v; want [value1 value2]", values) + } +} + +func TestHeader_SetBytes(t *testing.T) { + full := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" + var hdr Header + err := hdr.ParseBytes(false, []byte(full)) + if err != nil { + t.Fatal(err) + } + + hdr.SetBytes("X-Data", []byte("binary-value")) + got := hdr.Get("X-Data") + if string(got) != "binary-value" { + t.Errorf("X-Data = %q; want binary-value", got) + } +} + +func TestConnectionClose(t *testing.T) { + t.Run("HTTP11_NoConnectionHeader", func(t *testing.T) { + full := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" + var hdr Header + hdr.ParseBytes(false, []byte(full)) + if hdr.ConnectionClose() { + t.Error("HTTP/1.1 without Connection:close should not close") + } + }) + + t.Run("ExplicitClose", func(t *testing.T) { + full := "GET / HTTP/1.1\r\nConnection: close\r\nHost: test\r\n\r\n" + var hdr Header + hdr.ParseBytes(false, []byte(full)) + if !hdr.ConnectionClose() { + t.Error("Connection:close header should trigger close") + } + }) + + t.Run("HTTP10_NoKeepAlive", func(t *testing.T) { + full := "GET / HTTP/1.0\r\nHost: test\r\n\r\n" + var hdr Header + hdr.ParseBytes(false, []byte(full)) + if !hdr.ConnectionClose() { + t.Error("HTTP/1.0 without keep-alive should close") + } + }) + + t.Run("HTTP10_KeepAlive", func(t *testing.T) { + full := "GET / HTTP/1.0\r\nConnection: keep-alive\r\nHost: test\r\n\r\n" + var hdr Header + hdr.ParseBytes(false, []byte(full)) + if hdr.ConnectionClose() { + t.Error("HTTP/1.0 with keep-alive should not close") + } + }) +} + +func TestTryParse_AlreadyParsed(t *testing.T) { + full := "GET / HTTP/1.1\r\nHost: test\r\n\r\n" + var hdr Header + hdr.ParseBytes(false, []byte(full)) + + // Calling TryParse again should return error. + _, err := hdr.TryParse(false) + if err == nil { + t.Fatal("TryParse after completed parse should error") + } +} + +func TestParseResponse_BadStatusCode(t *testing.T) { + full := "HTTP/1.1 abc Bad\r\n\r\n" + var hdr Header + err := hdr.ParseBytes(true, []byte(full)) + if err == nil { + t.Fatal("expected error for non-numeric status code") + } +} + +func TestCookie_ParseBytes(t *testing.T) { + var c Cookie + err := c.ParseBytes([]byte("session=abc123; Path=/; Secure")) + if err != nil { + t.Fatal(err) + } + if string(c.Name()) != "session" { + t.Errorf("name = %q; want session", c.Name()) + } + if string(c.Value()) != "abc123" { + t.Errorf("value = %q; want abc123", c.Value()) + } + if string(c.Get("Path")) != "/" { + t.Errorf("Path = %q; want /", c.Get("Path")) + } + if !c.HasKeyOrSingleValue("Secure") { + t.Error("expected Secure flag") + } +} + +func TestCookie_CopyFrom(t *testing.T) { + var src Cookie + src.ParseBytes([]byte("key=val; HttpOnly")) + + var dst Cookie + dst.CopyFrom(src) + + if string(dst.Name()) != "key" { + t.Errorf("copied name = %q; want key", dst.Name()) + } + if string(dst.Value()) != "val" { + t.Errorf("copied value = %q; want val", dst.Value()) + } +} + +func TestCookie_ForEach(t *testing.T) { + var c Cookie + c.ParseBytes([]byte("a=1; b=2; c=3")) + + var keys []string + err := c.ForEach(func(key, value []byte) error { + keys = append(keys, string(key)) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(keys) != 3 { + t.Fatalf("expected 3 cookie entries, got %d", len(keys)) + } +} + +func TestHeader_MultilineValue(t *testing.T) { + // RFC 7230: obsolete line folding with \r\n followed by space/tab. + full := "GET / HTTP/1.1\r\nX-Multi: line1\r\n\tline2\r\nHost: test\r\n\r\n" + var hdr Header + err := hdr.ParseBytes(false, []byte(full)) + if err != nil { + t.Fatal(err) + } + val := hdr.Get("X-Multi") + if val == nil { + t.Fatal("X-Multi header not found") + } + // Value should contain both lines (raw, before normalization). + if !bytes.Contains(val, []byte("line1")) { + t.Error("missing line1 in multiline value") + } +} + +func TestHeader_ResponseRoundTrip(t *testing.T) { + var hdr Header + hdr.Reset(make([]byte, 0, 256)) + hdr.SetProtocol("HTTP/1.1") + hdr.SetStatus("404", "Not Found") + hdr.Add("Content-Type", "text/plain") + + buf, err := hdr.AppendResponse(nil) + if err != nil { + t.Fatal(err) + } + resp := string(buf) + if !strings.Contains(resp, "HTTP/1.1") { + t.Errorf("response missing protocol: %s", resp) + } + if !strings.Contains(resp, "404") { + t.Errorf("response missing status code: %s", resp) + } + if !strings.Contains(resp, "Not Found") { + t.Errorf("response missing status text: %s", resp) + } + if !strings.Contains(resp, "Content-Type: text/plain") { + t.Errorf("response missing header: %s", resp) + } + + // Parse back the generated response. + var hdr2 Header + err = hdr2.ParseBytes(true, buf) + if err != nil { + t.Fatalf("re-parse response: %v", err) + } + code, text := hdr2.Status() + if string(code) != "404" { + t.Errorf("re-parsed code = %q; want 404", code) + } + if string(text) != "Not Found" { + t.Errorf("re-parsed text = %q; want Not Found", text) + } +} + +func TestHeader_RequestRoundTrip(t *testing.T) { + var hdr Header + hdr.Reset(make([]byte, 0, 256)) + hdr.SetProtocol("HTTP/1.1") + hdr.SetMethod("POST") + hdr.SetRequestURI("/api/data") + hdr.Add("Host", "example.com") + hdr.Add("Content-Type", "application/json") + + buf, err := hdr.AppendRequest(nil) + if err != nil { + t.Fatal(err) + } + req := string(buf) + if !strings.HasPrefix(req, "POST /api/data HTTP/1.1\r\n") { + t.Errorf("unexpected request line: %s", req) + } + + // Parse back the generated request. + var hdr2 Header + err = hdr2.ParseBytes(false, buf) + if err != nil { + t.Fatalf("re-parse request: %v", err) + } + if string(hdr2.Method()) != "POST" { + t.Errorf("re-parsed method = %q; want POST", hdr2.Method()) + } + if string(hdr2.RequestURI()) != "/api/data" { + t.Errorf("re-parsed URI = %q; want /api/data", hdr2.RequestURI()) + } + if string(hdr2.Get("Host")) != "example.com" { + t.Errorf("re-parsed Host = %q; want example.com", hdr2.Get("Host")) + } +} + +func TestParseResponse_StatusCodeOnly(t *testing.T) { + // Response with status code but no status text. + full := "HTTP/1.1 204\r\n\r\n" + var hdr Header + err := hdr.ParseBytes(true, []byte(full)) + if err != nil { + t.Fatal(err) + } + code, text := hdr.Status() + if string(code) != "204" { + t.Errorf("code = %q; want 204", code) + } + _ = text // Text may be empty, that's OK. +} + +func TestParseResponse_HTTP10(t *testing.T) { + full := "HTTP/1.0 200 OK\r\nServer: old\r\n\r\n" + var hdr Header + err := hdr.ParseBytes(true, []byte(full)) + if err != nil { + t.Fatal(err) + } + code, _ := hdr.Status() + if string(code) != "200" { + t.Errorf("code = %q; want 200", code) + } + if !hdr.ConnectionClose() { + t.Error("HTTP/1.0 response should default to connection close") + } +} + +func TestParseRequest_NoProtocol(t *testing.T) { + // HTTP/0.9 style: just method and URI, no version. + full := "GET /simple\r\nHost: test\r\n\r\n" + var hdr Header + err := hdr.ParseBytes(false, []byte(full)) + if err != nil { + t.Fatal(err) + } + if string(hdr.Method()) != "GET" { + t.Errorf("method = %q; want GET", hdr.Method()) + } + if string(hdr.RequestURI()) != "/simple" { + t.Errorf("URI = %q; want /simple", hdr.RequestURI()) + } + if hdr.Protocol() != nil { + t.Errorf("protocol should be nil for version-less request, got %q", hdr.Protocol()) + } +} + +func TestCookie_QuotedValue(t *testing.T) { + var c Cookie + c.ParseBytes([]byte("token=\"abc123\"")) + if string(c.Value()) != "abc123" { + t.Errorf("quoted value = %q; want abc123", c.Value()) + } +} + +func TestParseRequest_InvalidHeaderSpaceBeforeColon(t *testing.T) { + // RFC 7230 ยง3.2.4: No whitespace allowed between header name and colon. + full := "GET / HTTP/1.1\r\nBad Header : value\r\n\r\n" + var hdr Header + err := hdr.ParseBytes(false, []byte(full)) + if err == nil { + t.Fatal("expected error for space before colon in header name") + } +} + +func splitInto(s string, n int) []string { + var chunks []string + for len(s) > 0 { + end := n + if end > len(s) { + end = len(s) + } + chunks = append(chunks, s[:end]) + s = s[end:] + } + return chunks +}