Further improvements to httphi API (#173)

* add Exchange.WriteBodyString

* Mux.MaxPathValues and other improvements

* Mux PathValue improvemnt and fixes

* MuxSlice more method muxing improvements

* diagram out interesting approach to form parsing for clanker

* refactor RequestParseForm and achieve greatness in API design

* explicit naming of headerCapacityKV value in kvBuffer.Reset

* fix Mux bug not matching paths correctly; httpraw HTTP V1 naming applied

* rename many examples,use httphi in examples,remove useless maxAwaitingConn field

* add ipv4.String

* add ipv4 UnspecifiedAddr and BroadcastAddr

* add ethernet.String
This commit is contained in:
Pat Whittingslow
2026-07-31 12:47:33 -03:00
committed by GitHub
parent 5c54030f19
commit 1884cfc9b7
35 changed files with 2192 additions and 406 deletions
@@ -274,7 +274,7 @@ func handleConnNet(conn net.Conn) error {
defer conn.Close()
conn.SetDeadline(time.Now().Add(10 * time.Second))
var hdr httpraw.Header
var hdr httpraw.HeaderV1
needMore := true
for needMore {
_, err := hdr.ReadFromLimited(conn, 1024)
@@ -291,7 +291,7 @@ func handleConnNet(conn net.Conn) error {
uri := string(hdr.RequestTarget())
fmt.Printf("< %s %s\n", method, uri)
var resp httpraw.Header
var resp httpraw.HeaderV1
resp.SetProtocol("HTTP/1.1")
resp.SetStatus("200", "OK")
resp.Set("Content-Type", "text/html")
@@ -368,7 +368,7 @@ func mockClient(stack *xnet.StackAsync, port uint16, subnet netip.Prefix) {
panic("mock client deadline exceeded to establish")
}
var hdr httpraw.Header
var hdr httpraw.HeaderV1
hdr.SetMethod("GET")
hdr.SetRequestTarget("/")
hdr.SetProtocol("HTTP/1.1")
+259
View File
@@ -0,0 +1,259 @@
#!/usr/bin/env bash
# fuzz.sh points ffuf at an already running main-httplinux server. Start it
# yourself first, in another terminal, so its log and any crash stay visible:
#
# go run ./examples/http-linux -port 8080
# ./examples/http-linux/fuzz.sh # all cases
# ./examples/http-linux/fuzz.sh paths cookie # named cases only
# URL=http://localhost:9000 ./examples/http-linux/fuzz.sh
#
# Install ffuf with: go install github.com/ffuf/ffuf/v2@latest
#
# Every case ends by reporting the server is still up: a case that "finds
# nothing" because the process died is the failure this is looking for.
set -u
URL="${URL:-http://localhost:8080}"
# Matched to the server's FixedNumGoroutines: in worker mode the router owns one
# exchange per goroutine and refuses a connection outright when none is free, so
# that count is what bounds concurrency. Going above it is correct backpressure,
# but it reaches ffuf as a connection error and hides the response a case was
# looking for. Raise it to exercise the drop path.
THREADS="${THREADS:-4}"
WORDDIR="$(mktemp -d)"
trap 'rm -rf "$WORDDIR"' EXIT
# ---------------------------------------------------------------------------
# Wordlists. Kept here rather than pulled from SecLists so a run is repeatable
# and every entry is aimed at the parser: percent escapes, separators the
# grammar gives meaning to, and lengths that cross the server's fixed buffers.
# ---------------------------------------------------------------------------
cat >"$WORDDIR/paths.txt" <<'EOF'
admin
login
search
health
echo
upload
users
files
users/alice
users/bob
users/carol
users/mallory
users/al%69ce
users/%zz
users/%2e%2e%2f
users/alice/extra
files/
files/readme.txt
files/logo.png
files/notes.md
files/a/b/c
files/%2e%2e/%2e%2e/etc/passwd
EOF
cat >"$WORDDIR/queries.txt" <<'EOF'
go
go+lang
go%20lang
%21%40%23
%zz
%
%2
a=b
a&b
a;b
""
EOF
cat >"$WORDDIR/passwords.txt" <<'EOF'
hunter2
password
admin
letmein
hunter2%00
hunter2+
hun%74er2
EOF
cat >"$WORDDIR/tokens.txt" <<'EOF'
s3cr3t-session-token
admin
""
"s3cr3t-session-token"
s3cr3t-session-token; debug
s3cr3t-session-token;debug
=====
;;;;;
EOF
cat >"$WORDDIR/headers.txt" <<'EOF'
plain
with spaces
%00%01%02
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
EOF
cat >"$WORDDIR/names.txt" <<'EOF'
a.bin
report.pdf
../escape.txt
%2e%2e%2fescape.txt
EOF
# grow prints a line of n 'A's, for the cases that walk a value past a buffer.
grow() { printf 'A%.0s' $(seq "$1"); printf '\n'; }
{ for n in 8 64 512 1024 2048 4096 8192; do grow "$n"; done; } >"$WORDDIR/long.txt"
alive() {
if curl -s -o /dev/null --max-time 5 "$URL/health"; then
printf ' server alive\n\n'
else
printf ' *** SERVER DOWN after this case ***\n\n'
exit 1
fi
}
case_header() { printf '=== %s: %s\n' "$1" "$2"; }
# ---------------------------------------------------------------------------
# Cases. Each one drives a different part of the request through the parser.
# ---------------------------------------------------------------------------
# paths walks the mux: literal patterns, the "{id}" single-segment wildcard and
# the "{path...}" wildcard that swallows slashes. -mc all because a 404 from an
# unregistered path is a correct answer worth seeing next to the 200s.
fuzz_paths() {
case_header paths "mux patterns, wildcards and percent escapes in the path"
ffuf -u "$URL/FUZZ" -w "$WORDDIR/paths.txt" -t "$THREADS" -s -timeout 5 -mc all -fc 404
alive
}
# recursion follows the "{path...}" wildcard down, which is the pattern a
# directory scanner exercises hardest.
fuzz_recursion() {
case_header recursion "\"{path...}\" wildcard walked recursively"
ffuf -u "$URL/files/FUZZ" -w "$WORDDIR/paths.txt" -t "$THREADS" -s -timeout 5 \
-recursion -recursion-depth 2 -recursion-strategy greedy -mc all -fc 404
alive
}
# longpath pushes the request-target past RequestHeaderBufferSize. The server
# should answer 431 or drop the connection, never serve a mangled path.
fuzz_longpath() {
case_header longpath "request-target grown past the request header buffer"
ffuf -u "$URL/FUZZ" -w "$WORDDIR/long.txt" -t 4 -s -timeout 5 -mc all
alive
}
# query drives RequestQueryValue and the percent decoder, including escapes that
# do not decode, which must come back 400 and not half decoded.
fuzz_query() {
case_header query "query string values, valid and malformed escapes"
ffuf -u "$URL/search?q=FUZZ" -w "$WORDDIR/queries.txt" -t "$THREADS" -s -timeout 5 -mc all
ffuf -u "$URL/search?q=go&limit=FUZZ" -w "$WORDDIR/queries.txt" -t "$THREADS" -s -timeout 5 -mc all
case_header query "query value grown past the request header buffer"
ffuf -u "$URL/search?q=FUZZ" -w "$WORDDIR/long.txt" -t 4 -s -timeout 5 -mc all
alive
}
# form posts "application/x-www-form-urlencoded" bodies, the case the credential
# check answers 200 for and everything else 401.
fuzz_form() {
case_header form "urlencoded body pairs; 200 is the credential that works"
ffuf -u "$URL/login" -X POST -w "$WORDDIR/passwords.txt" \
-H 'Content-Type: application/x-www-form-urlencoded' \
-d 'user=admin&pass=FUZZ' -t "$THREADS" -s -timeout 5 -mc all -fc 401
case_header form "body grown past the form buffer, which may not grow"
ffuf -u "$URL/login" -X POST -w "$WORDDIR/long.txt" \
-H 'Content-Type: application/x-www-form-urlencoded' \
-d 'user=admin&pass=FUZZ' -t 4 -s -timeout 5 -mc all
case_header form "pair count driven past the form's fixed pair table"
ffuf -u "$URL/login?a=1&b=2&c=3&d=4&e=5&f=6&g=7&h=8&i=9&j=10&k=11&l=12&m=13&n=14&o=15&p=16&q=17" \
-X POST -w "$WORDDIR/passwords.txt" \
-H 'Content-Type: application/x-www-form-urlencoded' \
-d 'user=admin&pass=FUZZ' -t "$THREADS" -s -timeout 5 -mc all
alive
}
# cookie drives the Cookie header parser: quoting, valueless attributes and the
# separators the grammar splits on.
fuzz_cookie() {
case_header cookie "Cookie header values; 200 is the session that works"
ffuf -u "$URL/admin" -w "$WORDDIR/tokens.txt" -b 'session=FUZZ' \
-t "$THREADS" -s -timeout 5 -mc all -fc 403
case_header cookie "cookie grown past the cookie buffer"
ffuf -u "$URL/admin" -w "$WORDDIR/long.txt" -b 'session=FUZZ' -t 4 -s -timeout 5 -mc all
alive
}
# headers fuzzes a header field value and the field count, /echo handing back
# the header block as the parser stored it.
fuzz_headers() {
case_header headers "header field values echoed back through the parser"
ffuf -u "$URL/echo" -w "$WORDDIR/headers.txt" -H 'X-Fuzz: FUZZ' \
-t "$THREADS" -s -timeout 5 -mc all
case_header headers "header value grown past the request header buffer"
ffuf -u "$URL/echo" -w "$WORDDIR/long.txt" -H 'X-Fuzz: FUZZ' -t 4 -s -timeout 5 -mc all
alive
}
# multipart fuzzes the part header block: the filename parameter picks whether a
# part is streamed to a sink or discarded.
fuzz_multipart() {
case_header multipart "multipart part headers and filenames"
ffuf -u "$URL/upload" -X POST -w "$WORDDIR/names.txt" \
-H 'Content-Type: multipart/form-data; boundary=X' \
-d $'--X\r\nContent-Disposition: form-data; name="f"; filename="FUZZ"\r\n\r\ndata\r\n--X--\r\n' \
-t "$THREADS" -s -timeout 5 -mc all
case_header multipart "part header grown past the multipart buffer, expect 413"
ffuf -u "$URL/upload" -X POST -w "$WORDDIR/long.txt" \
-H 'Content-Type: multipart/form-data; boundary=X' \
-d $'--X\r\nContent-Disposition: form-data; name="f"; filename="FUZZ"\r\n\r\ndata\r\n--X--\r\n' \
-t 4 -s -timeout 5 -mc all
alive
}
# methods sends a method per registration and a few the server never names.
# "/echo" is registered without one, so any method reaches it; "/login" is
# POST only and everything else must 404 there.
fuzz_methods() {
case_header methods "registered, unregistered and extension methods"
printf 'GET\nPOST\nPUT\nDELETE\nPATCH\nHEAD\nOPTIONS\nTRACE\nPROPFIND\nBREW\n' >"$WORDDIR/methods.txt"
ffuf -u "$URL/echo" -w "$WORDDIR/methods.txt" -X FUZZ -t "$THREADS" -s -timeout 5 -mc all
ffuf -u "$URL/login" -w "$WORDDIR/methods.txt" -X FUZZ -t "$THREADS" -s -timeout 5 -mc all -fc 404
alive
}
# clusterbomb crosses a path wordlist with a query wordlist, so the two parsers
# are driven by unrelated inputs in the same request.
fuzz_clusterbomb() {
case_header clusterbomb "path and query fuzzed together, every combination"
ffuf -u "$URL/PATH?q=QUERY" -mode clusterbomb \
-w "$WORDDIR/paths.txt:PATH" -w "$WORDDIR/queries.txt:QUERY" \
-t "$THREADS" -s -timeout 5 -mc all -fc 404
alive
}
ALL=(paths recursion longpath query form cookie headers multipart methods clusterbomb)
main() {
command -v ffuf >/dev/null || {
echo "ffuf not found: go install github.com/ffuf/ffuf/v2@latest" >&2
exit 1
}
curl -s -o /dev/null --max-time 5 "$URL/health" || {
echo "no server at $URL: start it with 'go run ./examples/http-linux'" >&2
exit 1
}
local cases=("$@")
[ ${#cases[@]} -eq 0 ] && cases=("${ALL[@]}")
for c in "${cases[@]}"; do
"fuzz_$c" || { echo "unknown case: $c" >&2; exit 1; }
done
echo "all cases done, server still up"
}
main "$@"
+475
View File
@@ -0,0 +1,475 @@
//go:build !tinygo && linux
package main
import (
"flag"
"io"
"log/slog"
"net"
"os"
"strconv"
"sync/atomic"
"time"
"github.com/soypat/lneto"
"github.com/soypat/lneto/http/httphi"
"github.com/soypat/lneto/http/httpraw"
)
const (
kB = 1 << 10
listenPort = 8080
bufferSizes = 2 * kB
// A browser sends around twenty header fields; a request carrying more
// than this is answered 431 rather than parsed into memory it was not
// given. Each field costs 8 bytes of table.
numHeaderFields = 32
readTimeout = 2 * time.Second
)
// Credentials the endpoints check. They are in the source on purpose: this is a
// target to point a fuzzer at, and a scan is only interesting when something is
// there to be found.
const (
adminUser = "admin"
adminPass = "hunter2"
sessionToken = "s3cr3t-session-token"
)
// Fixed corpora the handlers answer from, so a path scan separates hits from
// misses instead of finding one status code everywhere.
var (
users = [...]string{"alice", "bob", "carol"}
files = [...]string{"readme.txt", "logo.png", "notes.md"}
)
var (
flagPort = flag.Int("port", listenPort, "TCP port to listen on")
flagVerbose = flag.Bool("v", false, "log every request to stderr; a fuzzer at full rate makes this expensive")
flagThreads = flag.Int("threads", 8, "Number of goroutines to spawn.")
)
func main() {
flag.Parse()
if err := run(); err != nil {
println("Error:", err.Error())
os.Exit(1)
}
println("DONE")
}
func run() error {
ln, err := net.Listen("tcp", ":"+strconv.Itoa(*flagPort))
if err != nil {
return err
}
defer ln.Close()
print("listening on http://localhost:", *flagPort, "\n")
var server Server
// One scratch per router goroutine: the router serves that many requests at
// once, so a handler always finds one waiting for it.
server.initScratch(*flagThreads)
// "{$}" matches the empty path and nothing else, so an unregistered path
// gets a 404 instead of the homepage. A bare "/" is a catch-all.
server.Handle("GET /{$}", server.homepage)
server.Handle("GET /health", server.health)
server.Handle("GET /search", server.search)
server.Handle("POST /login", server.login)
server.Handle("GET /admin", server.admin)
server.Handle("GET /users/{id}", server.user)
server.Handle("GET /files/{path...}", server.file)
server.Handle("POST /upload", server.upload)
server.Handle("/echo", server.echo) // No method: any method matches.
var router httphi.Router
err = router.Configure(httphi.RouterConfig{
FixedNumGoroutines: *flagThreads,
RequestHeaderBufferSize: bufferSizes,
RequestNumHeaderKVCap: numHeaderFields,
ResponseHeaderMinBufferSize: bufferSizes,
Mux: &server.mux,
Logger: slog.Default(),
})
if err != nil {
return err
}
defer router.Shutdown()
for {
conn, err := ln.Accept()
if err != nil {
return err
}
// The connection owns the idle policy: a peer that opens a socket and
// then stalls fails its read instead of holding a router goroutine.
conn.SetReadDeadline(time.Now().Add(readTimeout))
err = router.Handle(conn)
if err != nil {
// Every goroutine is busy and the queue is full. Dropping the
// connection is the backpressure: memory stays bounded.
slog.Warn("dropped connection", slog.String("remote", conn.RemoteAddr().String()), slog.String("err", err.Error()))
conn.Close()
}
}
}
const (
htmlHead = `<html><body bgcolor="#000080" text="#00FF00"><center>` +
`<marquee><font face="Comic Sans MS" size="5" color="#FFFF00">` +
`*** WELCOME TO MY HOMEPAGE ***</font></marquee>` +
`<h1><blink>YOU ARE VISITOR #`
htmlTail = `!</blink></h1>` +
`<font color="#FF00FF">Sign my guestbook!</font><br><hr>` +
`<a href="/search?q=go">/search</a> | ` +
`<a href="/users/alice">/users/{id}</a> | ` +
`<a href="/files/">/files/</a> | ` +
`<a href="/admin">/admin</a> | ` +
`<a href="/echo">/echo</a> | ` +
`<a href="/health">/health</a>` +
`<br><hr>Best viewed in Netscape Navigator</center></body></html>`
// maxPage bounds the rendered page: both halves plus the visitor number.
maxPage = len(htmlHead) + 20 + len(htmlTail)
)
// Compile-time check that a scratch's render buffer holds the largest page a
// handler builds. A page outgrowing it would grow the buffer on the heap,
// which is the one thing this server is written not to do.
const _ = uint(outBufferSize - maxPage)
type Server struct {
// Visits counts served requests.
Visits atomic.Uint64
mux httphi.MuxSlice
// scratch is a fixed pool of per-request working memory, see [scratch].
scratch chan *scratch
}
// Sizes of a [scratch]. Every one of these is memory spent once per pooled
// scratch, so the pool's size times the sum below is what the handlers cost.
const (
formBufferSize = kB
formNumPairs = 16
cookieBufferSize = 512
cookieNumPairs = 8
multipartBufferSize = kB
maxMultipartParts = 8
outBufferSize = 2 * kB
tmpBufferSize = 256
)
// scratch is the memory a handler works in for the length of one request: the
// [Exchange] buffer holds the request header and the response header, and
// everything a handler parses or renders on top of that lives here.
//
// Parsers are handed their buffer once and forbidden to grow, so a request that
// sends more than the buffer holds is answered an error rather than served from
// memory the server never budgeted for.
type scratch struct {
form httpraw.Form
cookie httpraw.Cookie
// parts is reused across requests: its [httpraw.MultipartHeader] values keep
// the buffers their Name and Filename were copied into.
parts []httphi.MultipartSink
uploads countingSink
formBuf [formBufferSize]byte
cookieBuf [cookieBufferSize]byte
mpBuf [multipartBufferSize]byte
// out renders the response body, tmp holds a decoded value being read out of
// the request. They are separate because a decode reads into one while the
// body is being built in the other.
out [outBufferSize]byte
tmp [tmpBufferSize]byte
}
// initScratch fills the pool with n scratches and fixes each parser to its
// buffer. Sizing n to the router's goroutine count bounds handler memory the
// same way the router bounds its own.
func (sv *Server) initScratch(n int) {
sv.scratch = make(chan *scratch, n)
for range n {
s := new(scratch)
s.form.Reset(s.formBuf[:0], formNumPairs)
s.form.EnableBufferGrowth(false)
s.cookie.Reset(s.cookieBuf[:0], cookieNumPairs)
s.cookie.EnableBufferGrowth(false)
s.parts = make([]httphi.MultipartSink, 0, maxMultipartParts)
sv.scratch <- s
}
}
// acquireScratch takes a scratch out of the pool, blocking while none is free.
// With the pool sized to the router's fixed goroutine count it never blocks:
// a handler running is a goroutine that has not returned its scratch yet.
func (sv *Server) acquireScratch() *scratch { return <-sv.scratch }
func (sv *Server) releaseScratch(s *scratch) { sv.scratch <- s }
// Handle registers a handler and wraps it in the middleware every request runs
// through: the visit counter and, when asked for, the request log.
func (sv *Server) Handle(pattern string, handler httphi.HandlerFunc) {
sv.mux.Handle(pattern, func(exch *httphi.Exchange) {
sv.Visits.Add(1)
if *flagVerbose {
println(exch.RequestMethod().String(), exch.MuxPattern())
}
handler(exch)
})
}
func (sv *Server) homepage(exch *httphi.Exchange) {
s := sv.acquireScratch()
defer sv.releaseScratch(s)
page := append(s.out[:0], htmlHead...)
page = strconv.AppendUint(page, sv.Visits.Load(), 10)
page = append(page, htmlTail...)
exch.Respond(httphi.StatusOK, "text/html", page)
}
func (sv *Server) health(exch *httphi.Exchange) {
exch.RespondString(httphi.StatusOK, "text/plain", "ok\n")
}
// search reads the query string, i.e: "/search?q=go+lang&limit=2". Values are
// percent and '+' encoded on the wire, so this is the decoder's surface: a
// malformed escape is answered 400 and never half decoded into the response.
func (sv *Server) search(exch *httphi.Exchange) {
s := sv.acquireScratch()
defer sv.releaseScratch(s)
const decode = true
query, present := exch.RequestQueryAppend(s.tmp[:0], "q", decode)
if !present {
exch.RespondString(httphi.StatusBadRequest, "text/plain", "missing or malformed q parameter\n")
return
}
limit := len(users)
if raw, present := exch.RequestQueryValue("limit"); present {
n, ok := atoiBounded(raw, len(users))
if !ok {
exch.RespondString(httphi.StatusBadRequest, "text/plain", "limit must be a non-negative integer\n")
return
}
limit = n
}
body := append(s.out[:0], "query: "...)
body = append(body, query...)
body = append(body, '\n')
for _, user := range users[:limit] {
body = append(body, user...)
body = append(body, '\n')
}
exch.Respond(httphi.StatusOK, "text/plain", body)
}
// login reads "application/x-www-form-urlencoded" pairs out of the request body
// and the query string alike, the body winning a key both carry. It is where a
// credential scan lands:
//
// ffuf -X POST -u http://localhost:8080/login -d 'user=admin&pass=FUZZ' \
// -H 'Content-Type: application/x-www-form-urlencoded' -w passwords.txt -fc 401
func (sv *Server) login(exch *httphi.Exchange) {
s := sv.acquireScratch()
defer sv.releaseScratch(s)
// Checked ahead of the parse so a body in some other encoding is told what
// is wrong with it, the parser reporting only that it would not parse.
contentType := exch.RequestContentType()
if contentType != nil && !httpraw.MediaTypeIs(contentType, "application/x-www-form-urlencoded") {
exch.RespondString(httphi.StatusUnsupportedMediaType, "text/plain", "expected application/x-www-form-urlencoded\n")
return
}
const parseQuery, queryWins = true, false
err := exch.RequestParseForm(&s.form, parseQuery, queryWins)
if err != nil {
// A body larger than formBufferSize or more pairs than formNumPairs land
// here too: the form was told not to grow, so it refuses instead.
exch.RespondString(httphi.StatusBadRequest, "text/plain", "malformed or oversized form\n")
return
}
if err = s.form.Decode(); err != nil {
exch.RespondString(httphi.StatusBadRequest, "text/plain", "malformed percent escape in form\n")
return
}
user, pass := s.form.Get("user"), s.form.Get("pass")
if string(user) != adminUser || string(pass) != adminPass {
exch.RespondString(httphi.StatusUnauthorized, "text/plain", "bad credentials\n")
return
}
exch.StageHeader("Set-Cookie", "session="+sessionToken+"; Path=/; HttpOnly")
exch.RespondString(httphi.StatusOK, "text/plain", "welcome "+adminUser+"\n")
}
// admin is gated on the cookie [Server.login] hands out, so a scan of it fuzzes
// the cookie parser:
//
// ffuf -u http://localhost:8080/admin -b 'session=FUZZ' -w tokens.txt -fc 403
func (sv *Server) admin(exch *httphi.Exchange) {
s := sv.acquireScratch()
defer sv.releaseScratch(s)
err := exch.RequestParseCookie(&s.cookie, "Cookie")
if err != nil {
exch.RespondString(httphi.StatusUnauthorized, "text/plain", "no cookie\n")
return
}
if string(s.cookie.Get("session")) != sessionToken {
exch.RespondString(httphi.StatusForbidden, "text/plain", "forbidden\n")
return
}
body := append(s.out[:0], "admin panel\n"...)
// A valueless attribute, i.e: "session=...; debug", is stored with an empty
// key, so a plain Get would never find it.
if s.cookie.HasKeyOrSingleValue("debug") {
body = append(body, "requests served: "...)
body = strconv.AppendUint(body, sv.Visits.Load(), 10)
body = append(body, '\n')
}
exch.Respond(httphi.StatusOK, "text/plain", body)
}
// user serves the "{id}" wildcard, a single path segment. Segments are bound
// raw, so the value is decoded here and "/users/al%69ce" reaches alice.
func (sv *Server) user(exch *httphi.Exchange) {
s := sv.acquireScratch()
defer sv.releaseScratch(s)
const decode = true
id, err := exch.PathValueAppend(s.tmp[:0], "id", decode)
if err != nil {
exch.RespondString(httphi.StatusBadRequest, "text/plain", "malformed percent escape in path\n")
return
}
for _, user := range users {
if string(id) == user {
body := append(s.out[:0], `{"user":"`...)
body = append(body, id...)
body = append(body, "\"}\n"...)
exch.Respond(httphi.StatusOK, "application/json", body)
return
}
}
exch.RespondString(httphi.StatusNotFound, "text/plain", "no such user\n")
}
// file serves the "{path...}" wildcard, which takes the rest of the path
// including its slashes, so "/files/" and "/files/a/b" both reach here. That
// makes it what a recursive scan walks: ffuf -u http://localhost:8080/files/FUZZ -recursion.
func (sv *Server) file(exch *httphi.Exchange) {
s := sv.acquireScratch()
defer sv.releaseScratch(s)
path := exch.PathValue("path")
if len(path) == 0 {
body := append(s.out[:0], "index of /files/\n"...)
for _, file := range files {
body = append(body, file...)
body = append(body, '\n')
}
exch.Respond(httphi.StatusOK, "text/plain", body)
return
}
for _, file := range files {
if string(path) == file {
body := append(s.out[:0], "contents of "...)
body = append(body, path...)
body = append(body, '\n')
exch.Respond(httphi.StatusOK, "text/plain", body)
return
}
}
exch.RespondString(httphi.StatusNotFound, "text/plain", "no such file\n")
}
// upload streams a "multipart/form-data" body, counting each file part instead
// of storing it. Parts declare no length, so the body is read a bufferful at a
// time and the header of a part that outgrows the buffer is refused 413.
//
// ffuf -X POST -u http://localhost:8080/upload -w names.txt \
// -H 'Content-Type: multipart/form-data; boundary=X' \
// -d $'--X\r\nContent-Disposition: form-data; name="f"; filename="FUZZ"\r\n\r\ndata\r\n--X--\r\n'
func (sv *Server) upload(exch *httphi.Exchange) {
s := sv.acquireScratch()
defer sv.releaseScratch(s)
if !httpraw.MediaTypeIs(exch.RequestContentType(), "multipart/form-data") {
exch.RespondString(httphi.StatusUnsupportedMediaType, "text/plain", "expected multipart/form-data\n")
return
}
s.uploads.n = 0
// The sink is a field of the scratch, so handing it over as an io.WriteCloser
// boxes a pointer that is already on the heap and allocates nothing.
parts, err := exch.ReadMultiparts(s.parts[:0], s.mpBuf[:], func(hdr *httpraw.MultipartHeader) io.WriteCloser {
if len(hdr.Filename) == 0 {
return nil // A plain field, not a file: keep the header, drop the content.
}
return &s.uploads
})
// Kept even on failure: the headers parsed so far own buffers worth reusing.
// The slice grows with the number of parts, which only the connection's read
// deadline bounds, so a real server would cap it.
s.parts = parts
if err != nil {
if err == lneto.ErrShortBuffer {
exch.RespondString(httphi.StatusRequestEntityTooLarge, "text/plain", "part header too large\n")
} else {
exch.RespondString(httphi.StatusBadRequest, "text/plain", "malformed multipart body\n")
}
return
}
body := append(s.out[:0], "parts: "...)
body = strconv.AppendInt(body, int64(len(parts)), 10)
body = append(body, '\n')
for i := range parts {
body = append(body, parts[i].Header.Name...)
if len(parts[i].Header.Filename) > 0 {
body = append(body, " -> "...)
body = append(body, parts[i].Header.Filename...)
}
body = append(body, '\n')
}
body = append(body, "bytes stored: "...)
body = strconv.AppendInt(body, s.uploads.n, 10)
body = append(body, '\n')
exch.Respond(httphi.StatusOK, "text/plain", body)
}
// echo hands back the request line and the header block as the parser stored
// it, which is what tells a header fuzzer what its input turned into.
func (sv *Server) echo(exch *httphi.Exchange) {
s := sv.acquireScratch()
defer sv.releaseScratch(s)
body := append(s.out[:0], exch.RequestMethodRaw()...)
body = append(body, ' ')
body = append(body, exch.RequestTarget()...)
body = append(body, '\n')
if value := exch.RequestHeader("X-Fuzz"); value != nil {
body = append(body, "x-fuzz: "...)
body = append(body, value...)
body = append(body, '\n')
}
body = append(body, "-- parsed header --\n"...)
body = exch.RequestHeaderV1Raw().AppendHeaders(body)
exch.Respond(httphi.StatusOK, "text/plain", body)
}
// countingSink discards a multipart part and counts what it discarded, standing
// in for the file a real upload would write.
type countingSink struct{ n int64 }
func (c *countingSink) Write(b []byte) (int, error) { c.n += int64(len(b)); return len(b), nil }
func (c *countingSink) Close() error { return nil }
// atoiBounded parses a decimal number and clamps it to max, reporting false for
// anything that is not one. It works off the bytes rather than converting to a
// string, which would allocate on a path every request takes.
func atoiBounded(b []byte, max int) (int, bool) {
const maxDigits = 9 // Bounded so the accumulator below cannot overflow.
if len(b) == 0 || len(b) > maxDigits {
return 0, false
}
n := 0
for _, c := range b {
if c < '0' || c > '9' {
return 0, false
}
n = n*10 + int(c-'0')
}
return min(n, max), true
}
@@ -18,11 +18,12 @@ import (
"runtime"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/soypat/lneto"
"github.com/soypat/lneto/ethernet"
"github.com/soypat/lneto/http/httpraw"
"github.com/soypat/lneto/http/httphi"
"github.com/soypat/lneto/internal"
"github.com/soypat/lneto/internal/ltesto"
"github.com/soypat/lneto/internet/pcap"
@@ -34,6 +35,23 @@ import (
//go:embed index.html
var indexhtml string
// Router memory. The router allocates all of it on Configure and never again,
// so these are the whole cost of serving HTTP over the stack.
const (
// A browser sends around 700 bytes of header on a landing page request.
requestHeaderBuffer = 1024
// Response headers reuse whatever the request left unused on top of this,
// and the status line does not count towards it.
responseHeaderBuffer = 256
numHeaderFields = 16
// One exchange is allocated per worker, and a worker holds its exchange for
// the whole request, so this is what bounds requests served at once.
numWorkers = 2
// requestTimeout drops a peer that opens a connection and then stalls,
// rather than letting it hold one of the workers.
requestTimeout = 10 * time.Second
)
var softRand = time.Now().Unix()
func main() {
@@ -225,6 +243,28 @@ func run() (err error) {
svPort := uint16(flagPort)
fmt.Printf("Listening on %s:%d\n", ipv4.AppendFormatAddr(nil, stack.Addr4()), svPort)
// Routes are registered before Configure: the router reads the mux to size
// the exchanges it allocates, and refuses a mux with nothing registered.
server := httpServer{start: time.Now()}
// "{$}" matches the empty path and nothing else, so anything unregistered
// gets a 404 rather than the index page.
server.handle("GET /{$}", server.index)
server.handle("GET /stats", server.stats)
var router httphi.Router
err = router.Configure(httphi.RouterConfig{
FixedNumGoroutines: numWorkers,
RequestHeaderBufferSize: requestHeaderBuffer,
ResponseHeaderMinBufferSize: responseHeaderBuffer,
RequestNumHeaderKVCap: numHeaderFields,
Mux: &server.mux,
Logger: slog.Default(),
})
if err != nil {
return fmt.Errorf("configuring HTTP router: %w", err)
}
defer router.Shutdown()
// Serve connections in a loop.
for {
var conn tcp.Conn
@@ -254,65 +294,56 @@ func run() (err error) {
continue
}
fmt.Println("connection established from", net.IP(conn.RemoteAddr()).String())
go func() {
err = handleConnection(&conn)
if err != nil {
fmt.Println("handle error:", err)
}
}()
// The connection owns the idle policy: a peer that stalls fails its read
// instead of holding a worker. conn is declared inside the loop, so the
// worker keeps serving this one while the next iteration listens anew.
conn.SetDeadline(time.Now().Add(requestTimeout))
err = router.Handle(&conn)
if err != nil {
// Every worker is busy. Dropping is the backpressure that keeps the
// stack's memory bounded, see numWorkers.
slog.Warn("dropped connection", slog.String("err", err.Error()))
conn.Abort()
}
}
}
func handleConnection(conn *tcp.Conn) error {
conn.SetDeadline(time.Now().Add(10 * time.Second))
// httpServer holds what the handlers answer with. Routes are registered on its
// mux before [httphi.Router.Configure] runs, which reads the mux to size the
// path values every exchange must hold.
type httpServer struct {
mux httphi.MuxSlice
served atomic.Uint64
start time.Time
}
// Read HTTP request.
var hdr httpraw.Header
var needMore bool = true
for needMore {
_, err := hdr.ReadFromLimited(conn, 1024)
if err != nil {
return fmt.Errorf("reading request: %w", err)
}
const asResponse = false
needMore, err = hdr.TryParse(asResponse)
if err != nil && !needMore {
return fmt.Errorf("parsing request: %w", err)
}
}
// handle registers handler and wraps it in the logging and counting every
// request goes through, i.e: the "< GET /" line this example has always printed.
func (sv *httpServer) handle(pattern string, handler httphi.HandlerFunc) {
sv.mux.Handle(pattern, func(exch *httphi.Exchange) {
sv.served.Add(1)
fmt.Printf("< %s %s\n", exch.RequestMethodRaw(), exch.RequestTarget())
handler(exch)
})
}
method := string(hdr.Method())
uri := string(hdr.RequestTarget())
fmt.Printf("< %s %s\n", method, uri)
// index serves the embedded page. The body goes straight to the connection, so
// only its header ever sits in the exchange's buffer and the page's size does
// not enter into how the router is configured.
func (sv *httpServer) index(exch *httphi.Exchange) {
exch.RespondString(httphi.StatusOK, "text/html", indexhtml)
}
// Build response body.
// Build HTTP response.
var resp httpraw.Header
resp.SetProtocol("HTTP/1.1")
resp.SetStatus("200", "OK")
resp.Set("Content-Type", "text/html")
resp.Set("Content-Length", strconv.Itoa(len(indexhtml)))
resp.Set("Connection", "close")
response, err := resp.AppendResponse(nil)
if err != nil {
return fmt.Errorf("building response: %w", err)
}
response = append(response, indexhtml...)
// Send response.
_, err = conn.Write(response)
if err != nil {
return fmt.Errorf("writing response: %w", err)
}
err = conn.Flush()
if err != nil {
return fmt.Errorf("flushing response: %w", err)
}
fmt.Printf("> %d bytes sent\n", len(response))
conn.Close()
return nil
// stats reports what the stack has served, which is the quickest way to tell a
// working link from a page that came out of a browser cache.
func (sv *httpServer) stats(exch *httphi.Exchange) {
var buf [128]byte
body := append(buf[:0], "requests served: "...)
body = strconv.AppendUint(body, sv.served.Load(), 10)
body = append(body, "\nuptime: "...)
body = append(body, prettyDuration(time.Since(sv.start))...)
body = append(body, '\n')
exch.Respond(httphi.StatusOK, "text/plain", body)
}
func clear(buf []byte) {
@@ -41,8 +41,8 @@ func run() error {
defer ln.Close()
print("listening on http://localhost:", listenPort, "\n")
var mux httphi.MuxSlice
mux.Handle("GET /", homepage)
var server Server
server.Handle("GET /", server.homepage)
var router httphi.Router
err = router.Configure(httphi.RouterConfig{
@@ -50,8 +50,7 @@ func run() error {
RequestHeaderBufferSize: bufferSizes,
RequestNumHeaderKVCap: numHeaderFields,
ResponseHeaderMinBufferSize: bufferSizes,
MaxAwaitingConns: 256,
Mux: &mux,
Mux: &server.mux,
Logger: slog.Default(),
})
if err != nil {
@@ -89,18 +88,28 @@ const (
maxPage = len(htmlHead) + 20 + len(htmlTail)
)
// visits counts served requests. Handlers run on the router's goroutines, so
// every visitor gets their own number.
var visits atomic.Uint64
type Server struct {
// visits counts served requests.
Visits atomic.Uint64
mux httphi.MuxSlice
}
func homepage(exch *httphi.Exchange) {
func (sv *Server) Handle(pattern string, handler httphi.HandlerFunc) {
// Middleware for all incoming requests declared here.
sv.mux.Handle(pattern, func(exch *httphi.Exchange) {
println(exch.RequestMethod().String(), exch.MuxPattern())
handler(exch)
})
}
func (sv *Server) homepage(exch *httphi.Exchange) {
var page [maxPage]byte
n := copy(page[:], htmlHead)
n += len(strconv.AppendUint(page[n:n], visits.Add(1), 10))
n += len(strconv.AppendUint(page[n:n], sv.Visits.Add(1), 10))
n += copy(page[n:], htmlTail)
exch.StageHeader("Content-Type", "text/html")
exch.StageHeaderInt("Content-Length", int64(n), 10)
exch.WriteHeader(int(httphi.StatusOK))
exch.WriteBody(page[:n])
exch.Respond(200, "text/html", page[:n])
}
func (sv *Server) form(exch *httphi.Exchange) {
}
@@ -25,7 +25,7 @@ func run() error {
flag.IntVar(&port, "lport", 13337, "Local port over which to hit server")
flag.Parse()
// Prepare GET request.
var hdr httpraw.Header
var hdr httpraw.HeaderV1
hdr.SetMethod("GET")
hdr.SetRequestTarget("/")
hdr.SetProtocol("HTTP/1.1")
+1 -1
View File
@@ -168,7 +168,7 @@ func (d *dhcpInterceptor) buildDHCPResponse(buf []byte) (int, error) {
// DHCP responses must be broadcast since the client doesn't have
// an IP configured yet and the stack would drop unicast packets.
*efrm.DestinationHardwareAddr() = ethernet.BroadcastAddr()
*ifrm.DestinationAddr() = [4]byte{255, 255, 255, 255}
*ifrm.DestinationAddr() = ipv4.BroadcastAddr()
ifrm.SetTotalLength(totalIPLen)
ufrm.SetLength(udpLen)
// Source and destination IPs already set by dhcpv4.Server.Encapsulate.
+1 -1
View File
@@ -305,7 +305,7 @@ func run() (err error) {
})
timeHTTPCreate := timer("create HTTP GET request")
var hdr httpraw.Header
var hdr httpraw.HeaderV1
hdr.SetMethod("GET")
hdr.SetRequestTarget("/")
hdr.SetProtocol("HTTP/1.1")