soypat/lneto
1
0
Fork 0
mirror of https://github.com/soypat/lneto.git synced 2026-08-21 06:59:04 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Reduce heap allocs 2 (#46)

Browse Source 
* reduce heap allocations in tcp logging; omit use of AppendFloat which allocates a metric sh*tton

* debugheaplog: better heap statistic logging

* heap: use string for pcap.Frame.Protocol

* add potential to eliminate Flags.String heap alloc, remove incorrect HEAP comments

* add StackAsync.DebugErr and httpraw.SetBytes

* many heap alloc reductions and replacement of bytes.Equal with internal.BytesEqual
This commit is contained in:
Pat Whittingslow
2026-02-28 19:20:46 +01:00
committed by GitHub
parent eab43c4653
commit 989bb6a0b9
22 changed files with 511 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
http/httpraw/cookie.go
+1 -2
View File
@@ -2,7 +2,6 @@ package httpraw
import (
"bytes"
"errors"
)
// Cookie implements cookie key-value parsing. Methods function similarly to eponymous [Header] methods.
@@ -55,7 +54,7 @@ func (dst *Cookie) CopyFrom(c Cookie) {
// Parse parses the cookie's buffer in place.
func (c *Cookie) Parse() error {
if len(c.kvs) > 0 {
return errors.New("cookies already parsed, reset before parsing again")
return errCookiesParsed
}
off := 0
for {
http/httpraw/header.go
+20 -5
View File
@@ -2,9 +2,9 @@ package httpraw
import (
"bytes"
"errors"
"io"
"slices"
"unsafe"
)
const (
@@ -75,7 +75,9 @@ func (h *Header) ParseBytes(asResponse bool, b []byte) error {
// Parse parses accumulated data in-place with no copying. One can set HTTP header data buffer with [Header.Reset].
// It fails if HTTP data is incomplete.
func (h *Header) Parse(asResponse bool) error {
debuglog("http:parse:reset")
h.Reset(h.hbuf.buf)
debuglog("http:parse:start")
return h.parse(asResponse)
}
@@ -97,7 +99,7 @@ func (h *Header) Parse(asResponse bool) error {
// }
func (h *Header) TryParse(asResponse bool) (needMoreData bool, err error) {
if h.flags.hasAny(flagDoneParsingHeader) {
return false, errors.New("TryParse called after header parsed")
return false, errAlreadyParsed
} else if h.flags.hasAny(flagMangledBuffer) {
return false, errMangledBuffer
}
@@ -225,15 +227,18 @@ func (h *Header) Reset(buf []byte) {
panic("small buffer and flagNoBufferGrow set")
}
const persistentFlags = flagNoBufferGrow
debuglog("http:reset:hbuf")
h.hbuf.reset(buf)
*h = Header{
hbuf: h.hbuf,
flags: h.flags & persistentFlags,
}
debuglog("http:reset:done")
}
// Body returns the surplus data following headers. It is only valid as long as Parse* or Reset methods are not called.
func (h *Header) Body() ([]byte, error) {
debuglog("http:body")
if h.flags.hasAny(flagMangledBuffer) {
return nil, errMangledBuffer
} else if h.flags.hasAny(flagDoneParsingHeader) {
@@ -242,7 +247,14 @@ func (h *Header) Body() ([]byte, error) {
return nil, errUnparsed
}
// Set sets a key-value pair in the HTTP header. Calling Set mangles the buffer.
// SetBytes is equivalent to [Header.Set] but with a []byte value. Does not keep reference to value slice.
// Calling SetBytes Mangles the buffer.
func (h *Header) SetBytes(key string, value []byte) {
h.Set(key, unsafe.String(&value[0], len(value)))
}
// Set sets a key-value pair in the HTTP header.
// Calling Set mangles the buffer.
func (h *Header) Set(key, value string) {
hb := &h.hbuf
var useKv *argsKV
@@ -269,10 +281,13 @@ func (h *Header) Set(key, value string) {
// Get gets the first value of a key found in the headers. Use [Header.ForEach] to find multiple values corresponding to same key.
func (h *Header) Get(key string) []byte {
debuglog("http:get:start")
kv := h.peekHeader(key)
if kv.isValid() {
debuglog("http:get:found")
return h.hbuf.musttoken(kv.value)
}
debuglog("http:get:notfound")
return nil
}
@@ -338,7 +353,7 @@ func (h *Header) AppendRequest(dst []byte) ([]byte, error) {
if h.flags.hasAny(flagOOMReached) {
return dst, errOOM
} else if h.requestURI.len == 0 || h.method.len == 0 {
return dst, errors.New("need method/request URI to create request header")
return dst, errNeedMethodURI
} else if len(proto) == 0 {
return dst, errNoProto
}
@@ -368,7 +383,7 @@ func (h *Header) AppendResponse(dst []byte) ([]byte, error) {
if h.flags.hasAny(flagOOMReached) {
return dst, errOOM
} else if h.statusCode.len == 0 || h.statusText.len == 0 {
return dst, errors.New("invalid status code or text")
return dst, errBadStatusCodeTxt
} else if len(proto) == 0 {
return dst, errNoProto
}
http/httpraw/parse.go
+52 -21
View File
@@ -3,8 +3,11 @@ package httpraw
import (
"bytes"
"errors"
"log/slog"
"slices"
"unsafe"
"github.com/soypat/lneto/internal"
)
var (
@@ -16,8 +19,15 @@ var (
errOOM = errors.New("httpraw: buffer out of memory")
// Header.Set and Header.Add mangles the buffer.
// Call them after retrieving the Body. Do not call them before parsing the header (why would you even do that?).
errMangledBuffer = errors.New("httpraw: mangled buffer")
errNoCookies = errors.New("no cookie found")
errMangledBuffer = errors.New("httpraw: mangled buffer")
errNoCookies = errors.New("no cookie found")
errEmptyURI = errors.New("empty URI")
errLongStatusCode = errors.New("long status code")
errBadStatusCode = errors.New("invalid status code")
errAlreadyParsed = errors.New("TryParse called after header parsed")
errNeedMethodURI = errors.New("need method/request URI to create request header")
errBadStatusCodeTxt = errors.New("invalid status code or text")
errCookiesParsed = errors.New("cookies already parsed, reset before parsing again")
)
type headerBuf struct {
@@ -29,6 +39,20 @@ type headerBuf struct {
headers []argsKV
}
// reset sets the buffer data and discards all parsed data.
func (h *headerBuf) reset(buf []byte) {
if buf == nil {
buf = h.buf[:0] // Reuse buffer but discard raw data on nil input.
}
if cap(h.headers) == 0 {
h.headers = make([]argsKV, 16)
}
*h = headerBuf{
buf: buf,
headers: h.headers[:0],
}
}
type tokint = uint16
type headerSlice struct {
@@ -56,11 +80,16 @@ type scannerState struct {
}
func (h *Header) parse(asResponse bool) (err error) {
debuglog("http:firstline:start")
err = h.parseFirstLine(asResponse)
if err != nil {
debuglog("http:firstline:err")
return err
}
return h.parseNextHeaders()
debuglog("http:firstline:done")
err = h.parseNextHeaders()
debuglog("http:headers:done")
return err
}
func (h *Header) parseFirstLine(asResponse bool) (err error) {
@@ -90,9 +119,11 @@ func (hb *headerBuf) readFromBytes(b []byte) {
func (hb *headerBuf) free() int { return cap(hb.buf) - len(hb.buf) }
func (hb *headerBuf) parseNextHeaders(ss *scannerState) {
debuglog("http:nexthdr:loop")
for kv := hb.next(ss); kv.isValid(); kv = hb.next(ss) {
hb.headers = append(hb.headers, kv)
hb.headers = append(hb.headers, kv) // TODO(HEAP): inc=16B slice growth when capacity exceeded
}
debuglog("http:nexthdr:done")
}
func (hb *headerBuf) offBuf() []byte {
@@ -127,6 +158,7 @@ func (hb *headerBuf) scanUntilByte(c byte) []byte {
}
func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto headerSlice, flags flags, err error) {
debuglog("http:req:scan")
hb.off = 0 // Parsing first line resets offset.
var b []byte
hb.skipLeadingCRLF()
@@ -135,6 +167,7 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto
if len(b) < 5 {
return method, uri, proto, flags, errNeedMore
}
debuglog("http:req:parse")
methodEnd := max(0, bytes.IndexByte(b, ' '))
reqURIEnd := bytes.IndexByte(b[methodEnd+1:], ' ')
@@ -145,7 +178,7 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto
flags |= flagNoHTTP11
}
} else if reqURIEnd == 0 {
return method, uri, proto, flags, errors.New("empty URI")
return method, uri, proto, flags, errEmptyURI
} else {
// No version provided.
reqURIEnd = methodEnd + 1
@@ -158,6 +191,7 @@ func (hb *headerBuf) parseFirstLineRequest(initFlags flags) (method, uri, proto
}
func (hb *headerBuf) parseFirstLineResponse(initFlags flags) (statusCode, statusText headerSlice, flags flags, err error) {
debuglog("http:resp:scan")
hb.off = 0 // Parsing first line resets offset.
var b []byte
hb.skipLeadingCRLF()
@@ -166,23 +200,23 @@ func (hb *headerBuf) parseFirstLineResponse(initFlags flags) (statusCode, status
if len(b) < 5 {
return statusCode, statusText, flags, errNeedMore
}
debuglog("http:resp:parse")
statusCodeEnd := max(0, bytes.IndexByte(b, ' '))
if statusCodeEnd < 0 {
return statusCode, statusText, flags, errors.New("missing status code")
}
code := b[:statusCodeEnd]
text := b[statusCodeEnd:]
if len(code) > 3 {
return statusCode, statusText, flags, errors.New("long status code")
return statusCode, statusText, flags, errLongStatusCode
}
for i := range code {
if code[i] > '9' || code[i] < '0' {
return statusCode, statusText, flags, errors.New("invalid status code")
debuglog("http:resp:invalid-code")
return statusCode, statusText, flags, errBadStatusCode
}
}
statusCode = hb.slice(code)
statusText = hb.slice(text)
debuglog("http:resp:done")
return statusCode, statusText, flags, nil
}
@@ -360,17 +394,6 @@ func (hb *headerBuf) next(ss *scannerState) argsKV {
return resultKV
}
// reset sets the buffer data and discards all parsed data.
func (h *headerBuf) reset(buf []byte) {
if buf == nil {
buf = h.buf[:0] // Reuse buffer but discard raw data on nil input.
}
*h = headerBuf{
buf: buf,
headers: h.headers[:0],
}
}
// ConnectionClose returns true if 'Connection: close' header is set or if a invalid header was found.
func (h *Header) ConnectionClose() bool {
closed := h.flags.hasAny(flagConnClose) ||
@@ -402,3 +425,11 @@ func bytes2tok(buf, value []byte) headerSlice {
len: tokint(len(value)),
}
}
const enableDebug = false
func debuglog(msg string) {
if enableDebug {
internal.LogAttrs(nil, slog.LevelDebug, msg)
}
}