Tcp buffer bug fix (#79)

* start working on tracking down tcp buffer bug

* mtu refactor

* modularize test

* more precise testing

* tests fail, but is it the failure we are looking for?

* fix typo in espradio link (#76)

* implement a new backoff abstraction (#75)

* rewrite backoff api

* rewrite tcp.Conn.Write

* keep fixing small things

* much better Conn.Read implementation

* fix critical overflow bug in internal.ConnRWBackoff

---------

Co-authored-by: Joel Wetzell <jwetzell@yahoo.com>

backoff.go

@@ -0,0 +1,36 @@
+package lneto
+
+import (
+	"runtime"
+	"time"
+)
+
+// Flag return values for a BackoffStrategy.
+const (
+	BackoffFlagGosched = time.Duration(-1)
+	BackoffFlagNop     = time.Duration(-2)
+)
+
+// BackoffStrategy is the abstraction of a backoff strategy for retrying an operation.
+// It returns the amount of time to sleep for or a flag value:
+//   - Returns [BackoffFlagNop]: Signal no yielding function should be called.
+//     Useful for when the BackoffStrategy implements its own yield.
+//   - Returns [BackoffFlagGosched]: Signal [runtime.Gosched] should be called.
+//
+// consecutiveBackoffs starts at 1 and increments by 1 every time the operation is retried.
+// See internal/backoff.go for a implementation example.
+type BackoffStrategy func(consecutiveBackoffs int) (sleepOrFlag time.Duration)
+
+// Do applies the backoff strategy by calling backoff(consecutiveBackoffs)
+// and then the corresponding yield function for the returned value. See [BackoffStrategy].
+func (backoff BackoffStrategy) Do(consecutiveBackoffs int) {
+	sleep := backoff(consecutiveBackoffs)
+	switch sleep {
+	case BackoffFlagNop:
+		// No yield. Yield implemented by backoff.
+	case BackoffFlagGosched:
+		runtime.Gosched()
+	default:
+		time.Sleep(sleep)
+	}
+}

definitions.go

@@ -1,7 +1,5 @@
 package lneto
 
-import "time"
-
 // StackNode is an abstraction of a packet exchanging protocol controller. This is the building block for all protocols,
 // from Ethernet to IP to TCP, practically any protocol can be expressed as a StackNode and function completely.
 // Today protocols represented by StackNode also include NTP, DNS, DHCP, ARP, ICMP, UDP, mDNS.
@@ -39,12 +37,6 @@ type StackNode interface {
 	// SetFlagPending(flagPending func(numPendingEncapsulations int))
 }
 
-// BackoffStrategy is the abstraction of a backoff strategy for retrying an operation.
-// It returns the amount of time to sleep for. If returned value is 0 then runtime.Gosched is called.
-// A negative value results in no action taken.
-// consecutiveBackoffs starts at 1 and increments by 1 every time the operation is retried.
-type BackoffStrategy func(consecutiveBackoffs int) (sleep time.Duration)
-
 // IPProto represents the IP protocol number.
 type IPProto uint8
 

ethernet/definitions.go

@@ -5,13 +5,24 @@ import (
 )
 
 const (
+	// MaxMTU defines the maximum payload of a standard ethernet frame. Does NOT include ethernet header, FCS and VLAN tag.
+	// Ethernet frames can be larger but this is out of the 802.3 standard and going into jumbo frame territory.
+	MaxMTU = 1500
+	// MaxFrameLength (1522) defines the maximum length of a standard ethernet frame including headers, FCS and VLAN if present.
+	MaxFrameLength = MaxMTU + MaxOverheadSize
 	// MaxOverheadSize is the maximum overhead a packet can incur
 	// from transmitting an ethernet frame. Includes:
 	//  - 14 bytes of Ethernet header containing MAC addresses and ethernet type, always present
-	//  - 4 bytes of VLAN tag, if present.
+	//  - 4 bytes of VLAN tag, if present. See 802.1Q.
 	//  - 4 bytes of the 32 bit trailing CRC, if required by PHY.
-	MaxOverheadSize  = 14 + 4 + 4
+	MaxOverheadSize  = sizeHeaderNoVLAN + fcsOverhead + vlanOverhead
+	vlanOverhead     = 4
+	fcsOverhead      = 4
 	sizeHeaderNoVLAN = 14
+	// MinimumFrameLength as defined by IEEE 802.3 standards. Includes ethernet header.
+	MinimumFrameLength = 64
+	// Does not include VLAN/FCS for more conservative minimum MTU.
+	MinimumMTU = MinimumFrameLength - sizeHeaderNoVLAN
 )
 
 // AppendAddr appends the text representation of the hardware address to the destination buffer.

internal/backoff.go

@@ -1,17 +1,21 @@
 package internal
 
-import "time"
+import (
+	"time"
+)
 
 // ConnRWBackoff implements exponential backoff suitable for TCP connection
 // read/write polling. It starts at 1us and caps at 5ms, doubling on each consecutive backoff.
 func ConnRWBackoff(consecutiveBackoffs int) {
 	const (
-		minWait = time.Microsecond >> 1
+		minWait        = uint32(time.Microsecond) >> 1
-		maxWait = 5 * time.Millisecond
+		maxWait        = 5 * uint32(time.Millisecond)
+		maxShift       = 23
+		_overflowCheck = minWait << maxShift
 	)
-	wait := minWait << consecutiveBackoffs
+	wait := minWait << min(consecutiveBackoffs, maxShift)
 	if wait > maxWait {
 		wait = maxWait
 	}
-	time.Sleep(wait)
+	time.Sleep(time.Duration(wait))
 }

internet/definitions.go

@@ -159,6 +159,8 @@ func (h *handlers) encapsulateNode(node *node, buf []byte, offsetIP, offsetThisF
 		err = nil  // CLOSE error handled gracefully by deleting node.
 		node = nil // Node is destroyed in tryHandleError and invalidated.
 	}
+	// TODO(soypat): We have fuzz tests in place, maybe we can start returning the error up the chain to catch invalid settings at application level so that users don't have to have logs in place to understand invalid config/buffer size.
+	// Encapsulate should only fail with error on programmer errors.
 	if n > 0 {
 		return n, err
 	} else if err != nil {

internet/pcap/capture_test.go

@@ -35,7 +35,7 @@ func makeHttpPayload(body string) ([]byte, error) {
 }
 
 func TestCap(t *testing.T) {
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const httpBody = "{200,ok}"
 	var buf [mtu]byte
 	var gen ltesto.PacketGen

internet/stack-ethernet.go

@@ -79,7 +79,7 @@ func (ls *StackEthernet) Reset6(mac, gateway [6]byte, mtu, maxNodes int) error {
 // It validates the configuration parameters and resets internal state.
 // The connection ID is incremented on each call to invalidate existing connections.
 func (ls *StackEthernet) Configure(cfg StackEthernetConfig) error {
-	if cfg.MTU > (math.MaxUint16-ethernet.MaxOverheadSize) || cfg.MTU < 256 {
+	if cfg.MTU > ethernet.MaxMTU || cfg.MTU < ethernet.MinimumMTU {
 		return lneto.ErrInvalidConfig
 	} else if cfg.MaxNodes <= 0 {
 		return lneto.ErrInvalidConfig

internet/stack-ip.go

@@ -149,7 +149,7 @@ func (sb *StackIP) Demux(carrierData []byte, offset int) error {
 
 func (sb *StackIP) Encapsulate(carrierData []byte, offsetToIP, offsetToFrame int) (int, error) {
 	frame := carrierData[offsetToFrame:]
-	if len(frame) < 256 {
+	if len(frame) < ipv4.MinimumMTU {
 		return 0, io.ErrShortBuffer
 	}
 	ifrm, _ := ipv4.NewFrame(frame)

ipv4/definitions.go

@@ -3,6 +3,8 @@ package ipv4
 import "strconv"
 
 const (
+	// RFC791 defines the minimum MTU for an IPv4 packet as 68, meaning a payload of 48 bytes when no IPv4 options included.
+	MinimumMTU = 68
 	sizeHeader = 20
 )
 

tcp/conn.go

@@ -2,11 +2,11 @@ package tcp
 
 import (
 	"errors"
+	"io"
 	"log/slog"
 	"net"
 	"net/netip"
 	"os"
-	"runtime"
 	"sync"
 	"time"
 
@@ -213,29 +213,34 @@ func (conn *Conn) Write(b []byte) (int, error) {
 	}
 	n := 0
 	backoffs := 0
-	for {
+	for len(b) > 0 {
 		if err := conn.checkPipe(connid, &conn.wdead); err != nil {
-			return 0, err
+			return n, err
 		}
 		conn.mu.Lock()
+		towrite := min(conn.h.FreeOutput(), len(b))
 		var ngot int
-		ngot, err = conn.h.Write(b)
+		if towrite > 0 {
-		conn.mu.Unlock()
+			ngot, err = conn.h.Write(b[:towrite])
-		n += ngot
+			conn.mu.Unlock()
-		b = b[ngot:]
+			if err != nil && err != internal.ErrRingBufferFull {
-		if (err != nil && err != internal.ErrRingBufferFull) || n == plen {
+				break
-			break
+			} else if ngot != towrite {
-		} else if ngot > 0 {
+				panic("unreachable")
+			}
+			n += ngot
+			b = b[ngot:]
 			backoffs = 0
-			runtime.Gosched() // Do a little yield since we won't have data for sure otherwise.
 		} else {
+			// No data can be written.
+			conn.mu.Unlock()
+			conn.trace("TCPConn.Write:insuf-buf", slog.Int("missing", plen-n), slog.Uint64("lport", uint64(lport)), slog.Uint64("rport", uint64(rport)))
+			if conn.deadlineExceeded(&conn.wdead) {
+				return n, errDeadlineExceeded
+			}
 			backoffs++
 			conn.backoff(backoffs)
 		}
-		conn.trace("TCPConn.Write:insuf-buf", slog.Int("missing", plen-n), slog.Uint64("lport", uint64(lport)), slog.Uint64("rport", uint64(rport)))
-		if conn.deadlineExceeded(&conn.wdead) {
-			return n, errDeadlineExceeded
-		}
 	}
 	return n, err
 }
@@ -260,38 +265,48 @@ func (conn *Conn) Flush() error {
 // Read will block until data is available or connection closes.
 // Returns io.EOF when the remote has closed the connection and all buffered data has been read.
 func (conn *Conn) Read(b []byte) (int, error) {
-	connid, err := conn.lockPipeConnID()
+	conn.mu.Lock()
-	if err != nil {
+	connID := conn.h.connid
-		if conn.BufferedInput() > 0 {
+	lport := conn.h.localPort
-			return conn.handlerRead(b) // Ensure remaining buffered data is read.
+	rport := conn.h.remotePort
-		}
+	conn.mu.Unlock()
-		return 0, err
-	}
-	lport := conn.LocalPort()
-	rport := conn.RemotePort()
 	conn.trace("TCPConn.Read:start", slog.Uint64("lport", uint64(lport)), slog.Uint64("rport", uint64(rport)))
 	backoffs := 0
-	for conn.BufferedInput() == 0 {
+	n := 0
-		state := conn.State()
+	for len(b) > 0 {
-		if !state.RxDataOpen() {
+		conn.mu.Lock()
-			// No use waiting for data, jump to read and return corresponding error from there.
+		if connID != conn.h.connid {
-			break
+			conn.mu.Unlock()
-		} else if err := conn.checkPipe(connid, &conn.rdead); err != nil {
+			return n, net.ErrClosed
-			if conn.BufferedInput() > 0 {
+		}
-				return conn.handlerRead(b) // Ensure remaining buffered data is read.
+		avail := conn.h.BufferedInput()
-			}
+		if avail > 0 {
-			return 0, err
+			// Read branch.
+			ngot, err := conn.h.Read(b)
+			conn.mu.Unlock()
+			n += ngot
+			if err != nil {
+				return n, err
+			}
+			b = b[ngot:]
+		} else if n > 0 {
+			conn.mu.Unlock()
+			break
+		} else {
+			state := conn.h.State()
+			conn.mu.Unlock()
+			if state.IsClosed() {
+				return n, net.ErrClosed
+			} else if !state.RxDataOpen() {
+				return n, io.EOF
+			} else if conn.deadlineExceeded(&conn.rdead) {
+				return n, errDeadlineExceeded
+			}
+			backoffs++
+			conn.backoff(backoffs)
 		}
-		backoffs++
-		conn.backoff(backoffs)
 	}
-	return conn.handlerRead(b)
+	return n, nil
-}
-
-func (conn *Conn) handlerRead(b []byte) (int, error) {
-	conn.mu.Lock()
-	defer conn.mu.Unlock()
-	return conn.h.Read(b)
 }
 
 func (conn *Conn) lockPipeConnID() (uint64, error) {
@@ -452,7 +467,7 @@ func (conn *Conn) ConnectionID() *uint64 {
 
 func (conn *Conn) backoff(consecutiveBackoffs int) {
 	if conn._backoff != nil {
-		conn._backoff(consecutiveBackoffs)
+		conn._backoff.Do(consecutiveBackoffs)
 	} else {
 		internal.ConnRWBackoff(consecutiveBackoffs)
 	}

tcp/handler_test.go

@@ -5,10 +5,12 @@ import (
 	"fmt"
 	"math/rand"
 	"testing"
+
+	"github.com/soypat/lneto/ethernet"
 )
 
 func TestHandler(t *testing.T) {
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const maxpackets = 3
 	rng := rand.New(rand.NewSource(0))
 	client, server := newHandler(t, mtu, maxpackets), newHandler(t, mtu, maxpackets)
@@ -147,7 +149,7 @@ func establish(t *testing.T, client, server *Handler, packetBuf []byte) {
 // buffer for its SYN (advertising MSS=100), and the server should not send
 // segments with more than 100 bytes of payload.
 func TestHandler_MSSHonored(t *testing.T) {
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	rng := rand.New(rand.NewSource(0))
 	client, server := newHandler(t, mtu, 3), newHandler(t, mtu, 3)
 	setupClientServer(t, rng, client, server)
@@ -356,7 +358,7 @@ func TestTxBufferFreedOnACK(t *testing.T) {
 // stuck at Window=0 indefinitely after the app frees buffer space via Read().
 func TestWindowUpdateAfterRead(t *testing.T) {
 	const rxBufSize = 256
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const maxpackets = 4
 	rng := rand.New(rand.NewSource(99))
 
@@ -459,7 +461,7 @@ func TestWindowUpdateAfterRead(t *testing.T) {
 // half the buffer do NOT trigger a window update (Silly Window Syndrome avoidance).
 func TestWindowUpdateSWSAvoidance(t *testing.T) {
 	const rxBufSize = 256
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const maxpackets = 4
 	rng := rand.New(rand.NewSource(77))
 
@@ -566,7 +568,7 @@ func TestWindowUpdateSWSAvoidance(t *testing.T) {
 //  6. Handler.Send() called again → same thing → AddPacket panics because
 //     off=0 but lastPkt.end=0 != bufsize
 func TestWriteAfterRemoteFIN(t *testing.T) {
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const maxpackets = 3
 	rng := rand.New(rand.NewSource(11))
 	client, server := newHandler(t, mtu, maxpackets), newHandler(t, mtu, maxpackets)
@@ -635,7 +637,7 @@ func TestWriteAfterRemoteFIN(t *testing.T) {
 // This is a regression test for a bug where RST segments in non-synchronized
 // states were blocked by errRequireSequential, causing connection pool leaks.
 func TestRSTinSynReceived(t *testing.T) {
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const maxpackets = 3
 	rng := rand.New(rand.NewSource(2))
 	client, server := newHandler(t, mtu, maxpackets), newHandler(t, mtu, maxpackets)
@@ -717,7 +719,7 @@ func TestRSTinSynReceived(t *testing.T) {
 //
 // The bug was that reset() cleared bufRx when state became CLOSED.
 func TestBufferNotClearedOnPassiveClose(t *testing.T) {
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const maxpackets = 3
 	rng := rand.New(rand.NewSource(1))
 	client, server := newHandler(t, mtu, maxpackets), newHandler(t, mtu, maxpackets)
@@ -881,7 +883,7 @@ func TestBufferNotClearedOnPassiveClose(t *testing.T) {
 //  5. Handler.Send() called again → AddPacket panics because off=0 but
 //     lastPkt.end=0 != bufsize
 func TestChallengeACKWithBufferedData(t *testing.T) {
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const maxpackets = 3
 	rng := rand.New(rand.NewSource(42))
 	client, server := newHandler(t, mtu, maxpackets), newHandler(t, mtu, maxpackets)

udp/conn.go

@@ -303,7 +303,7 @@ func (conn *Conn) FreeInput() int {
 
 func (conn *Conn) backoff(consecutiveBackoffs int) {
 	if conn._backoff != nil {
-		conn._backoff(consecutiveBackoffs)
+		conn._backoff.Do(consecutiveBackoffs)
 	} else {
 		internal.ConnRWBackoff(consecutiveBackoffs)
 	}

x/xnet/xnet_arp_test.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"net/netip"
 	"testing"
+
+	"github.com/soypat/lneto/ethernet"
 )
 
 func TestARPLocal(t *testing.T) {
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
 	const seed = 1
 	s1, s2, c1, c2 := newTCPStacks(t, seed, mtu)
 	routerHw := [6]byte{1, 2, 3, 4, 5, 6}

x/xnet/xnet_bench_test.go

@@ -9,8 +9,8 @@ import (
 )
 
 func BenchmarkARPExchange(b *testing.B) {
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
-	const frameSize = MTU + ethernet.MaxOverheadSize
+	const frameSize = ethernet.MaxFrameLength
 	c1, c2 := new(StackAsync), new(StackAsync)
 	queryAddr := netip.AddrFrom4([4]byte{192, 168, 1, 2})
 
@@ -76,8 +76,8 @@ func BenchmarkARPExchange(b *testing.B) {
 }
 
 func BenchmarkTCPHandshake(b *testing.B) {
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
-	const frameSize = MTU + ethernet.MaxOverheadSize
+	const frameSize = ethernet.MaxFrameLength
 	const svPort = 8080
 	client, sv := new(StackAsync), new(StackAsync)
 	clconn, svconn := new(tcp.Conn), new(tcp.Conn)

x/xnet/xnet_concurrent_test.go

@@ -4,14 +4,19 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"log/slog"
 	"math/rand"
 	"net/netip"
+	"os"
 	"runtime"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/soypat/lneto"
 	"github.com/soypat/lneto/ethernet"
+	"github.com/soypat/lneto/internal"
+	"github.com/soypat/lneto/ipv4"
 	"github.com/soypat/lneto/tcp"
 )
 
@@ -142,8 +147,8 @@ func TestTCPListener_ConcurrentEcho(t *testing.T) {
 }
 
 func kernelLoop(ctx context.Context, server *StackAsync, clients []StackAsync) {
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
-	const carrierDataSize = MTU + ethernet.MaxOverheadSize
+	const carrierDataSize = ethernet.MaxFrameLength
 	buf := make([]byte, carrierDataSize)
 	rng := rand.New(rand.NewSource(1)) // Seed 1 for deterministic but randomized order
 	order := make([]int, len(clients))
@@ -285,3 +290,148 @@ func runClient(t *testing.T, id int, stack *StackAsync, conn *tcp.Conn,
 	}
 	return true
 }
+
+func TestCloseTransmitsPending(t *testing.T) {
+	const mtu = ipv4.MinimumMTU
+	const tcpbufsize = mtu * 2
+	const tcpDataPerPkt = mtu - 14 - 20 - 20 // Ethernet=14, IPv4=20, TCP=20
+	const expectPkts = 2*tcpbufsize/tcpDataPerPkt + 1
+	const queueSize = 5
+	const port1, port2 = 10, 20
+	tst := testerFrom(t, mtu)
+	tst.buf = tst.buf[:mtu+14]
+	s1, s2, c1, c2 := newTCPStacks(t, 0x1337_c0de, mtu)
+	t.Run("sync", func(t *testing.T) {
+		// testCloseTransmitsPending(tst, s1, s2, c1, c2, queueSize, tcpbufsize, tcpbufsize, tcpbufsize)
+	})
+	t.Run("async", func(t *testing.T) {
+		testCloseTransmitsPending(tst, s1, s2, c1, c2, queueSize, tcpbufsize, tcpbufsize, 2*tcpbufsize)
+	})
+
+}
+
+func testCloseTransmitsPending(tst *tester, s1, s2 *StackAsync, c1, c2 *tcp.Conn, queueSize, tx1Buf, rx2Buf, datalen int) {
+	t := tst.t
+	buf := tst.buf
+	defer func() {
+		c1.Abort()
+		c2.Abort()
+		// Ensure they are unregistered.
+		s1.EgressIP(buf)
+		s2.EgressIP(buf)
+	}()
+	logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelDebug - 99,
+	}))
+	err := c1.Configure(tcp.ConnConfig{
+		RxBuf:             nil,
+		TxBuf:             make([]byte, tx1Buf),
+		TxPacketQueueSize: queueSize,
+		RWBackoff:         backoffGosched,
+		Logger:            logger,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = c2.InternalHandler().SetBuffers(nil, make([]byte, rx2Buf), queueSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	const (
+		port1, port2 = 10, 20
+	)
+	tst.TestTCPSetupAndEstablish(s1, s2, c1, c2, port1, port2)
+	if c1.FreeOutput() != tx1Buf {
+		t.Fatalf("want %d free bytes, got %d", tx1Buf, c1.FreeOutput())
+	}
+	data := make([]byte, datalen)
+	for i := range datalen {
+		data[i] = byte(i)
+	}
+	deadline := time.Now().Add(3600 * time.Second)
+	err = c1.SetDeadline(deadline)
+	err2 := c2.SetDeadline(deadline)
+	if err != nil || err2 != nil {
+		t.Fatal(err)
+	}
+
+	async := datalen > tx1Buf
+	if async {
+		// Since data does not fit in TCP Tx buffer the test must be run asynchronously.
+		c1.InternalHandler().SetLoggers(logger, logger)
+		// c1.InternalHandler().SetLoggers(nil, nil)
+		go func() {
+			n, err := c1.Write(data)
+			if err != nil {
+				t.Error("async write", err)
+			} else if n != len(data) {
+				t.Error("io.Writer faulty implementation")
+			}
+			err = c1.Close()
+			if err != nil {
+				t.Fatal("async close", err)
+			}
+		}()
+	} else {
+		n, err := c1.Write(data)
+		if err != nil || n != len(data) {
+			t.Fatal(err, n)
+		}
+		err = c1.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	exchanges := -1
+	exchanging := 1
+	tcpData := 0
+	totalRead := 0
+	for exchanging > 0 || c1.State().TxDataOpen() {
+		exchanges++
+		exchanging = exchangeEthernetOnce(t, s1, s2, buf)
+		frm, ok := getTCPFrame(buf[:exchanging])
+		if ok {
+			n := len(frm.Payload())
+			tcpData += n
+			if async && tcpData > 0 {
+				ngot, err := c2.Read(buf[:n])
+				if err != nil {
+					t.Error(err)
+				} else if ngot != n {
+					t.Errorf("want %d data read c1->c1, got %d", n, ngot)
+				} else if !internal.BytesEqual(buf[:n], data[totalRead:totalRead+n]) {
+					t.Errorf("exch%d data rx mismatch, want:\n%q\ngot:\n%q\n", exchanges, data[totalRead:totalRead+n], buf[:n])
+				}
+				totalRead += ngot
+				runtime.Gosched()                            // Yield to let c1 write via goroutine.
+				acks := exchangeEthernetOnce(t, s2, s1, buf) // Send ACK s1's way.
+				if acks == 0 {
+					t.Error("no data sent back to s1")
+				}
+			}
+		}
+	}
+	if c1.BufferedUnsent() != 0 {
+		t.Errorf("done %s: want no data left unsent got %d/%d", c1.State(), c1.BufferedUnsent(), len(data))
+	}
+	if tcpData != datalen {
+		t.Errorf("done %s: want %d bytes sent, got %d", c1.State(), len(data), tcpData)
+	}
+	if t.Failed() {
+		t.Logf("test params: txsz1=%d rxsz2=%d queuesize=%d data(sent/had)=%d/%d", tx1Buf, rx2Buf, queueSize, tcpData, datalen)
+	}
+	if totalRead < datalen {
+		n, err := c2.Read(buf)
+		if err != nil {
+			t.Error(err)
+		} else if !internal.BytesEqual(buf[:n], data[totalRead:]) {
+			t.Errorf("expected last bytes equal: want:\n%q\ngot:\n%q\n", data[totalRead:], buf[:n])
+		}
+	}
+
+}
+
+func backoffGosched(consecutiveBackoffs int) (sleep time.Duration) {
+	return lneto.BackoffFlagGosched
+}

x/xnet/xnet_deadline_test.go

@@ -5,12 +5,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/soypat/lneto/ethernet"
 	"github.com/soypat/lneto/tcp"
 )
 
 func TestTCPConn_SetDeadline_Established(t *testing.T) {
 	const seed = 9999
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const svPort = 8080
 	client, sv, clconn, svconn := newTCPStacks(t, seed, MTU)
 	tst := testerFrom(t, MTU)
@@ -37,7 +38,7 @@ func TestTCPConn_SetDeadline_Established(t *testing.T) {
 
 func TestTCPConn_ReadDeadlineExceeded(t *testing.T) {
 	const seed = 10001
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const svPort = 8080
 	client, sv, clconn, svconn := newTCPStacks(t, seed, MTU)
 	tst := testerFrom(t, MTU)
@@ -63,7 +64,7 @@ func TestTCPConn_ReadDeadlineExceeded(t *testing.T) {
 
 func TestTCPConn_WriteDeadlineExceeded(t *testing.T) {
 	const seed = 10002
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const svPort = 8080
 	client, sv, clconn, svconn := newTCPStacks(t, seed, MTU)
 	tst := testerFrom(t, MTU)
@@ -88,7 +89,7 @@ func TestTCPConn_WriteDeadlineExceeded(t *testing.T) {
 
 func TestTCPConn_FlushEmptyNoop(t *testing.T) {
 	const seed = 10003
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const svPort = 8080
 	client, sv, clconn, svconn := newTCPStacks(t, seed, MTU)
 	tst := testerFrom(t, MTU)

x/xnet/xnet_dns_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestDNS_QueryReceivesAnswer(t *testing.T) {
 	const seed = 9876
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 
 	// Create client stack with DNS server configured.
 	client := new(StackAsync)
@@ -47,7 +47,7 @@ func TestDNS_QueryReceivesAnswer(t *testing.T) {
 	}
 
 	// Client sends DNS query.
-	const carrierDataSize = MTU + ethernet.MaxOverheadSize
+	const carrierDataSize = ethernet.MaxFrameLength
 	var buf [carrierDataSize]byte
 	n, err := client.EgressEthernet(buf[:])
 	if err != nil {

x/xnet/xnet_fuzz_test.go

@@ -17,9 +17,9 @@ import (
 )
 
 func FuzzStackPacketHTTP(f *testing.F) {
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const seed = 1
-	var buf [MTU + ethernet.MaxOverheadSize]byte
+	var buf [ethernet.MaxFrameLength]byte
 	s1, s2, c1, c2 := newTCPStacks(f, seed, MTU)
 	var hdr httpraw.Header
 	err := s1.ListenTCP(c1, 80)
@@ -78,7 +78,7 @@ func FuzzStackPacketHTTP(f *testing.F) {
 	}
 
 	f.Fuzz(func(t *testing.T, pktnum int, a []byte) {
-		var buf [MTU + ethernet.MaxOverheadSize]byte
+		var buf [ethernet.MaxFrameLength]byte
 		s1, s2, c1, c2 := newTCPStacks(t, seed, MTU)
 		err = s1.EnableICMP(true)
 		if err != nil {
@@ -268,8 +268,8 @@ func testStackSeeded(t *testing.T, seed1, seed2 int64) {
 		}
 	}
 
-	const mtu = 1500
+	const mtu = ethernet.MaxMTU
-	const mfl = mtu + 14 // frame length includes ethernet header
+	const mfl = mtu + ethernet.MaxOverheadSize // frame length includes ethernet header
 	var buf [mfl]byte
 	var s1, s2 StackAsync
 	v1, v2 := byte(seed1), byte(seed2)

x/xnet/xnet_icmp_test.go

@@ -42,11 +42,11 @@ func TestStackAsync_ICMPEcho(t *testing.T) {
 				t.Fatal(err)
 			}
 			echoSent := exchangeEthernetOnce(t, sender, receiver, buf)
-			if !echoSent {
+			if echoSent == 0 {
 				t.Error("ECHO not sent")
 			}
 			echoReplySent := exchangeEthernetOnce(t, receiver, sender, buf)
-			if !echoReplySent {
+			if echoReplySent == 0 {
 				t.Error("ECHOREPLY not sent")
 			}
 			n, err = sender.EgressEthernet(buf)
@@ -70,19 +70,19 @@ func TestStackAsync_ICMPEcho(t *testing.T) {
 }
 
 // exchangeEthernetOnce sends one Ethernet frame from src to dst if available.
-func exchangeEthernetOnce(t *testing.T, src, dst *StackAsync, buf []byte) bool {
+func exchangeEthernetOnce(t *testing.T, src, dst *StackAsync, buf []byte) int {
 	t.Helper()
 	n, err := src.EgressEthernet(buf)
 	if err != nil {
-		t.Fatal(err)
+		t.Error(err)
 	}
 	if n == 0 {
-		return false
+		return 0
 	}
 	if err := dst.IngressEthernet(buf[:n]); err != nil {
-		t.Fatal(err)
+		t.Error(err)
 	}
-	return true
+	return n
 }
 
 // newICMPStacks creates two test stacks with distinct static addresses and hardware addresses.

x/xnet/xnet_listener_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestStackAsyncListener_SingleConnection(t *testing.T) {
 	const seed int64 = 1234
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const carrierSize = MTU + ethernet.MaxOverheadSize
 	const svPort = 80
 	const clPort = 1337
@@ -122,7 +122,7 @@ func TestStackAsyncListener_SingleConnection(t *testing.T) {
 
 func TestStackAsyncListener_MultiSequentialConn(t *testing.T) {
 	const seed int64 = 1234
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const carrierSize = MTU + ethernet.MaxOverheadSize
 	const svPort = 80
 	const clPort = 1337

x/xnet/xnet_mdns_test.go

@@ -14,7 +14,7 @@ import (
 )
 
 func TestMDNS_QueryResponse(t *testing.T) {
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	svcName, err := dns.NewName("My Web._http._tcp.local")
 	if err != nil {
 		t.Fatal(err)
@@ -196,7 +196,7 @@ func TestMDNS_QueryResponse(t *testing.T) {
 }
 
 func TestMDNS_SRVThroughStack(t *testing.T) {
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	svcName, err := dns.NewName("My Web._http._tcp.local")
 	if err != nil {
 		t.Fatal(err)
@@ -275,7 +275,7 @@ func newMDNSStack(t *testing.T, hostname string, seed int64,
 	mdnsCfg mdns.ClientConfig,
 ) (*StackAsync, *mdns.Client) {
 	t.Helper()
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	stack := new(StackAsync)
 	err := stack.Reset(StackConfig{
 		Hostname:          hostname,
@@ -335,7 +335,7 @@ func mdnsQueryRespond(t *testing.T, querier, responder *StackAsync, buf []byte)
 }
 
 func TestMDNS_RealWorldQueries(t *testing.T) {
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 
 	responderMAC := [6]byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x01}
 	querierMAC := [6]byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x02}

x/xnet/xnet_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/soypat/lneto"
 	"github.com/soypat/lneto/arp"
 	"github.com/soypat/lneto/ethernet"
 	"github.com/soypat/lneto/internal"
@@ -29,7 +30,7 @@ const (
 
 func TestTCPConn_ReadBlocksUntilDataAvailable(t *testing.T) {
 	const seed = 5678
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const svPort = 8080
 	client, sv, clconn, svconn := newTCPStacks(t, seed, MTU)
 	tst := testerFrom(t, MTU)
@@ -151,7 +152,7 @@ func TestStackAsyncTCP_multipacket(t *testing.T) {
 
 func TestStackAsyncTCP_singlepacket(t *testing.T) {
 	const seed = 1234
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const svPort = 80
 	client, sv, clconn, svconn := newTCPStacks(t, seed, MTU)
 	tst := testerFrom(t, MTU)
@@ -283,8 +284,19 @@ func (tst *tester) TestTCPHandshake(stack1, stack2 *StackAsync) {
 		noExchange(0),
 		noExchange(1),
 	}
-	for _, wants := range exch {
+	var got [len(exch)]struct {
-		tst.TCPExchange(wants, stack1, stack2)
+		seg tcp.Segment
+	}
+	for i, wants := range exch {
+		haveFailed := tst.t.Failed()
+		got[i].seg = tst.TCPExchange(wants, stack1, stack2)
+		if haveFailed != tst.t.Failed() {
+			tst.t.Logf("print out sent segments (%d):\n", i+1)
+			for k := range i + 1 {
+				str := tcp.StringExchange(got[k].seg, 255, 255, exch[k].SourceIdx == 0) // states unknown.
+				tst.t.Log(str)
+			}
+		}
 	}
 }
 
@@ -734,7 +746,7 @@ func (tst *tester) getARPOperation() arp.Operation {
 // The bug was that reset() cleared bufRx when state became CLOSED.
 func TestTCPConn_BufferNotClearedOnPassiveClose(t *testing.T) {
 	const seed = 9999
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
 	const svPort = 8080
 	client, sv, clconn, svconn := newTCPStacks(t, seed, MTU)
 	tst := testerFrom(t, MTU)
@@ -915,8 +927,8 @@ func TestTCPConn_BufferNotClearedOnPassiveClose(t *testing.T) {
 }
 
 func TestStackAsync_ICMPEchoChecksum(t *testing.T) {
-	const MTU = 1500
+	const MTU = ethernet.MaxMTU
-	const MaxFrameLength = MTU + 14 + 4 // Ethernet header+FCS.
+	const MaxFrameLength = MTU + ethernet.MaxOverheadSize // Ethernet header+FCS+VLAN.
 	stackAddr := netip.AddrFrom4([4]byte{192, 168, 1, 99})
 	stackMAC := [6]byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}
 	routerAddr := [4]byte{192, 168, 1, 1}
@@ -1017,3 +1029,19 @@ const (
 	protoIPv4     = "IPv4"
 	protoTCP      = "TCP"
 )
+
+func getTCPFrame(etherFrame []byte) (tcp.Frame, bool) {
+	efrm, err := ethernet.NewFrame(etherFrame)
+	if err != nil || efrm.EtherTypeOrSize() != ethernet.TypeIPv4 {
+		return tcp.Frame{}, false
+	}
+	ifrm, err := ipv4.NewFrame(efrm.Payload())
+	if err != nil || ifrm.Protocol() != lneto.IPProtoTCP {
+		return tcp.Frame{}, false
+	}
+	tfrm, err := tcp.NewFrame(ifrm.Payload())
+	if err != nil {
+		return tcp.Frame{}, false
+	}
+	return tfrm, true
+}