tcp: listener produces RST response when conns exhausted

tcp/listener.go

@@ -28,6 +28,17 @@ type Listener struct {
 	poolGet    func() (*Conn, any, Value)
 	poolReturn func(*Conn)
 	logger
+	// rstQueue stores pending RST responses for SYNs rejected due to pool exhaustion.
+	// Per RFC 9293 §3.5.3: RST.SEQ=0, RST.ACK=SEG.SEQ+1, flags=RST|ACK.
+	rstQueue    [4]rstEntry
+	rstQueueLen uint8
+}
+
+// rstEntry holds the minimum state needed to construct a stateless RST response.
+type rstEntry struct {
+	remoteAddr [4]byte // IPv4 remote address.
+	remotePort uint16
+	ackNum     Value // SEG.SEQ + 1.
 }
 
 type handler struct {
@@ -170,6 +181,26 @@ func (listener *Listener) Encapsulate(carrierData []byte, offsetToIP, offsetToFr
 			err = nil
 		}
 	}
+	// Drain one RST entry if no connection data was sent. Lower priority than connection traffic.
+	if n == 0 && listener.rstQueueLen > 0 && offsetToIP >= 0 {
+		listener.rstQueueLen--
+		entry := &listener.rstQueue[listener.rstQueueLen]
+		tfrm, err := NewFrame(carrierData[offsetToFrame:])
+		if err == nil {
+			tfrm.SetSourcePort(listener.port)
+			tfrm.SetDestinationPort(entry.remotePort)
+			tfrm.SetSegment(Segment{
+				SEQ:   0,
+				ACK:   entry.ackNum,
+				Flags: FlagRST | FlagACK,
+			}, 5)
+			tfrm.SetUrgentPtr(0)
+			err = internal.SetIPAddrs(carrierData[offsetToIP:offsetToFrame], 0, nil, entry.remoteAddr[:])
+			if err == nil {
+				return sizeHeaderTCP, nil
+			}
+		}
+	}
 	if n == 0 {
 		listener.maintainConns()
 	}
@@ -217,6 +248,13 @@ func (listener *Listener) Demux(carrierData []byte, tcpFrameOffset int) error {
 	conn, userData, iss := listener.poolGet()
 	if conn == nil {
 		slog.Error("tcpListener:no-free-conn")
+		if len(srcaddr) == 4 && listener.rstQueueLen < uint8(len(listener.rstQueue)) {
+			entry := &listener.rstQueue[listener.rstQueueLen]
+			entry.remotePort = src
+			entry.ackNum = tfrm.Seq() + 1
+			copy(entry.remoteAddr[:], srcaddr)
+			listener.rstQueueLen++
+		}
 		return lneto.ErrPacketDrop
 	}
 	err = conn.OpenListen(dst, iss)