Add round-robin implementation (#66)

* add handler.encapsulateNode

* implement round robin handler approach

* simplify round robin implementation

* leave TODO

* internet.node touch up

* fix conflict resolve f-up

* replace certain ErrShortBuffer with ErrTruncatedFrame error

internet/definitions.go

@@ -11,24 +11,27 @@ import (
 
 // node is a concrete StackNode as stored in Stacks. Methods are devirtualized for performance benefits, especially on TinyGo.
 type node struct {
+	// currConnID stores the stack node *connID value on registration.
 	currConnID uint64
-	connID     *uint64
+	// connID is StackNode.ConnectionID() return value.
+	connID *uint64
 	// cbnode has different definitions in tinygo and normal Go compiled programs
 	// for performance and heap control reasons.
 	callbacks cbnode
-	// demux       func([]byte, int) error
-	// encapsulate func([]byte, int, int) (int, error)
-	proto uint16
-	port  uint16
 	// remoteAddr will be set on active(outbound) port connections
 	// that require an ARP to set the remoteAddr beforehand.
 	remoteAddr []byte
+	proto      uint16 // StackNode.Protocol()
+	lport      uint16 // StackNode.LocalPort()
 }
 
 type handlers struct {
+	nodes []node
+	// encapsIdx stores the index of next node to check for encapsulation.
+	encapsIdx int
+
 	context string
 	logger
-	nodes []node
 }
 
 func (h *handlers) reset(context string, maxNodes int) {
@@ -53,7 +56,7 @@ func (h *handlers) registerByPortProto(n node) error {
 	if err != nil {
 		return err
 	}
-	if h.nodeByPortProto(n.port, n.proto) != nil {
+	if h.nodeByPortProto(n.lport, n.proto) != nil {
 		return lneto.ErrAlreadyRegistered
 	}
 	h.nodes = append(h.nodes, n)
@@ -64,7 +67,7 @@ func (h *handlers) prepAdd() error {
 	if h.full() {
 		h.compact()
 		if h.full() {
-			return lneto.ErrBufferFull
+			return lneto.ErrExhausted
 		}
 	}
 	return nil
@@ -104,7 +107,7 @@ func (h *handlers) nodeByProto(proto uint16) *node {
 func (h *handlers) nodeByPort(port uint16) *node {
 	for i := range h.nodes {
 		node := &h.nodes[i]
-		if node.port == port && !node.IsInvalid() {
+		if node.lport == port && !node.IsInvalid() {
 			return node
 		}
 	}
@@ -114,7 +117,7 @@ func (h *handlers) nodeByPort(port uint16) *node {
 func (h *handlers) nodeByPortProto(port uint16, protocol uint16) *node {
 	for i := range h.nodes {
 		node := &h.nodes[i]
-		if node.port == port && node.proto == protocol && !node.IsInvalid() {
+		if node.lport == port && node.proto == protocol && !node.IsInvalid() {
 			return node
 		}
 	}
@@ -147,24 +150,37 @@ func (h *handlers) demuxByPort(buf []byte, offset int, port uint16) (*node, erro
 	return node, err
 }
 
+func (h *handlers) encapsulateNode(node *node, buf []byte, offsetIP, offsetThisFrame int) (n int, err error) {
+	if node.IsInvalid() {
+		return 0, nil
+	}
+	n, err = node.callbacks.Encapsulate(buf, offsetIP, offsetThisFrame)
+	if h.tryHandleError(node, err) {
+		err = nil  // CLOSE error handled gracefully by deleting node.
+		node = nil // Node is destroyed in tryHandleError and invalidated.
+	}
+	if n > 0 {
+		return n, err
+	} else if err != nil {
+		// Make sure not to hang on one handler that keeps returning an error.
+		h.error("handlers:encapsulate", slog.String("func", "encapsulateAny"), slog.String("ctx", h.context), slog.String("err", err.Error()))
+	}
+	return 0, nil
+}
+
 // encapsulateAny finds a node suitable to write and encapsulates the package.
 // If no data is sent it returns the last error encountered.
-func (h *handlers) encapsulateAny(buf []byte, offsetIP, offsetThisFrame int) (_ *node, n int, err error) {
-	for i := range h.nodes {
-		node := &h.nodes[i]
-		if node.IsInvalid() {
-			continue
-		}
-		n, err = node.callbacks.Encapsulate(buf, offsetIP, offsetThisFrame)
-		if h.tryHandleError(node, err) {
-			err = nil  // CLOSE error handled gracefully by deleting node.
-			node = nil // Node is destroyed in tryHandleError and invalidated.
-		}
-		if n > 0 {
-			return node, n, err
-		} else if err != nil {
-			// Make sure not to hang on one handler that keeps returning an error.
-			h.error("handlers:encapsulate", slog.String("func", "encapsulateAny"), slog.String("ctx", h.context), slog.String("err", err.Error()))
+func (h *handlers) encapsulateAny(buf []byte, offsetIP, offsetThisFrame int) (hn *node, n int, err error) {
+	// Round robin approach to encapsulation.
+	// TODO(soypat): benchmark impact of round robin. Consider removing fields from handlers to make it more lean and potentially get perf improvements that way.
+	i := h.encapsIdx
+	for range h.nodes {
+		hn := &h.nodes[i]
+		n, err = h.encapsulateNode(hn, buf, offsetIP, offsetThisFrame)
+		i = incLim(i, len(h.nodes))
+		if n > 0 || err != nil {
+			h.encapsIdx = i
+			return hn, n, err
 		}
 	}
 	return nil, 0, err // Return last written error.
@@ -196,7 +212,7 @@ func nodeFromStackNode(s lneto.StackNode, port uint16, protocol uint64, remoteAd
 		connID:     connIDPtr,
 		callbacks:  makecbnode(s),
 		proto:      uint16(protocol),
-		port:       port,
+		lport:      port,
 		remoteAddr: remoteAddr, // SHARED MEMORY- used to signal.
 	}
 }
@@ -205,3 +221,11 @@ func nodeFromStackNode(s lneto.StackNode, port uint16, protocol uint64, remoteAd
 func (n *node) destroy() {
 	*n = node{}
 }
+
+func incLim(v, max int) int {
+	v++
+	if v == max {
+		v = 0
+	}
+	return v
+}

internet/stack-udpport.go

@@ -24,7 +24,7 @@ func (sudp *StackUDPPort) SetStackNode(node lneto.StackNode, raddr []byte, rmpor
 
 func (sudp *StackUDPPort) Protocol() uint64 { return uint64(lneto.IPProtoUDP) }
 
-func (sudp *StackUDPPort) LocalPort() uint16 { return sudp.h.port }
+func (sudp *StackUDPPort) LocalPort() uint16 { return sudp.h.lport }
 
 func (sudp *StackUDPPort) ConnectionID() *uint64 { return sudp.h.connID }
 
@@ -42,7 +42,7 @@ func (sudp *StackUDPPort) Demux(carrierData []byte, frameOffset int) error {
 		return sudp.vld.ErrPop()
 	}
 	dst := ufrm.DestinationPort()
-	if dst != sudp.h.port {
+	if dst != sudp.h.lport {
 		return lneto.ErrPacketDrop // Not meant for us.
 	}
 	// TODO remote ip address handling.
@@ -70,7 +70,7 @@ func (sudp *StackUDPPort) Encapsulate(carrierData []byte, offsetToIP, offsetToFr
 	if err != nil {
 		return 0, err
 	}
-	ufrm.SetSourcePort(sudp.h.port)
+	ufrm.SetSourcePort(sudp.h.lport)
 	ufrm.SetDestinationPort(sudp.rmport)
 	if len(sudp.raddr) > 0 && offsetToIP >= 0 {
 		err = internal.SetIPAddrs(carrierData[offsetToIP:], 0, nil, sudp.raddr)