Repository: scionproto/scion
Pull request #4748 (2025-04-08 15:09:52)

router: add support for headroom in packet buffers
This will be needed to preserve performance once we have multiple underlays, if some of then need to add headers. Not to mention that we already have one case of moving the payload in the buffer: when making SCMP responses with a packet quote.

Contributes to: #4732 

jiceatscion: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/scionproto/scion/4748)
<!-- Reviewable:end -->

romshark: 




*[`router/dataplane.go` line 183 at r4](https://reviewable.io/reviews/scionproto/scion/4748#-ONy4bL88EMVB0Y8LT1c:-ONy4bL88EMVB0Y8LT1d:bbmyhrx) ([raw file](https://github.com/scionproto/scion/blob/92bab2b2e3af6b4ca1af6edeee83ed26cc2f0a4f/router/dataplane.go#L183)):*
> ```Go
> }
> 
> // PacketPool allocates are resets packets. There is one packet pool per instance of the dataplane,
> ```

The first sentence looks broken, did you mean "and"?


<!-- Sent from Reviewable.io -->

romshark: 




*[`router/dataplane.go` line 743 at r4](https://reviewable.io/reviews/scionproto/scion/4748#-ONy5Hq_CiLdPHJZeyxB:-ONy5Hq_CiLdPHJZeyxC:bxj711z) ([raw file](https://github.com/scionproto/scion/blob/92bab2b2e3af6b4ca1af6edeee83ed26cc2f0a4f/router/dataplane.go#L743)):*
> ```Go
> 
> 	// We round-up the minimum headroom generously so that the extra room is sufficient to allow the
> 	// quoting of most packets by scmp cheaply (that is, without moving the bytes). Our packet
> ```

SCMP (uppercase)


<!-- Sent from Reviewable.io -->

romshark: 




*[`router/dataplane.go` line 2314 at r4](https://reviewable.io/reviews/scionproto/scion/4748#-ONy5XMh05t8of4-fPs9:-ONy5XMh05t8of4-fPsA:bxj711z) ([raw file](https://github.com/scionproto/scion/blob/92bab2b2e3af6b4ca1af6edeee83ed26cc2f0a4f/router/dataplane.go#L2314)):*
> ```Go
> 		}
> 		fmt.Printf("Packet to quote: %d bytes. Total hdrs: %d\n", quoteLen, hdrLen)
> 		// Now that we know the length, we can serialize the scmp headers an the quoted packet. If
> ```

SCMP (uppercase)


<!-- Sent from Reviewable.io -->

romshark: 




*[`router/dataplane.go` line 2314 at r4](https://reviewable.io/reviews/scionproto/scion/4748#-ONy5b-HE7kW9zVR-PTd:-ONy5b-HE7kW9zVR-PTe:bvoagsm) ([raw file](https://github.com/scionproto/scion/blob/92bab2b2e3af6b4ca1af6edeee83ed26cc2f0a4f/router/dataplane.go#L2314)):*
> ```Go
> 		}
> 		fmt.Printf("Packet to quote: %d bytes. Total hdrs: %d\n", quoteLen, hdrLen)
> 		// Now that we know the length, we can serialize the scmp headers an the quoted packet. If
> ```

"an" -> "and"


<!-- Sent from Reviewable.io -->

romshark: 




*[`router/dataplane.go` line 2329 at r4](https://reviewable.io/reviews/scionproto/scion/4748#-ONy5oBOEc1w41Mv0GkD:-ONy5oBOEc1w41Mv0GkE:bxj711z) ([raw file](https://github.com/scionproto/scion/blob/92bab2b2e3af6b4ca1af6edeee83ed26cc2f0a4f/router/dataplane.go#L2329)):*
> ```Go
> 		} else {
> 			// Serialize in front of the quoted packet. The quoted packet must be included in the
> 			// serialize buffer before we packet the scmp header in from of it. AppendBytes will do
> ```

SCMP (uppercase)


<!-- Sent from Reviewable.io -->


Patch:
diff --git a/pkg/slayers/pkt_auth.go b/pkg/slayers/pkt_auth.go
index b9da278d0..78e4eeca3 100644
--- a/pkg/slayers/pkt_auth.go
+++ b/pkg/slayers/pkt_auth.go
@@ -186,9 +186,10 @@ func (o PacketAuthOption) Reset(
 	copy(o.OptData[12:], p.Auth)
 
 	o.OptAlign = [2]uint8{4, 2}
-	// reset unused/implicit fields
-	o.OptDataLen = 0
-	o.ActualLength = 0
+
+	// Set the length field properly. Otherwise the option's data cannot be parsed.
+	o.OptDataLen = byte(len(o.OptData))
+	o.ActualLength = len(o.OptData) + 2
 	return nil
 }
 
diff --git a/pkg/slayers/scmp_msg.go b/pkg/slayers/scmp_msg.go
index 794231296..2e71f9355 100644
--- a/pkg/slayers/scmp_msg.go
+++ b/pkg/slayers/scmp_msg.go
@@ -23,6 +23,12 @@ import (
 	"github.com/scionproto/scion/pkg/private/serrors"
 )
 
+// MaxSCMPHeaderSize is the size of the largest possible SCMP header: including the fixed 4 byte
+// header plus the variable size info block. The maximum is 28 because at 24 bytes
+// SCMPInternalConnectivityDown is the largest info block at the time of writing.  Knowing this
+// allows the router to reduce the cost of constructing SCMP packets that include a packet quote.
+const MaxSCMPHeaderSize = 28
+
 const scmpRawInterfaceLen = 8
 
 // SCMPExternalInterfaceDown message contains the data for that error.
@@ -511,3 +517,21 @@ func decodeSCMPPacketTooBig(data []byte, pb gopacket.PacketBuilder) error {
 	pb.AddLayer(s)
 	return pb.NextDecoder(s.NextLayerType())
 }
+
+// ScmpHeaderSize returns the serialized size a an ScmpHeader given its type. The returned size
+// includes that of the common SCMPHeader (4 bytes), plus the size of the type-dependent info
+// block. This could be implemented as a method of each info block structure, but these headers are
+// accessed via the gopacket.SerializableLayer interface which has no size method. This needs to be
+// updated if new SCMP info blocks are added.
+func ScmpHeaderSize(typeCode SCMPType) int {
+	switch typeCode {
+	case SCMPTypeExternalInterfaceDown:
+		return 20
+	case SCMPTypeInternalConnectivityDown:
+		return 28
+	case SCMPTypeTracerouteRequest, SCMPTypeTracerouteReply:
+		return 24
+	default:
+		return 8
+	}
+}
diff --git a/router/dataplane.go b/router/dataplane.go
index a23d19f03..70caa2434 100644
--- a/router/dataplane.go
+++ b/router/dataplane.go
@@ -75,6 +75,10 @@ const (
 	// Needed to compute required padding
 	ptrSize = unsafe.Sizeof(&struct{ int }{})
 	is32bit = 1 - (ptrSize-4)/4
+
+	// For SCMP packet quoting. A strict minimum of 28 is required. Much more is recommended.
+	minHeadroom      = 512
+	_           uint = minHeadroom - slayers.MaxSCMPHeaderSize // assert >= 28
 )
 
 // BatchConn is a connection that supports batch reads and writes.
@@ -117,8 +121,10 @@ const (
 // size-aligned. We want to fit neatly into cache lines, so we need to fit in 64 bytes. The padding
 // required to occupy exactly 64 bytes depends on the architecture.
 type Packet struct {
-	// The useful part of the raw packet at a point in time (i.e. a slice of the full buffer).
-	// It can be any portion of the full buffer; not necessarily the start.
+	// The useful part of the raw packet at a point in time (i.e. a slice of the full buffer).  It
+	// can be any portion of the full buffer; not necessarily the start. This code maintains the
+	// invariant that RawPacket always represents the portion of a packet that immediately follows
+	// any underlay provider header. See also dataplane.underlayHeadroom.
 	RawPacket []byte
 	// The entire packet buffer. We don't need it as a slice; we know its size.
 	buffer *[bufSize]byte
@@ -158,16 +164,60 @@ func (p *Packet) init(buffer *[bufSize]byte) *Packet {
 	return p
 }
 
-// reset() makes the packet ready to receive a new underlay message.
-// A cleared dstAddr is represented with a zero-length IP so we keep reusing the IP storage bytes.
-func (p *Packet) Reset() {
+// reset() makes the packet ready to receive a new underlay message. We adjust the RawPacket slice
+// relative to the buffer, so there's enough headroom for any underlay headers.
+func (p *Packet) reset(headroom int) {
 	*p = Packet{
-		buffer:    p.buffer,    // keep the buffer
-		RawPacket: p.buffer[:], // restore the full packet capacity
+		buffer:    p.buffer,            // keep the buffer
+		RawPacket: p.buffer[headroom:], // restore the full packet capacity (minus headroom).
 	}
 	// Everything else is reset to zero value.
 }
 
+// WithHeader returns the a slice of the underlying packet buffer that represents the same bytes as
+// p.rawPacket[:] plus the n prededing bytes. This slice is meant to be used when receiving a raw
+// packet with an n bytes header, such that the payload is exactly at p.rawPacket[0:]. p.RawPacket
+// is *not* modified. This method panics if n is greater than the available headroom in the packet
+// buffer.
+func (p *Packet) WithHeader(n int) []byte {
+	headroom := len(p.buffer) - cap(p.RawPacket) - n
+
+	// A negative value is a panicable offense.
+	return p.buffer[headroom:]
+}
+
+// PacketPool allocates and resets packets. There is one packet pool per instance of the dataplane,
+// shared between all its underlay instances. This structure can be shared by copying (and doing so
+// is more efficient) because headroom is never changed after construction and channel is a
+// reference type.
+type PacketPool struct {
+	pool     chan *Packet
+	headroom int
+}
+
+// Get fetches a packet from the pool and returns it initialized with the proper headroom. That is,
+// pkt.rawPacket[0:] is where the packet's payload must go. Underlay providers may use any part of
+// that, and MUST update the pkt.rawPacket slice to indicate where the packet's payload starts.
+// However they may only use the preceding portion of the packet buffer to store a link-layer
+// header. See also WithHeader
+func (p *PacketPool) Get() *Packet {
+	pkt := <-p.pool
+	pkt.reset(p.headroom)
+	return pkt
+}
+
+// Put returns the given packet to the pool.
+func (p *PacketPool) Put(pkt *Packet) {
+	p.pool <- pkt
+
+}
+
+// makePacketPool creates a packetpool of size poolSize, that configures packet buffers with the
+// given headroom. The pool is initially empty. Packets must be added separately.
+func makePacketPool(poolSize, headroom int) PacketPool {
+	return PacketPool{pool: make(chan *Packet, poolSize), headroom: headroom}
+}
+
 // DataPlane contains a SCION Border Router's forwarding logic. It reads packets
 // from multiple sockets, performs routing, and sends them to their destinations
 // (after updating the path, if that is needed).
@@ -197,7 +247,15 @@ type dataPlane struct {
 	// packet structure is fetched from the pool passed-around through the various channels and
 	// returned to the pool. To reduce the cost of copying, the packet structure is passed by
 	// reference.
-	packetPool chan *Packet
+	packetPool PacketPool
+
+	// underlayHeadRoom is the minimum headroom that must be reserved at the front of every packet
+	// to ensure that every underlay provider can prepend its underlay header without copying. It
+	// is established by collecting the headroom requirement of every underlay provider. Underlay
+	// providers deliver incoming packets such that the RawPacket slice starts exactly after the
+	// link layer header. Underlay providers may use the preceding part of the packet buffer to
+	// receive the link layer header.
+	underlayHeadroom int
 }
 
 var (
@@ -632,7 +690,6 @@ type RunConfig struct {
 func (d *dataPlane) Run(ctx context.Context) error {
 	d.mtx.Lock()
 	if d.numInterfaces == 0 {
-		// if len(d.interfaces) == 0 {
 		// Not stritcly an error but we really can't do anything; most maps aren't even allocated,
 		// due to lazy initialization.
 		return nil
@@ -681,16 +738,31 @@ func (d *dataPlane) Run(ctx context.Context) error {
 // initializePacketPool calculates the size of the packet pool based on the
 // current dataplane settings and allocates all the buffers
 func (d *dataPlane) initPacketPool(processorQueueSize int) {
+	// collect pool size and headroom reqs
 	poolSize := d.numInterfaces*d.RunConfig.BatchSize +
 		(d.RunConfig.NumProcessors+d.RunConfig.NumSlowPathProcessors)*(processorQueueSize+1) +
-		d.numInterfaces*(2*d.RunConfig.BatchSize)
+		d.numInterfaces*2*d.RunConfig.BatchSize
+	headroom := 0
+	for _, u := range d.underlays {
+		h := u.Headroom()
+		if headroom < h {
+			headroom = h
+		}
+	}
+	d.underlayHeadroom = headroom
 
-	log.Debug("Initialize packet pool of size", "poolSize", poolSize)
-	d.packetPool = make(chan *Packet, poolSize)
+	// We round-up the minimum headroom generously so that the extra room is sufficient to allow the
+	// quoting of most packets by SCMP cheaply (that is, without moving the bytes). Our packet
+	// buffers are sized at 9000 bytes while in most cases the interface MTU is lower.
+	if headroom < minHeadroom {
+		headroom = minHeadroom
+	}
+	log.Debug("Initialize packet pool", "poolSize", poolSize, "headroom", headroom)
+	d.packetPool = makePacketPool(poolSize, headroom)
 	pktBuffers := make([][bufSize]byte, poolSize)
 	pktStructs := make([]Packet, poolSize)
 	for i := 0; i < poolSize; i++ {
-		d.packetPool <- pktStructs[i].init(&pktBuffers[i])
+		d.packetPool.Put(pktStructs[i].init(&pktBuffers[i]))
 	}
 }
 
@@ -707,14 +779,6 @@ func (d *dataPlane) initQueues(processorQueueSize int) ([]chan *Packet, []chan *
 	return procQs, slowQs
 }
 
-func (d *dataPlane) getPacketFromPool() *Packet {
-	return <-d.packetPool
-}
-
-func (d *dataPlane) returnPacketToPool(pkt *Packet) {
-	d.packetPool <- pkt
-}
-
 func (d *dataPlane) runProcessor(id int, q <-chan *Packet, slowQ chan<- *Packet) {
 	log.Debug("Initialize processor with", "id", id)
 	processor := newPacketProcessor(d)
@@ -738,30 +802,30 @@ func (d *dataPlane) runProcessor(id int, q <-chan *Packet, slowQ chan<- *Packet)
 			case slowQ <- p:
 			default:
 				metrics[sc].DroppedPacketsBusySlowPath.Inc()
-				d.returnPacketToPool(p)
+				d.packetPool.Put(p)
 			}
 			continue
 		case pDone: // Packets that don't need more processing (e.g. BFD)
-			d.returnPacketToPool(p)
+			d.packetPool.Put(p)
 			continue
 		case pDiscard: // Everything else
 			metrics[sc].DroppedPacketsInvalid.Inc()
-			d.returnPacketToPool(p)
+			d.packetPool.Put(p)
 			continue
 		default: // Newly added dispositions need to be handled.
 			log.Debug("Unknown packet disposition", "disp", disp)
-			d.returnPacketToPool(p)
+			d.packetPool.Put(p)
 			continue
 		}
 		fwLink := d.interfaces[p.egress]
 		if fwLink == nil {
 			log.Debug("Error determining forwarder. Egress is invalid", "egress", p.egress)
+			d.packetPool.Put(p)
 			metrics[sc].DroppedPacketsInvalid.Inc()
-			d.returnPacketToPool(p)
 			continue
 		}
 		if !fwLink.Send(p) {
-			d.returnPacketToPool(p)
+			d.packetPool.Put(p)
 			metrics[sc].DroppedPacketsBusyForwarder.Inc()
 		}
 	}
@@ -780,7 +844,7 @@ func (d *dataPlane) runSlowPathProcessor(id int, q <-chan *Packet) {
 			log.Debug("Error processing packet", "err", err)
 			sc := ClassOfSize(len(p.RawPacket))
 			p.Link.Metrics()[sc].DroppedPacketsInvalid.Inc()
-			d.returnPacketToPool(p)
+			d.packetPool.Put(p)
 			continue
 		}
 		// All slowpath packets are responses to the sender. Therefore, the egress link is always
@@ -790,11 +854,11 @@ func (d *dataPlane) runSlowPathProcessor(id int, q <-chan *Packet) {
 		if egressLink == nil {
 			// Someone tried to send a freshly made packet on the slow path?
 			log.Debug("Error determining return link. No ingress link")
-			d.returnPacketToPool(p)
+			d.packetPool.Put(p)
 			continue
 		}
 		if !egressLink.Send(p) {
-			d.returnPacketToPool(p)
+			d.packetPool.Put(p)
 		}
 	}
 }
@@ -2098,12 +2162,11 @@ func (b *bfdSend) Send(bfd *layers.BFD) error {
 		ohp.FirstHop.Mac = path.MAC(b.mac, ohp.Info, ohp.FirstHop, b.macBuffer)
 	}
 
-	p := b.dataPlane.getPacketFromPool()
-	p.Reset()
+	p := b.dataPlane.packetPool.Get()
 
 	serBuf := newSerializeProxy(p.RawPacket) // set for prepend-only by default. Perfect here.
 
-	// serialized bytes lend directly into p.RawPacket (alignedd at the end).
+	// serialized bytes lend directly into p.RawPacket (aligned at the end).
 	err := gopacket.SerializeLayers(&serBuf, gopacket.SerializeOptions{FixLengths: true},
 		b.scn, bfd)
 	if err != nil {
@@ -2121,7 +2184,7 @@ func (b *bfdSend) Send(bfd *layers.BFD) error {
 	if !fwLink.Send(p) {
 		// We do not care if some BFD packets get bounced under high load. If it becomes a problem,
 		// the solution is do use BFD's demand-mode. To be considered in a future refactoring.
-		b.dataPlane.returnPacketToPool(p)
+		b.dataPlane.packetPool.Put(p)
 	}
 	return err
 }
@@ -2226,43 +2289,77 @@ func (p *slowPathPacketProcessor) prepareSCMP(
 				p.hasValidAuth(time.Now()))
 	}
 
-	var quote []byte
+	sopts := gopacket.SerializeOptions{
+		ComputeChecksums: true,
+		FixLengths:       true,
+	}
+	var serBuf serializeProxy
+
+	// First write the SCMP message only without the SCION header(s) to get a buffer that we can
+	// feed to the MAC computation. If this is an error response, then it has to include a quote of
+	// the packet at the end of the SCMP message.
+
 	if isError {
-		// add quote for errors.
-		hdrLen := slayers.CmnHdrLen + scionL.AddrHdrLen() + scionL.Path.Len()
+		// There is headroom built into the packet buffer so we can wrap the whole packet into a new
+		// one without copying it. We need to reclaim that headroom so we can prepend. We can figure
+		// the current headroom, even if it was changed, by comparing the capacity of the slice with
+		// our constant buffer size.
+		quoteLen := len(p.pkt.RawPacket)
+		headroom := len(p.pkt.buffer) - cap(p.pkt.RawPacket)
+		hdrLen := slayers.CmnHdrLen + scionL.AddrHdrLen() + scionL.Path.Len() +
+			slayers.ScmpHeaderSize(scmpH.TypeCode.Type())
+
 		if needsAuth {
 			hdrLen += e2eAuthHdrLen
 		}
-		switch scmpH.TypeCode.Type() {
-		case slayers.SCMPTypeExternalInterfaceDown:
-			hdrLen += 20
-		case slayers.SCMPTypeInternalConnectivityDown:
-			hdrLen += 28
-		default:
-			hdrLen += 8
-		}
-		quote = p.pkt.RawPacket
 		maxQuoteLen := slayers.MaxSCMPPacketLen - hdrLen
-		if len(quote) > maxQuoteLen {
-			quote = quote[:maxQuoteLen]
+		if quoteLen > maxQuoteLen {
+			quoteLen = maxQuoteLen
+		}
+		// Now that we know the length, we can serialize the SCMP headers and the quoted packet. If
+		// we don't fit in the headroom we copy the quoted packet to the end. We are required to
+		// leave space for a worst-case underlay header too. TODO(multi_underlay): since we know
+		// that this goes back via the link it came from, we could be content with leaving just
+		// enough headroom for this specific underlay.
+		if hdrLen+p.d.underlayHeadroom > headroom {
+			// Not enough headroom. Pack at end.
+			quote := p.pkt.RawPacket[:quoteLen]
+			serBuf = newSerializeProxy(p.pkt.RawPacket)
+			err = gopacket.SerializeLayers(&serBuf, sopts, &scmpH, scmpP, gopacket.Payload(quote))
+			if err != nil {
+				return serrors.JoinNoStack(
+					errCannotRoute, err, "details", "serializing SCMP message")
+			}
+		} else {
+			// Serialize in front of the quoted packet. The quoted packet must be included in the
+			// serialize buffer before we pack the SCMP header in from of it. AppendBytes will do
+			// that; it exposes the underlying buffer but doesn't modify it.
+			p.pkt.RawPacket = p.pkt.buffer[0:(quoteLen + headroom)]
+			serBuf = newSerializeProxyStart(p.pkt.RawPacket, headroom)
+			_, _ = serBuf.AppendBytes(quoteLen) // Implementation never fails.
+			err = scmpP.SerializeTo(&serBuf, sopts)
+			if err != nil {
+				return serrors.JoinNoStack(
+					errCannotRoute, err, "details", "serializing SCMP message")
+			}
+			err = scmpH.SerializeTo(&serBuf, sopts)
+			if err != nil {
+				return serrors.JoinNoStack(
+					errCannotRoute, err, "details", "serializing SCMP message")
+			}
+		}
+	} else {
+		// We do not need to preserve the packet. Just pack our headers at the end of the buffer.
+		// (this is what serializeProxy does by default).
+		serBuf = newSerializeProxy(p.pkt.RawPacket)
+		err = gopacket.SerializeLayers(&serBuf, sopts, &scmpH, scmpP)
+		if err != nil {
+			return serrors.JoinNoStack(errCannotRoute, err, "details", "serializing SCMP message")
 		}
 	}
 
-	serBuf := newSerializeProxy(p.pkt.RawPacket) // Prepend-only by default. It's all we need.
-	sopts := gopacket.SerializeOptions{
-		ComputeChecksums: true,
-		FixLengths:       true,
-	}
-	// First write the SCMP message only without the SCION header(s) to get a buffer that we
-	// can (re-)use as input in the MAC computation. Note that we move the quoted part of the packet
-	// to the end of the buffer (go supports overlaps properly).
-	// TODO(jiceatscion): in the future we may be able to leave room at the head of the
-	// buffer on ingest, so we won't need to move the quote at all.
-	err = gopacket.SerializeLayers(&serBuf, sopts, &scmpH, scmpP, gopacket.Payload(quote))
-	if err != nil {
-		return serrors.JoinNoStack(errCannotRoute, err, "details", "serializing SCMP message")
-	}
-
+	// serBuf now starts with the SCMP Headers and ends with the truncated quoted packet, if any.
+	// This is what gets checksumed.
 	if needsAuth {
 		var e2e slayers.EndToEndExtn
 		scionL.NextHdr = slayers.End2EndClass
@@ -2304,9 +2401,13 @@ func (p *slowPathPacketProcessor) prepareSCMP(
 	} else {
 		scionL.NextHdr = slayers.L4SCMP
 	}
+
+	// Our SCION header is ready. Prepend it.
 	if err := scionL.SerializeTo(&serBuf, sopts); err != nil {
 		return serrors.JoinNoStack(errCannotRoute, err, "details", "serializing SCION header")
 	}
+
+	// serBuf now has the exact slice that represents the packet.
 	p.pkt.RawPacket = serBuf.Bytes()
 
 	log.Debug("SCMP", "typecode", scmpH.TypeCode)
diff --git a/router/serialize_proxy.go b/router/serialize_proxy.go
index 2492b7403..7f18c4e8f 100644
--- a/router/serialize_proxy.go
+++ b/router/serialize_proxy.go
@@ -43,7 +43,7 @@ func newSerializeProxy(buf []byte) serializeProxy {
 }
 
 // newSerializeProxyStart returns a new serializeProxy. The initial prepend/append point is set to
-// the given start value. This has the same effect as calling clear(statr).
+// the given start value. This has the same effect as calling clear(start).
 func newSerializeProxyStart(buf []byte, start int) serializeProxy {
 	serBuf := serializeProxy{
 		data: buf,
diff --git a/router/underlay.go b/router/underlay.go
index 042fa297a..5e8cb44bd 100644
--- a/router/underlay.go
+++ b/router/underlay.go
@@ -67,11 +67,12 @@ type Link interface {
 
 // A provider of connectivity over some underlay implementation
 //
-// For any given underlay, there are three kinds of Link implementations to choose from.
-// The difference between them is the intent regarding addressing.
+// For any given underlay, there are three kinds of Link implementations to choose from. The
+// difference between them is the intent regarding addressing.
 //
 // TODO(multi_underlay): The local internal address is explicitly a udpip underlay address as the
-// main router code still assumes that the internal network underlay is always "udp/ip".
+// main router code as well as the entire end-host stack still assume that the internal network
+// underlay is always "udp/ip".
 type UnderlayProvider interface {
 
 	// SetConnOpener is a unit testing device: it allows the replacement of the function
@@ -84,6 +85,14 @@ type UnderlayProvider interface {
 	// NumConnections returns the current number of configured connections.
 	NumConnections() int
 
+	// Headroom returns the length of the largest header possibly added by this underlay.
+	// The router core will ensure that all received packets are stored at an offset in the packet
+	// buffer, such that the largest underlay header declared across all underlay providers can
+	// be prepended to the SCION header without having to copy the packet or to allocate a separate
+	// buffer. Note that, in most cases, this depends on the cooperation of the ingest underlay
+	// provider.
+	Headroom() int
+
 	// SetDispatchPorts sets the range of auto-dispatched ports and default endhost port (the shim
 	// dispatcher port). When translating a SCION port into an underlay port, any port between the
 	// values of start and end remains unchanged, while any other will be replaced by the value of
@@ -101,7 +110,7 @@ type UnderlayProvider interface {
 	// incoming packets to its output channels and will send packets present on its input
 	// channels. Only connection in existence at the time of calling Start() will be
 	// started. Calling Start has no effect on already running connections.
-	Start(ctx context.Context, pool chan *Packet, proQs []chan *Packet)
+	Start(ctx context.Context, pool PacketPool, proQs []chan *Packet)
 
 	// Stop puts the provider in the stopped state. In that state, the provider no longer delivers
 	// incoming packets and ignores packets present on its input channels. The provider is fully
diff --git a/router/underlayproviders/udpip/udpip.go b/router/underlayproviders/udpip/udpip.go
index f7c5f3d39..9299f595e 100644
--- a/router/underlayproviders/udpip/udpip.go
+++ b/router/underlayproviders/udpip/udpip.go
@@ -90,7 +90,7 @@ type provider struct {
 
 type udpLink interface {
 	router.Link
-	start(ctx context.Context, procQs []chan *router.Packet, pool chan *router.Packet)
+	start(ctx context.Context, procQs []chan *router.Packet, pool router.PacketPool)
 	stop()
 	receive(size int, srcAddr *net.UDPAddr, p *router.Packet)
 }
@@ -127,6 +127,12 @@ func (u *provider) NumConnections() int {
 	return len(u.allLinks)
 }
 
+func (u *provider) Headroom() int {
+	// This underlay does not add any header of its own: the UDP socket API manages the header
+	// independently.
+	return 0
+}
+
 func (u *provider) SetDispatchPorts(start, end, redirect uint16) {
 	u.dispatchStart = start
 	u.dispatchEnd = end
@@ -157,7 +163,7 @@ func (u *provider) DelSvc(svc addr.SVC, host addr.Host, port uint16) error {
 // The queues to be used by the receiver task are supplied at this point because they must be
 // sized according to the number of connections that will be started.
 func (u *provider) Start(
-	ctx context.Context, pool chan *router.Packet, procQs []chan *router.Packet,
+	ctx context.Context, pool router.PacketPool, procQs []chan *router.Packet,
 ) {
 	u.mu.Lock()
 	if len(procQs) == 0 {
@@ -211,7 +217,7 @@ type udpConnection struct {
 
 // start puts the connection in the running state. In that state, the connection can deliver
 // incoming packets and ignores packets present on its input channel.
-func (u *udpConnection) start(batchSize int, pool chan *router.Packet) {
+func (u *udpConnection) start(batchSize int, pool router.PacketPool) {
 	wasRunning := u.running.Swap(true)
 	if wasRunning {
 		return
@@ -247,7 +253,7 @@ func (u *udpConnection) stop() {
 	}
 }
 
-func (u *udpConnection) receive(batchSize int, pool chan *router.Packet) {
+func (u *udpConnection) receive(batchSize int, pool router.PacketPool) {
 	log.Debug("Receive", "connection", u.name)
 
 	// A collection of socket messages, as the readBatch API expects them. We keep using the same
@@ -264,8 +270,7 @@ func (u *udpConnection) receive(batchSize int, pool chan *router.Packet) {
 
 		// Give a new buffer to the msgs elements that have been used in the previous loop.
 		for i := 0; i < batchSize-numReusable; i++ {
-			p := <-pool
-			p.Reset()
+			p := pool.Get()
 			packets[i] = p
 			msgs[i].Buffers[0] = p.RawPacket
 		}
@@ -306,7 +311,7 @@ func (u *udpConnection) receive(batchSize int, pool chan *router.Packet) {
 	// We have to stop receiving. Return the unused packets to the pool to avoid creating
 	// a memory leak (the process is not required to exit - e.g. in tests).
 	for _, p := range packets[batchSize-numReusable : batchSize] {
-		pool <- p
+		pool.Put(p)
 	}
 }
 
@@ -335,7 +340,7 @@ func readUpTo(queue <-chan *router.Packet, n int, needsBlocking bool, pkts []*ro
 	return i
 }
 
-func (u *udpConnection) send(batchSize int, pool chan *router.Packet) {
+func (u *udpConnection) send(batchSize int, pool router.PacketPool) {
 	log.Debug("Send", "connection", u.name)
 
 	// We use this somewhat like a ring buffer.
@@ -377,13 +382,13 @@ func (u *udpConnection) send(batchSize int, pool chan *router.Packet) {
 		}
 		router.UpdateOutputMetrics(metrics, pkts[:written])
 		for _, p := range pkts[:written] {
-			pool <- p
+			pool.Put(p)
 		}
 		if written != toWrite {
 			// Only one is dropped at this time. We'll retry the rest.
 			sc := router.ClassOfSize(len(pkts[written].RawPacket))
 			metrics[sc].DroppedPacketsInvalid.Inc()
-			pool <- pkts[written]
+			pool.Put(pkts[written])
 			toWrite -= (written + 1)
 			// Shift the leftovers to the head of the buffers.
 			for i := 0; i < toWrite; i++ {
@@ -419,7 +424,7 @@ type connectedLink struct {
 	name       string // For logs
 	egressQ    chan<- *router.Packet
 	metrics    *router.InterfaceMetrics
-	pool       chan *router.Packet
+	pool       router.PacketPool
 	bfdSession *bfd.Session
 	seed       uint32
 	ifID       uint16
@@ -500,7 +505,7 @@ func (u *provider) newConnectedLink(
 func (l *connectedLink) start(
 	ctx context.Context,
 	procQs []chan *router.Packet,
-	pool chan *router.Packet,
+	pool router.PacketPool,
 ) {
 	// procQs and pool are never known before all configured links have been instantiated.  So we
 	// get them only now. We didn't need it earlier since the connections have not been started yet.
@@ -571,7 +576,7 @@ func (l *connectedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet
 	procID, err := computeProcID(p.RawPacket, len(l.procQs), l.seed)
 	if err != nil {
 		log.Debug("Error while computing procID", "err", err)
-		l.pool <- p
+		l.pool.Put(p)
 		metrics[sc].DroppedPacketsInvalid.Inc()
 		return
 	}
@@ -582,7 +587,7 @@ func (l *connectedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet
 	select {
 	case l.procQs[procID] <- p:
 	default:
-		l.pool <- p
+		l.pool.Put(p)
 		metrics[sc].DroppedPacketsBusyProcessor.Inc()
 	}
 }
@@ -595,7 +600,7 @@ type detachedLink struct {
 	name       string // For logs
 	egressQ    chan<- *router.Packet
 	metrics    *router.InterfaceMetrics
-	pool       chan *router.Packet
+	pool       router.PacketPool
 	bfdSession *bfd.Session
 	remote     *net.UDPAddr
 	seed       uint32
@@ -669,7 +674,7 @@ func (u *provider) newDetachedLink(
 func (l *detachedLink) start(
 	ctx context.Context,
 	procQs []chan *router.Packet,
-	pool chan *router.Packet,
+	pool router.PacketPool,
 ) {
 	// procQs and pool are never known before all configured links have been instantiated.  So we
 	// get them only now. We didn't need it earlier since the connections have not been started yet.
@@ -746,7 +751,7 @@ func (l *detachedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet)
 	procID, err := computeProcID(p.RawPacket, len(l.procQs), l.seed)
 	if err != nil {
 		log.Debug("Error while computing procID", "err", err)
-		l.pool <- p
+		l.pool.Put(p)
 		metrics[sc].DroppedPacketsInvalid.Inc()
 		return
 	}
@@ -757,7 +762,7 @@ func (l *detachedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet)
 	select {
 	case l.procQs[procID] <- p:
 	default:
-		l.pool <- p
+		l.pool.Put(p)
 		metrics[sc].DroppedPacketsBusyProcessor.Inc()
 	}
 }
@@ -766,7 +771,7 @@ type internalLink struct {
 	procQs           []chan *router.Packet
 	egressQ          chan *router.Packet
 	metrics          *router.InterfaceMetrics
-	pool             chan *router.Packet
+	pool             router.PacketPool
 	svc              *router.Services[netip.AddrPort]
 	seed             uint32
 	dispatchStart    uint16
@@ -836,7 +841,7 @@ func (u *provider) NewInternalLink(
 func (l *internalLink) start(
 	ctx context.Context,
 	procQs []chan *router.Packet,
-	pool chan *router.Packet,
+	pool router.PacketPool,
 ) {
 	// procQs and pool are never known before all configured links have been instantiated. So we
 	// get them only now. We didn't need it earlier since the connections have not been started yet.
@@ -935,7 +940,7 @@ func (l *internalLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet)
 	procID, err := computeProcID(p.RawPacket, len(l.procQs), l.seed)
 	if err != nil {
 		log.Debug("Error while computing procID", "err", err)
-		l.pool <- p
+		l.pool.Put(p)
 		metrics[sc].DroppedPacketsInvalid.Inc()
 		return
 	}
@@ -952,7 +957,7 @@ func (l *internalLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet)
 	select {
 	case l.procQs[procID] <- p:
 	default:
-		l.pool <- p
+		l.pool.Put(p)
 		metrics[sc].DroppedPacketsBusyProcessor.Inc()
 	}
 }


Test patch:
diff --git a/router/dataplane_internal_test.go b/router/dataplane_internal_test.go
index 48b366d62..91a4d7143 100644
--- a/router/dataplane_internal_test.go
+++ b/router/dataplane_internal_test.go
@@ -95,7 +95,7 @@ func TestReceiver(t *testing.T) {
 
 	dp.initPacketPool(64)
 	procCh, _ := dp.initQueues(64)
-	initialPoolSize := len(dp.packetPool)
+	initialPoolSize := len(dp.packetPool.pool)
 	dp.setRunning()
 	dp.underlays["udpip"].Start(context.Background(), dp.packetPool, procCh)
 	ptrMap := make(map[uintptr]struct{})
@@ -103,7 +103,7 @@ func TestReceiver(t *testing.T) {
 		select {
 		case pkt := <-procCh[0]:
 			// make sure that the pool size has decreased
-			assert.Greater(t, initialPoolSize, len(dp.packetPool))
+			assert.Greater(t, initialPoolSize, len(dp.packetPool.pool))
 			// make sure that the packet has the right size
 			assert.Equal(t, 84+i%10*18, len(pkt.RawPacket))
 			// make sure that the source address was set correctly
@@ -128,7 +128,7 @@ func TestReceiver(t *testing.T) {
 	dp.setStopping()
 
 	// make sure that the packet pool has the expected size after the test
-	assert.Equal(t, initialPoolSize-dp.RunConfig.BatchSize-20, len(dp.packetPool))
+	assert.Equal(t, initialPoolSize-dp.RunConfig.BatchSize-20, len(dp.packetPool.pool))
 	dp.underlays["udpip"].Stop()
 }
 
@@ -231,20 +231,19 @@ func TestForwarder(t *testing.T) {
 	procQs, _ := dp.initQueues(64)
 	intf := dp.interfaces[0]
 	extf := dp.interfaces[42]
-	initialPoolSize := len(dp.packetPool)
+	initialPoolSize := len(dp.packetPool.pool)
 	dp.setRunning()
 	dp.underlays["udpip"].Start(context.Background(), dp.packetPool, procQs)
 	dstAddr := &net.UDPAddr{IP: net.IP{10, 0, 200, 200}}
 	for i := 0; i < 255; i++ {
-		pkt := <-dp.packetPool
-		pkt.Reset()
+		pkt := dp.packetPool.Get()
 		pkt.RawPacket = pkt.RawPacket[:1]
 		pkt.RawPacket[0] = byte(i)
 		if i < 100 {
 			pkt.RemoteAddr = unsafe.Pointer(dstAddr)
 		}
 
-		assert.NotEqual(t, initialPoolSize, len(dp.packetPool))
+		assert.NotEqual(t, initialPoolSize, len(dp.packetPool.pool))
 
 		// Normal use would be
 		// intf.Send(pkt):
@@ -254,7 +253,7 @@ func TestForwarder(t *testing.T) {
 		} else {
 			if i == 100 {
 				// take a short break, else the two links will run in parallel and our ordering
-				// check will see it.
+				// check will see it (since we use the same mock connection for both).
 				time.Sleep(200 * time.Millisecond)
 			}
 			extf.SendBlocking(pkt)
@@ -266,7 +265,7 @@ func TestForwarder(t *testing.T) {
 		dp.underlays["udpip"].Stop()
 		dp.setStopping()
 		time.Sleep(100 * time.Millisecond)
-		assert.Equal(t, initialPoolSize, len(dp.packetPool))
+		assert.Equal(t, initialPoolSize, len(dp.packetPool.pool))
 	case <-time.After(100 * time.Millisecond):
 		dp.underlays["udpip"].Stop()
 		dp.setStopping()
@@ -467,20 +466,15 @@ func TestSlowPathProcessing(t *testing.T) {
 			dp := tc.prepareDP(ctrl)
 
 			rp := tc.mockMsg()
-			// Cannot use NewPacket. We need a packet that can grow to accommodate SCMP w/ quote.
-			pkt := Packet{}
-			pkt.init(&[bufSize]byte{})
-			pkt.Reset()
+			pkt := NewPacket(rp, nil, nil, tc.srcInterface, 0)
 			pkt.Link = newMockLink(tc.srcInterface)
-			pkt.RawPacket = pkt.RawPacket[:len(rp)]
-			copy(pkt.RawPacket, rp)
 
 			processor := newPacketProcessor(dp)
-			disp := processor.processPkt(&pkt)
+			disp := processor.processPkt(pkt)
 			assert.Equal(t, pSlowPath, disp)
 			assert.Equal(t, tc.expectedSlowPathRequest, pkt.slowPathRequest)
 			slowPathProcessor := newSlowPathProcessor(dp)
-			err := slowPathProcessor.processPacket(&pkt)
+			err := slowPathProcessor.processPacket(pkt)
 			assert.NoError(t, err)
 
 			// here we parse the outgoing packet to verify that it contains the correct SCMP
diff --git a/router/export_test.go b/router/export_test.go
index 06cd18022..f300bc53e 100644
--- a/router/export_test.go
+++ b/router/export_test.go
@@ -64,12 +64,13 @@ var _ Link = new(MockLink)
 
 func newMockLink(ingress uint16) Link { return &MockLink{ifID: ingress} }
 
-// NewPacket makes a mock packet. It has one shortcoming which makes it unsuited for some tests: The
-// packet buffer is strictly no bigger than the supplied bytes; which means that it cannot be used
-// to respond via SCMP. Also, it refers to a mock link that has the scope Internal in all cases.
+// NewPacket makes a mock packet. It has shortcomings which makes it unsuited for some tests: it
+// refers to a mock link that has the scope Internal in all cases, and a blank remote address.
 func NewPacket(raw []byte, src, dst *net.UDPAddr, ingress, egress uint16) *Packet {
+	pktBuf := &([bufSize]byte{})
 	p := Packet{
-		RawPacket: make([]byte, len(raw)),
+		buffer:    pktBuf,
+		RawPacket: pktBuf[minHeadroom:],
 		egress:    egress,
 		Link:      newMockLink(ingress),
 	}
@@ -79,6 +80,7 @@ func NewPacket(raw []byte, src, dst *net.UDPAddr, ingress, egress uint16) *Packe
 	if dst != nil {
 		p.RemoteAddr = unsafe.Pointer(dst)
 	}
+	p.RawPacket = p.RawPacket[:len(raw)]
 	copy(p.RawPacket, raw)
 	return &p
 }