Repository: scionproto/scion
Pull request #4518 (2024-05-03 11:25:28)

testing: incremental improvements to the router benchmark tooling
Summary of changes:
* The packet size is now configurable. CI uses a constant while the benchmark command accepts an optional argument.
* The benchmark command sets the mtu to 9000 on the host side and instructs the user to do the same on the router side. In the CI test variant: an mtu of 9000 (not 8000) is used on both sides.
* The performance index formula gained a tentative value for the NIC_CONSTANT constant.
* The benchmark's total number of packets (per testcase) is no longer tunable. Instead, the duration is what is configurable. Both the CI test and the benchmark command use 13 seconds per test case.
* The benchmark's output includes a bit more information: bit rate and raw (not corrected for preemptions) packet rate.
* A useless and noisy error message was silenced in brload: that which was output for every foreign packet received (turns out that some Realtek hardware sends proprietary management traffic as Ethernet frames).
* Improved the relevance of mmbm by deliberately misaligning buffers. Also make sure to not use more buffers than can fit in L3 cache when probing for L2 cache + TLB misses.
* Added zip as a dependency after finding it isn't part of Debian by default.
* Added an option (via constant for now) to profile the router during the CI test.

matzf: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/scionproto/scion/4518)
<!-- Reviewable:end -->

jiceatscion: Matthias, I think I have addressed your comments. I also added the profiling. Is this good to go?


Patch:
diff --git a/acceptance/router_benchmark/benchmark.py b/acceptance/router_benchmark/benchmark.py
index 86518b322..9f942b1e9 100755
--- a/acceptance/router_benchmark/benchmark.py
+++ b/acceptance/router_benchmark/benchmark.py
@@ -95,6 +95,8 @@ class RouterBMTool(cli.Application, RouterBM):
                               help="The coremark score of the subject machine.")
     mmbm = cli.SwitchAttr(["m", "mmbm"], int, default=0,
                           help="The mmbm score of the subject machine.")
+    packet_size = cli.SwitchAttr(["s", "size"], int, default=172,
+                                 help="Test packet size (includes all headers - floored at 154).")
     intf_map: dict[str, Intf] = {}
     brload: LocalCommand = local["./bin/brload"]
     brload_cpus: list[int] = []
@@ -158,8 +160,7 @@ class RouterBMTool(cli.Application, RouterBM):
             if i.name == host_intf:
                 break
         else:
-            # TODO: instructions/warning regarding inability to enable jumbo frames.
-            # sudo("ip", "link", "set", host_intf, "mtu", "8000")
+            sudo("ip", "link", "set", host_intf, "mtu", "9000")
 
             # Do not assign the host addresses but create one link-local addr.
             # Brload needs some src IP to send arp requests. (This requires rp_filter
@@ -291,16 +292,16 @@ INSTRUCTIONS:
     bmtools includes two microbenchmarks: scion-coremark and scion-mmbm. Those will run
     automatically and the results will be used to improve the benchmark report.
 
-    Optinal: If you did not install bmtools.ipk, install and run those microbenchmark and make a
+    Optional: If you did not install bmtools.ipk, install and run those microbenchmarks and make a
     note of the results: (scion-coremark; scion-mmbm).
 
 2 - Configure the following interfaces on your router (The procedure depends on your router
-    UI):
+    UI) - All interfaces should have the mtu set to 9000:
     - One physical interface with addresses: {", ".join(multiplexed)}
 {nl.join(['    - One physical interface with address: ' + s for s in exclusives])}
 
     IMPORTANT: if you're using a partitioned network (eg. multiple switches or no switches),
-    the "must reach" annotation matters. The 'h' number is the order in which the corresponding host
+    the "must reach" annotation matters. The '#' number is the order in which the corresponding host
     interface must be given on the command line in step 7.
 
 3 - Connect the corresponding ports into your test switch (best if dedicated for the test).
diff --git a/acceptance/router_benchmark/benchmarklib.py b/acceptance/router_benchmark/benchmarklib.py
index a4666b986..cbcced3f2 100644
--- a/acceptance/router_benchmark/benchmarklib.py
+++ b/acceptance/router_benchmark/benchmarklib.py
@@ -33,13 +33,11 @@ logger = logging.getLogger(__name__)
 # to retrieve a frame. That one is hardware dependent and must be found by a third benchmark, so
 # it is not theoretically a constant, but keeping it here to not forget. Until then, our performance
 # index isn't really valid cross-hardware. M_COEF=400 gives roughly consistent results with the
-# hardware we have. So, using that until we know more.
+# hardware we have. So, using that until we know more. NIC_CONSTANT seems to be around
+# 1 microsecond. Using that, provisionally.
 
 M_COEF = 400
-NIC_CONSTANT = 0
-
-# TODO(jiceatscion): get it from or give it to brload?
-BM_PACKET_LEN = 172
+NIC_CONSTANT = 1.0/1000000
 
 # Intf: description of an interface configured for brload's use. Depending on context
 # mac and peermac may be unused. "mac" is the MAC address configured on the side of the subject
@@ -55,31 +53,36 @@ class Results:
     cores: int = 0
     coremark: int = 0
     mmbm: int = 0
+    packet_size: int = 0
     cases: list[dict] = []
     failed: list[dict] = []
     checked: bool = False
 
-    def __init__(self, cores: int, coremark: int, mmbm: int):
+    def __init__(self, cores: int, coremark: int, mmbm: int, packet_size: int):
         self.cores = cores
         self.coremark = coremark
         self.mmbm = mmbm
+        self.packet_size = packet_size
 
     def perf_index(self, rate: int) -> float:
         # TODO(jiceatscion): The perf index assumes that line speed isn't the bottleneck.
         # It almost never is, but ideally we'd need to run iperf3 to verify.
         # mmbm is in mebiBytes/s, rate is in pkt/s
         return rate * (1.0 / self.coremark +
-                       M_COEF * BM_PACKET_LEN / (self.mmbm * 1024 * 1024) +
+                       M_COEF * self.packet_size / (self.mmbm * 1024 * 1024) +
                        NIC_CONSTANT)
 
-    def add_case(self, name: str, rate: int, droppage: int):
+    def add_case(self, name: str, rate: int, droppage: int, raw_rate: int):
         dropRatio = round(float(droppage) / (rate + droppage), 2)
         saturated = dropRatio > 0.03
         perf = 0.0
         if self.cores == 3 and self.coremark and self.mmbm:
             perf = round(self.perf_index(rate), 1)
         self.cases.append({"case": name,
-                           "perf": perf, "rate": rate, "drop": dropRatio, "full": saturated})
+                           "perf": perf, "rate": rate, "drop": dropRatio,
+                           "bit_rate": rate * self.packet_size * 8,
+                           "raw_pkt_rate": raw_rate,
+                           "full": saturated})
 
     def CI_check(self, expectations: dict[str, int]):
         self.checked = True
@@ -147,6 +150,7 @@ class RouterBM():
     This class is a Mixin that borrows the following attributes from the host class:
     * coremark: the coremark benchmark results.
     * mmbm: the mmbm benchmark results.
+    * packet_size: the packet_size to use in the test cases.
     * intf_map: the map "label->actual_interface" map to be passed to brload.
     * brload: "localCmd" wraper for the brload executable (plumbum.machines.LocalCommand)
     * brload_cpus: [int] cpus where it is acceptable to run brload ([] means any)
@@ -154,7 +158,7 @@ class RouterBM():
     * prom_address: the address of the prometheus API a string in the form "host:port"
     """
 
-    def exec_br_load(self, case: str, map_args: list[str], count: int) -> str:
+    def exec_br_load(self, case: str, map_args: list[str], duration: int) -> str:
         # For num-streams, attempt to distribute uniformly on many possible number of cores.
         # 840 is a multiple of 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 15, 20, 21, 24, 28, ...
         brload_args = [
@@ -163,8 +167,9 @@ class RouterBM():
             "--artifacts", self.artifacts,
             *map_args,
             "--case", case,
-            "--num-packets", str(count),
+            "--duration", f"{duration}s",
             "--num-streams", "840",
+            "--packet-size", f"{self.packet_size}",
         ]
         if self.brload_cpus:
             brload_args = [
@@ -176,20 +181,20 @@ class RouterBM():
     def run_test_case(self, case: str, map_args: list[str]) -> (int, int):
         logger.debug(f"==> Starting load {case}")
 
-        output = self.exec_br_load(case, map_args, 10000000)
-        beg = "0"
+        # We transmit for 13 seconds and then ignore the first 3.
+        output = self.exec_br_load(case, map_args, 13)
         end = "0"
         for line in output.splitlines():
             if line.startswith("metricsBegin"):
-                _, beg, _, end = line.split()
+                end = line.split()[3]  # "... metricsEnd: <end>"
 
         logger.debug(f"==> Collecting {case} performance metrics...")
 
         # The raw metrics are expressed in terms of core*seconds. We convert to machine*seconds
         # which allows us to provide a projected packet/s; ...more intuitive than packets/core*s.
-        # We measure the rate over 10s. For best results we sample the end of the middle 10s of the
-        # run. "beg" is the start time of the real action and "end" is the end time.
-        sampleTime = (int(beg) + int(end) + 10) / 2
+        # We measure the rate over 10s. For best results we only look at the last 10 seconds.
+        # "end" reports a time when the transmission was still going on at maximum rate.
+        sampleTime = int(end)
         prom_query = urlencode({
             'time': f'{sampleTime}',
             'query': (
@@ -218,6 +223,31 @@ class RouterBM():
             processed = int(float(val))
             break
 
+        # Collect the raw packet rate too. Just so we can discover if the cpu-availability
+        # correction is bad.
+        prom_query = urlencode({
+            'time': f'{sampleTime}',
+            'query': (
+                'sum by (instance, job) ('
+                f'  rate(router_output_pkts_total{{job="BR", type="{case}"}}[10s])'
+                ')'
+            )
+        })
+        conn = HTTPConnection(self.prom_address)
+        conn.request("GET", f"/api/v1/query?{prom_query}")
+        resp = conn.getresponse()
+        if resp.status != 200:
+            raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}")
+
+        # There's only one router, so whichever metric we get is the right one.
+        pld = json.loads(resp.read().decode("utf-8"))
+        raw = 0
+        results = pld["data"]["result"]
+        for result in results:
+            ts, val = result["value"]
+            raw = int(float(val))
+            break
+
         # Collect dropped packets metrics, so we can verify that the router was well saturated.
         # If not, the metrics aren't very useful.
         prom_query = urlencode({
@@ -248,7 +278,7 @@ class RouterBM():
             dropped = int(float(val))
             break
 
-        return processed, dropped
+        return processed, dropped, raw
 
     # Fetch and log the number of cores used by Go. This may inform performance
     # modeling later.
@@ -289,18 +319,18 @@ class RouterBM():
         # Run one test (30% size) as warm-up to trigger any frequency scaling, else the first test
         # can get much lower performance.
         logger.debug("Warmup")
-        self.exec_br_load(test_cases[0], map_args, 3000000)
+        self.exec_br_load(test_cases[0], map_args, 5)
 
         # Fetch the core count once. It doesn't change while the router is running.
         # We can't get it until the router has done some work, but the warmup is enough.
         cores = self.core_count()
 
         # At long last, run the tests.
-        results = Results(cores, self.coremark, self.mmbm)
+        results = Results(cores, self.coremark, self.mmbm, self.packet_size)
         for test_case in test_cases:
             logger.info(f"Case: {test_case}")
-            rate, droppage = self.run_test_case(test_case, map_args)
-            results.add_case(test_case, rate, droppage)
+            rate, droppage, raw = self.run_test_case(test_case, map_args)
+            results.add_case(test_case, rate or 1, droppage, raw)
 
         return results
         logger.info("Benchmarked")
diff --git a/acceptance/router_benchmark/brload/main.go b/acceptance/router_benchmark/brload/main.go
index 24662a5b4..ce23989d0 100644
--- a/acceptance/router_benchmark/brload/main.go
+++ b/acceptance/router_benchmark/brload/main.go
@@ -15,6 +15,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -37,7 +38,7 @@ import (
 	"github.com/scionproto/scion/private/keyconf"
 )
 
-type Case func(payload string, mac hash.Hash) (string, string, []byte)
+type Case func(packetSize int, mac hash.Hash) (string, string, []byte, []byte)
 
 type caseChoice string
 
@@ -70,12 +71,13 @@ var (
 		"out_transit": cases.OutTransit,
 		"br_transit":  cases.BrTransit,
 	}
-	logConsole string
-	dir        string
-	numPackets int
-	numStreams uint16
-	caseToRun  caseChoice
-	interfaces []string
+	logConsole   string
+	dir          string
+	testDuration time.Duration
+	packetSize   int
+	numStreams   uint16
+	caseToRun    caseChoice
+	interfaces   []string
 )
 
 func main() {
@@ -97,7 +99,8 @@ func main() {
 			os.Exit(run(cmd))
 		},
 	}
-	runCmd.Flags().IntVar(&numPackets, "num-packets", 10, "Number of packets to send")
+	runCmd.Flags().DurationVar(&testDuration, "duration", time.Second*15, "Test duration")
+	runCmd.Flags().IntVar(&packetSize, "packet-size", 172, "Total size of each packet sent")
 	runCmd.Flags().Uint16Var(&numStreams, "num-streams", 4,
 		"Number of independent streams (flowID) to use")
 	runCmd.Flags().StringVar(&logConsole, "log.console", "error",
@@ -160,10 +163,8 @@ func run(cmd *cobra.Command) int {
 	registerScionPorts()
 
 	log.Info("BRLoad acceptance tests:")
-
-	payloadString := "actualpayloadbytes"
 	caseFunc := allCases[string(caseToRun)] // key already checked.
-	caseDevIn, caseDevOut, rawPkt := caseFunc(payloadString, hfMAC)
+	caseDevIn, caseDevOut, payload, rawPkt := caseFunc(packetSize, hfMAC)
 
 	writePktTo, ok := handles[caseDevIn]
 	if !ok {
@@ -186,30 +187,40 @@ func run(cmd *cobra.Command) int {
 	go func() {
 		defer log.HandlePanic()
 		defer close(listenerChan)
-		listenerChan <- receivePackets(packetChan, payloadString)
+		listenerChan <- receivePackets(packetChan, payload)
 	}()
 
-	// We started everything that could be started. So the best window for perf mertics
-	// opens somewhere around now.
-	metricsBegin := time.Now().Unix()
 	// Because we're using IPV4 only, the UDP checksum is optional, so we are allowed to
 	// just set it to zero instead of recomputing it. The IP checksum does not cover the payload, so
 	// we don't need to update it.
 	binary.BigEndian.PutUint16(rawPkt[40:42], 0)
 
-	for i := 0; i < numPackets; i++ {
-		// Rotate through flowIDs. We patch it directly into the SCION header of the packet.  The
-		// SCION header starts at offset 42. The flowID is the 20 least significant bits of the
-		// first 32 bit field. To make our life simpler, we only use the last 16 bits (so no more
-		// than 64K flows).
-		binary.BigEndian.PutUint16(rawPkt[44:46], uint16(i%int(numStreams)))
-		if err := writePktTo.WritePacketData(rawPkt); err != nil {
-			log.Error("writing input packet", "case", string(caseToRun), "error", err)
-			return 1
+	// We started everything that could be started. So the best window for perf mertics
+	// opens somewhere around now.
+	begin := time.Now()
+	metricsBegin := begin.Unix()
+
+	numPkt := 0
+	for time.Since(begin) < testDuration {
+		// Check the time only once every 10000 packets
+		for i := 0; i < 10000; i++ {
+			// Rotate through flowIDs. We patch it directly into the SCION header of the packet. The
+			// SCION header starts at offset 42. The flowID is the 20 least significant bits of the
+			// first 32 bit field. To make our life simpler, we only use the last 16 bits (so no
+			// more than 64K flows).
+			binary.BigEndian.PutUint16(rawPkt[44:46], uint16(numPkt%int(numStreams)))
+			numPkt++
+			if err := writePktTo.WritePacketData(rawPkt); err != nil {
+				log.Error("writing input packet", "case", string(caseToRun), "error", err)
+				return 1
+			}
 		}
 	}
+
 	metricsEnd := time.Now().Unix()
-	// The test harness looks for this output.
+
+	// The test harness looks for this output. [metricsBegin, metricsEnd] needs to be fully
+	// contained in the period when we were actually transmitting, but can be a bit smaller.
 	fmt.Printf("metricsBegin: %d metricsEnd: %d\n", metricsBegin, metricsEnd)
 
 	// Get the results from the packet listener.
@@ -240,7 +251,7 @@ func run(cmd *cobra.Command) int {
 // The number of consumed packets is returned.
 // Currently we are content with receiving a single correct packet and we terminate after
 // that.
-func receivePackets(packetChan chan gopacket.Packet, payload string) int {
+func receivePackets(packetChan chan gopacket.Packet, payload []byte) int {
 	numRcv := 0
 
 	for {
@@ -251,15 +262,18 @@ func receivePackets(packetChan chan gopacket.Packet, payload string) int {
 			return numRcv
 		}
 		if err := got.ErrorLayer(); err != nil {
-			log.Error("error decoding packet", "err", err)
+			// This isn't an error. There is all sort of traffic that we might not know about
+			// and not be able to read.
+			// log.Error("error decoding packet", "err", err)
 			continue
 		}
 		layer := got.Layer(gopacket.LayerTypePayload)
 		if layer == nil {
-			log.Error("error fetching packet payload: no PayLoad")
+			// Don't treat this as an error. This could be random traffic we don't know about.
 			continue
 		}
-		if string(layer.LayerContents()) == payload {
+		if bytes.Equal(layer.LayerContents(), payload) {
+			// That's ours.
 			// To return the count of all packets received, just remove the "return" below.
 			// Return will occur once packetChan closes (which happens after a short timeout at
 			// the end of the test).
diff --git a/acceptance/router_benchmark/cases/BUILD.bazel b/acceptance/router_benchmark/cases/BUILD.bazel
index d527e51a5..0fdf993ff 100644
--- a/acceptance/router_benchmark/cases/BUILD.bazel
+++ b/acceptance/router_benchmark/cases/BUILD.bazel
@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "br_transit.go",
+        "helpers.go",
         "in.go",
         "in_transit.go",
         "out.go",
diff --git a/acceptance/router_benchmark/cases/br_transit.go b/acceptance/router_benchmark/cases/br_transit.go
index 947a57a43..13022f98b 100644
--- a/acceptance/router_benchmark/cases/br_transit.go
+++ b/acceptance/router_benchmark/cases/br_transit.go
@@ -18,8 +18,6 @@ import (
 	"hash"
 	"time"
 
-	"github.com/google/gopacket"
-
 	"github.com/scionproto/scion/pkg/private/util"
 	"github.com/scionproto/scion/pkg/slayers"
 	"github.com/scionproto/scion/pkg/slayers/path"
@@ -34,7 +32,7 @@ import (
 
 // BrTransit generates one packet of "br_transit" traffic over the router under test.
 // The outcome is a raw packet which the test must feed to the router. The flowID field is 0.
-func BrTransit(payload string, mac hash.Hash) (string, string, []byte) {
+func BrTransit(packetSize int, mac hash.Hash) (string, string, []byte, []byte) {
 
 	var (
 		originIA       = ISDAS(2)
@@ -47,11 +45,6 @@ func BrTransit(payload string, mac hash.Hash) (string, string, []byte) {
 		targetHost     = HostAddr(targetIP)
 	)
 
-	options := gopacket.SerializeOptions{
-		FixLengths:       true,
-		ComputeChecksums: true,
-	}
-
 	ethernet, ip, udp := Underlay(srcIP, srcPort, dstIP, dstPort)
 
 	// Fully correct (hopefully) path.
@@ -119,15 +112,6 @@ func BrTransit(payload string, mac hash.Hash) (string, string, []byte) {
 	scionudp.DstPort = 50000
 	scionudp.SetNetworkLayerForChecksum(scionL)
 
-	payloadBytes := []byte(payload)
-
-	// Prepare input packet
-	input := gopacket.NewSerializeBuffer()
-	if err := gopacket.SerializeLayers(input, options,
-		ethernet, ip, udp, scionL, scionudp, gopacket.Payload(payloadBytes),
-	); err != nil {
-		panic(err)
-	}
-
-	return DeviceName(1, 2), DeviceName(1, 3), input.Bytes()
+	payload, packet := mkPacket(packetSize, ethernet, ip, udp, scionL, scionudp)
+	return DeviceName(1, 2), DeviceName(1, 3), payload, packet
 }
diff --git a/acceptance/router_benchmark/cases/helpers.go b/acceptance/router_benchmark/cases/helpers.go
new file mode 100644
index 000000000..4abd096cd
--- /dev/null
+++ b/acceptance/router_benchmark/cases/helpers.go
@@ -0,0 +1,64 @@
+// Copyright 2023 SCION Association
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cases
+
+import (
+	"github.com/google/gopacket"
+)
+
+// We make the assumption os protocol sanity: the length and checksum fields are of fixed size
+// and so do not affect the packet size.
+var hdrOptions = gopacket.SerializeOptions{
+	FixLengths:       false,
+	ComputeChecksums: false,
+}
+
+func headerLength(layers ...gopacket.SerializableLayer) int {
+	sb := gopacket.NewSerializeBuffer()
+	if err := gopacket.SerializeLayers(sb, hdrOptions, layers...); err != nil {
+		panic(err)
+	}
+	return len(sb.Bytes())
+}
+
+func mkPayload(payloadLen int) []byte {
+	if payloadLen < 0 {
+		payloadLen = 0
+	}
+	payload := make([]byte, payloadLen)
+	copy(payload[:], []byte("actualpayloadbytes"))
+	return gopacket.Payload(payload)
+}
+
+var pktOptions = gopacket.SerializeOptions{
+	FixLengths:       true,
+	ComputeChecksums: true,
+}
+
+func mkPacket(
+	packetSize int,
+	layers ...gopacket.SerializableLayer,
+) ([]byte, []byte) {
+
+	// We want to make a packet of a specific length, header included.
+	hdrLen := headerLength(layers...)
+	sb := gopacket.NewSerializeBuffer()
+	payload := mkPayload(packetSize - hdrLen)
+	err := gopacket.SerializeLayers(sb, pktOptions, append(layers, gopacket.Payload(payload))...)
+	if err != nil {
+		panic(err)
+	}
+	return payload, sb.Bytes()
+}
diff --git a/acceptance/router_benchmark/cases/in.go b/acceptance/router_benchmark/cases/in.go
index 1e4c139a5..f8d41a050 100644
--- a/acceptance/router_benchmark/cases/in.go
+++ b/acceptance/router_benchmark/cases/in.go
@@ -18,8 +18,6 @@ import (
 	"hash"
 	"time"
 
-	"github.com/google/gopacket"
-
 	"github.com/scionproto/scion/pkg/private/util"
 	"github.com/scionproto/scion/pkg/slayers"
 	"github.com/scionproto/scion/pkg/slayers/path"
@@ -34,7 +32,7 @@ import (
 
 // oneIn generates one packet of incoming traffic into AS1 at br1a. The outcome is a raw packet
 // that the test must feed into the router. The flow ID is 0.
-func In(payload string, mac hash.Hash) (string, string, []byte) {
+func In(packetSize int, mac hash.Hash) (string, string, []byte, []byte) {
 
 	var (
 		originIA       = ISDAS(2)
@@ -47,11 +45,6 @@ func In(payload string, mac hash.Hash) (string, string, []byte) {
 		targetHost     = HostAddr(targetIP)
 	)
 
-	options := gopacket.SerializeOptions{
-		FixLengths:       true,
-		ComputeChecksums: true,
-	}
-
 	ethernet, ip, udp := Underlay(srcIP, srcPort, dstIP, dstPort)
 
 	// Fully correct (hopefully) path.
@@ -106,14 +99,6 @@ func In(payload string, mac hash.Hash) (string, string, []byte) {
 	scionudp.DstPort = 50000
 	scionudp.SetNetworkLayerForChecksum(scionL)
 
-	payloadBytes := []byte(payload)
-
-	// Prepare input packet
-	input := gopacket.NewSerializeBuffer()
-	if err := gopacket.SerializeLayers(input, options,
-		ethernet, ip, udp, scionL, scionudp, gopacket.Payload(payloadBytes),
-	); err != nil {
-		panic(err)
-	}
-	return DeviceName(1, 2), DeviceName(1, 0), input.Bytes()
+	payload, packet := mkPacket(packetSize, ethernet, ip, udp, scionL, scionudp)
+	return DeviceName(1, 2), DeviceName(1, 0), payload, packet
 }
diff --git a/acceptance/router_benchmark/cases/in_transit.go b/acceptance/router_benchmark/cases/in_transit.go
index 7a51c32ea..3258fec2d 100644
--- a/acceptance/router_benchmark/cases/in_transit.go
+++ b/acceptance/router_benchmark/cases/in_transit.go
@@ -18,8 +18,6 @@ import (
 	"hash"
 	"time"
 
-	"github.com/google/gopacket"
-
 	"github.com/scionproto/scion/pkg/private/util"
 	"github.com/scionproto/scion/pkg/slayers"
 	"github.com/scionproto/scion/pkg/slayers/path"
@@ -34,7 +32,7 @@ import (
 
 // oneInTransit generates one packet of "in_transit" traffic over the router under test.
 // The outcome is a raw packet that the test must feed into the router. The flow ID is 0.
-func InTransit(payload string, mac hash.Hash) (string, string, []byte) {
+func InTransit(packetSize int, mac hash.Hash) (string, string, []byte, []byte) {
 
 	var (
 		originIA       = ISDAS(2)
@@ -47,11 +45,6 @@ func InTransit(payload string, mac hash.Hash) (string, string, []byte) {
 		targetHost     = HostAddr(targetIP)
 	)
 
-	options := gopacket.SerializeOptions{
-		FixLengths:       true,
-		ComputeChecksums: true,
-	}
-
 	ethernet, ip, udp := Underlay(srcIP, srcPort, dstIP, dstPort)
 
 	// Fully correct (hopefully) path.
@@ -119,14 +112,6 @@ func InTransit(payload string, mac hash.Hash) (string, string, []byte) {
 	scionudp.DstPort = 50000
 	scionudp.SetNetworkLayerForChecksum(scionL)
 
-	payloadBytes := []byte(payload)
-
-	// Prepare input packet
-	input := gopacket.NewSerializeBuffer()
-	if err := gopacket.SerializeLayers(input, options,
-		ethernet, ip, udp, scionL, scionudp, gopacket.Payload(payloadBytes),
-	); err != nil {
-		panic(err)
-	}
-	return DeviceName(1, 2), DeviceName(1, 0), input.Bytes()
+	payload, packet := mkPacket(packetSize, ethernet, ip, udp, scionL, scionudp)
+	return DeviceName(1, 2), DeviceName(1, 0), payload, packet
 }
diff --git a/acceptance/router_benchmark/cases/out.go b/acceptance/router_benchmark/cases/out.go
index b844b5e36..16b186721 100644
--- a/acceptance/router_benchmark/cases/out.go
+++ b/acceptance/router_benchmark/cases/out.go
@@ -18,8 +18,6 @@ import (
 	"hash"
 	"time"
 
-	"github.com/google/gopacket"
-
 	"github.com/scionproto/scion/pkg/private/util"
 	"github.com/scionproto/scion/pkg/slayers"
 	"github.com/scionproto/scion/pkg/slayers/path"
@@ -34,7 +32,7 @@ import (
 
 // oneOut generates one packet of outgoing traffic from AS1 at br1a. The outcome is a raw packet
 // that the test must feed into the router. The flow ID is 0.
-func Out(payload string, mac hash.Hash) (string, string, []byte) {
+func Out(packetSize int, mac hash.Hash) (string, string, []byte, []byte) {
 
 	var (
 		originIA       = ISDAS(1)
@@ -47,11 +45,6 @@ func Out(payload string, mac hash.Hash) (string, string, []byte) {
 		targetHost     = HostAddr(targetIP)
 	)
 
-	options := gopacket.SerializeOptions{
-		FixLengths:       true,
-		ComputeChecksums: true,
-	}
-
 	ethernet, ip, udp := Underlay(srcIP, srcPort, dstIP, dstPort)
 
 	// Fully correct (hopefully) path.
@@ -107,15 +100,6 @@ func Out(payload string, mac hash.Hash) (string, string, []byte) {
 	scionudp.DstPort = 50000
 	scionudp.SetNetworkLayerForChecksum(scionL)
 
-	payloadBytes := []byte(payload)
-
-	// Prepare input packet
-	input := gopacket.NewSerializeBuffer()
-	if err := gopacket.SerializeLayers(input, options,
-		ethernet, ip, udp, scionL, scionudp, gopacket.Payload(payloadBytes),
-	); err != nil {
-		panic(err)
-	}
-
-	return DeviceName(1, 0), DeviceName(1, 2), input.Bytes()
+	payload, packet := mkPacket(packetSize, ethernet, ip, udp, scionL, scionudp)
+	return DeviceName(1, 0), DeviceName(1, 2), payload, packet
 }
diff --git a/acceptance/router_benchmark/cases/out_transit.go b/acceptance/router_benchmark/cases/out_transit.go
index fc010ed15..d469c984b 100644
--- a/acceptance/router_benchmark/cases/out_transit.go
+++ b/acceptance/router_benchmark/cases/out_transit.go
@@ -18,8 +18,6 @@ import (
 	"hash"
 	"time"
 
-	"github.com/google/gopacket"
-
 	"github.com/scionproto/scion/pkg/private/util"
 	"github.com/scionproto/scion/pkg/slayers"
 	"github.com/scionproto/scion/pkg/slayers/path"
@@ -34,7 +32,7 @@ import (
 
 // oneOutTransit generates one packet of "out_transit" traffic over the router under test.
 // The outcome is a raw packet that the test must feed to the router. The flow ID is 0.
-func OutTransit(payload string, mac hash.Hash) (string, string, []byte) {
+func OutTransit(packetSize int, mac hash.Hash) (string, string, []byte, []byte) {
 
 	var (
 		originIA       = ISDAS(4)
@@ -47,11 +45,6 @@ func OutTransit(payload string, mac hash.Hash) (string, string, []byte) {
 		targetHost     = HostAddr(targetIP)
 	)
 
-	options := gopacket.SerializeOptions{
-		FixLengths:       true,
-		ComputeChecksums: true,
-	}
-
 	ethernet, ip, udp := Underlay(srcIP, srcPort, dstIP, dstPort)
 
 	// Fully correct (hopefully) path.
@@ -120,14 +113,6 @@ func OutTransit(payload string, mac hash.Hash) (string, string, []byte) {
 	scionudp.DstPort = 50000
 	scionudp.SetNetworkLayerForChecksum(scionL)
 
-	payloadBytes := []byte(payload)
-
-	// Prepare input packet
-	input := gopacket.NewSerializeBuffer()
-	if err := gopacket.SerializeLayers(input, options,
-		ethernet, ip, udp, scionL, scionudp, gopacket.Payload(payloadBytes),
-	); err != nil {
-		panic(err)
-	}
-	return DeviceName(1, 0), DeviceName(1, 2), input.Bytes()
+	payload, packet := mkPacket(packetSize, ethernet, ip, udp, scionL, scionudp)
+	return DeviceName(1, 0), DeviceName(1, 2), payload, packet
 }
diff --git a/tools/env/debian/pkgs.txt b/tools/env/debian/pkgs.txt
index 987b493ea..d3b9c339d 100644
--- a/tools/env/debian/pkgs.txt
+++ b/tools/env/debian/pkgs.txt
@@ -19,3 +19,4 @@ sqlite3
 sudo
 tzdata
 zlib1g-dev
+zip
diff --git a/tools/mmbm/main.go b/tools/mmbm/main.go
index 43ced1f7b..9b0dee5b7 100644
--- a/tools/mmbm/main.go
+++ b/tools/mmbm/main.go
@@ -39,11 +39,14 @@ import (
 const allBufs = 8192
 const cycleStep = 5 // Must not divide any working set size, must not be 1.
 
-// Block is a packet buffer representation arranged to make it easy to copy 1, 172 or bufSize
+// Block is a packet buffer representation arranged to make it easy to copy 1, 172 or 4096
 // bytes. It is exported to make sure go cannot optimize fields out.
 type Block struct {
-	packet [172]uint8
-	tail   [4096 - 172]uint8
+	page struct {
+		packet [172]uint8
+		tail   [4096 - 172]uint8
+	}
+	missalign [64]uint8 //nolint:unused
 }
 
 var buf [allBufs]Block
@@ -64,10 +67,10 @@ func nextPair(max int) (int, int) {
 func writeBuf(numBufs int, cpSize int) {
 	for i := 0; i < numBufs; i++ {
 		for j := 0; j < 172; j++ {
-			buf[i].packet[j] = uint8(j % 256)
+			buf[i].page.packet[j] = uint8(j % 256)
 		}
 		for j := 172; j < cpSize; j++ {
-			buf[i].tail[j-172] = uint8(j % 256)
+			buf[i].page.tail[j-172] = uint8(j % 256)
 		}
 	}
 }
@@ -76,7 +79,7 @@ func writeBuf(numBufs int, cpSize int) {
 func benchmarkCopyByte(N int, numBufs int) {
 	for i := N; i > 0; i-- {
 		dst, src := nextPair(numBufs)
-		buf[dst].packet[0] = buf[src].packet[0]
+		buf[dst].page.packet[0] = buf[src].page.packet[0]
 	}
 }
 
@@ -84,15 +87,15 @@ func benchmarkCopyByte(N int, numBufs int) {
 func benchmarkCopyPacket(N int, numBufs int) {
 	for i := N; i > 0; i-- {
 		dst, src := nextPair(numBufs)
-		buf[dst].packet = buf[src].packet
+		buf[dst].page.packet = buf[src].page.packet
 	}
 }
 
-// benchmarkCopyBlock copies a short packet from a block to another
+// benchmarkCopyBlock copies a large packet from a block to another
 func benchmarkCopyBlock(N int, numBufs int) {
 	for i := N; i > 0; i-- {
 		dst, src := nextPair(numBufs)
-		buf[dst] = buf[src]
+		buf[dst].page = buf[src].page
 	}
 }
 
@@ -161,7 +164,7 @@ func main() {
 	largeCacheMiss, _ := tc("largeCacheMiss", 1024, 4096)
 
 	// Per packet, 4k copy, Cache misses, and TLB misses
-	largeAllMiss, _ := tc("largeAllMiss", 8192, 4096)
+	largeAllMiss, _ := tc("largeAllMiss", 3071, 4096)
 
 	// All the results here. Best avoid printfs before. It tickles the gc.
 


Test patch:
diff --git a/acceptance/router_benchmark/test.py b/acceptance/router_benchmark/test.py
index b7b51c715..6fe8e4aff 100644
--- a/acceptance/router_benchmark/test.py
+++ b/acceptance/router_benchmark/test.py
@@ -20,6 +20,7 @@ import json
 import logging
 import shutil
 import time
+import os
 
 from acceptance.common import base
 from benchmarklib import Intf, RouterBM
@@ -32,6 +33,12 @@ from random import randint
 
 logger = logging.getLogger(__name__)
 
+# Default packet length for CI testing
+BM_PACKET_SIZE = 172
+
+# Router profiling ON or OFF?
+PROFILING = False
+
 # Those values are valid expectations only when running in the CI environment.
 TEST_CASES = {
     "in": 720000,
@@ -188,6 +195,7 @@ class RouterBMTest(base.TestBase, RouterBM):
     # Used by the RouterBM mixin:
     coremark: int = 0
     mmbm: int = 0
+    packet_size: int = BM_PACKET_SIZE
     intf_map: dict[str, Intf] = {}
     brload: LocalCommand = None
     brload_cpus: list[int] = [0]
@@ -316,7 +324,7 @@ class RouterBMTest(base.TestBase, RouterBM):
             peer_mac = mac_for_ip(req.peer_ip)
             mac = mac_for_ip(req.ip)
             sudo("ip", "link", "add", host_intf, "type", "veth", "peer", "name", br_intf)
-            sudo("ip", "link", "set", host_intf, "mtu", "8000")
+            sudo("ip", "link", "set", host_intf, "mtu", "9000")
             sudo("ip", "link", "set", host_intf, "arp", "off")  # Make sure the real addr isn't used
 
             # Do not assign the host addresses but create one link-local addr.
@@ -327,7 +335,7 @@ class RouterBMTest(base.TestBase, RouterBM):
 
             sudo("sysctl", "-qw", f"net.ipv6.conf.{host_intf}.disable_ipv6=1")
             sudo("ethtool", "-K", br_intf, "rx", "off", "tx", "off")
-            sudo("ip", "link", "set", br_intf, "mtu", "8000")
+            sudo("ip", "link", "set", br_intf, "mtu", "9000")
             sudo("ip", "link", "set", br_intf, "address", mac)
 
             # The network namespace
@@ -353,6 +361,10 @@ class RouterBMTest(base.TestBase, RouterBM):
         # Ship it.
         self.intf_map[req.label] = Intf(host_intf, mac, peer_mac)
 
+        # If that's an exclusive interface pair, we can use it to profile the router:
+        if req.exclusive == "true":
+            self.profiling_addr = req.ip
+
     def fetch_horsepower(self):
         try:
             coremark_exe = self.get_executable("coremark")
@@ -427,9 +439,6 @@ class RouterBMTest(base.TestBase, RouterBM):
             t = IntfReq._make(elems)
             self.create_interface(t, "benchmark")
 
-        # We don't need that symlink any more
-        sudo("rm", "/var/run/netns/benchmark")
-
         # Now the router can start.
         docker("run",
                "-v", f"{self.artifacts}/conf:/etc/scion",
@@ -446,6 +455,21 @@ class RouterBMTest(base.TestBase, RouterBM):
         # They'll be used to produce a performance index.
         self.fetch_horsepower()
 
+        # Optionally profile the router
+        if PROFILING:
+            docker("run",
+                   "--rm",
+                   "-d",
+                   "-u", os.getuid(),
+                   "-v", f"{self.artifacts}:/out",
+                   "--network", "container:prometheus",
+                   "curlimages/curl",
+                   f"{self.profiling_addr}:30442/debug/pprof/profile?seconds=70",
+                   "-o", "/out/cpu.pprof")
+
+        # We don't need that symlink any more
+        sudo("rm", "/var/run/netns/benchmark")
+
     def teardown(self):
         docker["logs", "router"].run_fg(retcode=None)
         docker("rm", "-f", "prometheus")