Repository: scionproto/scion
Pull request #4496 (2024-03-26 19:03:52)

testing: extend router benchmark tooling
Enables the benchmarking of a SCION router running on an external device.
A packaged test config exists only for openwrt at the moment.



matzf: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/scionproto/scion/4496)
<!-- Reviewable:end -->


Patch:
diff --git a/BUILD.bazel b/BUILD.bazel
index 7cd9b18cc..c129c2677 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -205,6 +205,7 @@ pkg_tar(
     name = "scion-ci",
     srcs = [
         "//acceptance/cmd/sig_ping_acceptance",
+        "//acceptance/router_benchmark/brload",
         "//pkg/private/xtest/graphupdater",
         "//tools/braccept",
         "//tools/buildkite/cmd/buildkite_artifacts",
diff --git a/acceptance/router_benchmark/BUILD.bazel b/acceptance/router_benchmark/BUILD.bazel
index 4ba18b294..420219394 100644
--- a/acceptance/router_benchmark/BUILD.bazel
+++ b/acceptance/router_benchmark/BUILD.bazel
@@ -1,4 +1,6 @@
 load("//acceptance/common:raw.bzl", "raw_test")
+load("//tools/lint:py.bzl", "py_binary", "py_library")
+load("@com_github_scionproto_scion_python_deps//:requirements.bzl", "requirement")
 
 exports_files(
     [
@@ -15,6 +17,10 @@ exports_files(
 args = [
     "--executable",
     "brload:$(location //acceptance/router_benchmark/brload:brload)",
+    "--executable",
+    "coremark:$(location //tools/coremark:coremark)",
+    "--executable",
+    "mmbm:$(location //tools/mmbm:mmbm)",
     "--docker-image=$(location //docker:router.tarball)",
 ]
 
@@ -22,6 +28,8 @@ data = [
     ":conf",
     "//docker:router.tarball",
     "//acceptance/router_benchmark/brload:brload",
+    "//tools/coremark:coremark",
+    "//tools/mmbm:mmbm",
 ]
 
 raw_test(
@@ -32,4 +40,22 @@ raw_test(
     homedir = "$(rootpath //docker:router.tarball)",
     # This test uses sudo and accesses /var/run/netns.
     local = True,
+    deps = ["benchmarklib"],
+)
+
+py_library(
+    name = "benchmarklib",
+    srcs = ["benchmarklib.py"],
+)
+
+# To ensure that the linter runs over this. Cannot actually be run with
+# bazel run; it is meant to be executed from the command line.
+py_binary(
+    name = "benchmark",
+    srcs = ["benchmark.py"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "benchmarklib",
+        requirement("plumbum"),
+    ],
 )
diff --git a/acceptance/router_benchmark/benchmark.py b/acceptance/router_benchmark/benchmark.py
new file mode 100755
index 000000000..86518b322
--- /dev/null
+++ b/acceptance/router_benchmark/benchmark.py
@@ -0,0 +1,360 @@
+#!/usr/bin/env python3
+
+# Copyright 2023 SCION Association
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import ipaddress
+import json
+import logging
+import os
+import ssl
+import time
+import traceback
+
+from benchmarklib import Intf, RouterBM
+from collections import namedtuple
+from plumbum import cli
+from plumbum import cmd
+from plumbum import local
+from plumbum.cmd import docker
+from plumbum.machines import LocalCommand
+from random import randint
+from urllib.request import urlopen
+
+logger = logging.getLogger(__name__)
+
+TEST_CASES = [
+    "in",
+    "out",
+    "in_transit",
+    "out_transit",
+    "br_transit",
+]
+
+# Convenience types to carry interface params.
+IntfReq = namedtuple("IntfReq", "label, prefix_len, ip, peer_ip, exclusive")
+
+
+def sudo(*args: [str]) -> str:
+    # -A, --askpass makes sure command is failing and does not wait for
+    # interactive password input.
+    return cmd.sudo("-A", *args)
+
+
+class RouterBMTool(cli.Application, RouterBM):
+    """Evaluates the performance of an external router running the SCION reference implementation.
+
+    The performance is reported in terms of packets per available machine*second (i.e. one
+    accumulated second of all configured CPUs available to execute the router code).
+
+    The router can actually be anything that has compatible metrics scrapable by
+    prometheus. So that's presumably the reference implementation or a fork thereof.
+
+    This test runs against a single router. The outgoing packets are not observed, the incoming
+    packets are fed directly by the test driver (brload). The other routers in the topology are a
+    fiction, they exist only in the routers configuration.
+
+    The topology (./conf/topology.json) is the following:
+
+    AS2 (br2) ---+== (br1a) AS1 (br1b) ---- (br4) AS4
+                 |
+    AS3 (br3) ---+
+
+    Only br1a is executed and observed.
+
+    Pretend traffic is injected by brload's. See the test cases for details.
+
+    """
+
+    avail_interfaces: list[str] = []
+    mx_interface: str = None
+    to_flush: list[str] = []
+    scrape_addr: str = None
+
+    log_level = cli.SwitchAttr(["l", "loglevel"], str, default='warning', help="Logging level")
+
+    doit = cli.Flag(["r", "run"],
+                    help="Run the benchmark, as opposed to seeing the instructions.")
+    json = cli.Flag(["j", "json"],
+                    help="Output the report in json format.")
+
+    # Used by the RouterBM mixin:
+    coremark = cli.SwitchAttr(["c", "coremark"], int, default=0,
+                              help="The coremark score of the subject machine.")
+    mmbm = cli.SwitchAttr(["m", "mmbm"], int, default=0,
+                          help="The mmbm score of the subject machine.")
+    intf_map: dict[str, Intf] = {}
+    brload: LocalCommand = local["./bin/brload"]
+    brload_cpus: list[int] = []
+    artifacts = f"{os.getcwd()}/acceptance/router_benchmark"
+    prom_address: str = "localhost:9090"
+
+    def host_interface(self, excl: bool):
+        """Returns the next host interface that we should use for a brload links.
+
+        If excl is true, we pick one and never pick that one again.
+        Else, we pick one the first time it's needed and keep it for reuse.
+        """
+        if excl:
+            return self.avail_interfaces.pop()
+
+        if self.mx_interface is None:
+            self.mx_interface = self.avail_interfaces.pop()
+
+        return self.mx_interface
+
+    def config_interface(self, req: IntfReq):
+        """Configure an interfaces according to brload's requirements.
+
+        The device must not be in use for anything else. The devices are picked from a list
+        supplied by the user.
+
+        We probably do not:
+          sudo("ip", "addr", "add", f"{req.peer_ip}/{req.prefix_len}", "dev", host_intf)
+
+        It causes trouble: if an IP is assigned, the kernel responds with "unbound port" icmp
+        messages to the router traffic, which breaks the bound UDP connections that the router uses
+        for external interfaces.
+
+        Args:
+          req: A requested router-side network interface. It comprises:
+            * A label by which brload identifies that interface.
+            * The IP address to be assigned to that interface.
+            * The IP address of one neighbor.
+
+        """
+        exclusive = req.exclusive == "true"
+        host_intf = self.host_interface(exclusive)
+
+        # We need a means of connecting to the router's internal interface (from prometheus and
+        # to scrape the horsepower microbenchmark results. We pick one address of
+        # the router's subnet that's not otherwise used. This must NOT be "PeerIP".
+        # brload requires the internal interface to be "exclusive", that's our clue.
+        if exclusive:
+            net = ipaddress.ip_network(f"{req.ip}/{req.prefix_len}", strict=False)
+            hostAddr = next(net.hosts()) + 126
+            self.scrape_addr = req.ip
+            sudo("ip", "addr", "add", f"{hostAddr}/{req.prefix_len}",
+                 "broadcast", str(net.broadcast_address), "dev", host_intf)
+            self.to_flush.append(host_intf)
+
+        logger.debug(f"=> Configuring interface {host_intf} for: {req}...")
+
+        # We do multiplex most requested router interfaces onto one physical interface, so, we
+        # must check that we haven't already configured the physical one.
+        for i in self.intf_map.values():
+            if i.name == host_intf:
+                break
+        else:
+            # TODO: instructions/warning regarding inability to enable jumbo frames.
+            # sudo("ip", "link", "set", host_intf, "mtu", "8000")
+
+            # Do not assign the host addresses but create one link-local addr.
+            # Brload needs some src IP to send arp requests. (This requires rp_filter
+            # to be off on the router side, else, brload's arp requests are discarded).
+            sudo("ip", "addr", "add", f"169.254.{randint(0, 255)}.{randint(0, 255)}/16",
+                 "broadcast", "169.254.255.255",
+                 "dev", host_intf, "scope", "link")
+            sudo("sysctl", "-qw", f"net.ipv6.conf.{host_intf}.disable_ipv6=1")
+            self.to_flush.append(host_intf)
+
+        # Fit for duty.
+        sudo("ip", "link", "set", host_intf, "up")
+
+        # Ship it. Leave mac addresses alone. In this standalone test we use the real one.
+        self.intf_map[req.label] = Intf(host_intf, None, None)
+
+    def fetch_horsepower(self) -> tuple[int]:
+        try:
+            url = f"https://{self.scrape_addr}/horsepower.json"
+            resp = urlopen(url, context=ssl._create_unverified_context())
+            hp = json.loads(resp.read().decode("ascii"))
+        except Exception as e:
+            logger.warning(f"Fetching coremark and mmbm from {url}... {e}")
+            return
+
+        if self.coremark == 0:
+            self.coremark = round(hp["coremark"])
+
+        if self.mmbm == 0:
+            self.mmbm = round(hp["mmbm"])
+
+    def setup(self, avail_interfaces: list[str]):
+        logger.info("Preparing...")
+
+        # Check that the given interfaces are safe to use. We will wreck their config.
+        for intf in avail_interfaces:
+            output = sudo("ip", "addr", "show", "dev", intf)
+            if len(output.splitlines()) > 2:
+                logger.error(f"""\
+                Interface {intf} appears to be in some kind of use. Cowardly refusing to modify it.
+                If you have a network manager, tell it to disable or ignore that interface.
+                Else, how about \"sudo ip addr flush dev {intf}\"?
+                """)
+                raise RuntimeError("Interface in use")
+
+        # Looks safe.
+        self.avail_interfaces = avail_interfaces
+
+        # Run test brload test with --show-interfaces and set up the interfaces as it says.
+        # We supply the label->host-side-name mapping to brload when we start it.
+        logger.debug("==> Configuring host interfaces...")
+
+        output = self.brload("show-interfaces")
+
+        lines = sorted(output.splitlines())
+        for line in lines:
+            elems = line.split(",")
+            if len(elems) != 5:
+                continue
+            logger.debug(f"Requested by brload: {line}")
+            t = IntfReq._make(elems)
+            self.config_interface(t)
+
+        # Start an instance of prometheus configured to scrape the router.
+        logger.debug("==> Starting prometheus...")
+        docker("run",
+               "-v", f"{self.artifacts}/conf:/etc/scion",
+               "-d",
+               "--network", "host",
+               "--name", "prometheus_bm",
+               "prom/prometheus:v2.47.2",
+               "--config.file", "/etc/scion/prometheus.yml")
+
+        time.sleep(2)
+
+        # Collect the horsepower microbenchmark numbers if we can.
+        # They'll be used to produce a performance index.
+        self.fetch_horsepower()
+
+        logger.info("Prepared")
+
+    def cleanup(self, retcode: int):
+        docker("rm", "-f", "prometheus_bm")
+        for intf in self.to_flush:
+            sudo("ip", "addr", "flush", "dev", intf)
+        return retcode
+
+    def instructions(self):
+        output = self.brload("show-interfaces")
+
+        exclusives = []
+        multiplexed = []
+        reqs = []
+        intf_index = 0
+
+        # We sort the requests from brload because the interface that is picked for each can depends
+        # on the order in which we process them and we need to be consistent from run to run so
+        # the instructions we give the user actually work.
+        # (assuming brload's code doesn't change in-between).
+
+        lines = sorted(output.splitlines())
+        for line in lines:
+            elems = line.split(",")
+            if len(elems) != 5:
+                continue
+            req = IntfReq._make(elems)
+            reqs.append(req)
+            self.avail_interfaces.append(str(intf_index))  # Use numbers as placeholders
+            intf_index += 1
+
+        # TODO: Because of multiplexing, there are fewer real interfaces than labels requested
+        # by brload. So, not all placeholders get used (fine) and it happens that the low indices
+        # are the ones not used (confusing for the user). Currently we end-up with 1 and 2
+        # (and no 0), which is acceptable but fortuitous.
+        for req in reqs:
+            e = req.exclusive == "true"
+            a = f"{req.ip}/{req.prefix_len}"
+            i = self.host_interface(e)
+            if e:
+                exclusives.append(f"{a} (must reach: #{i})")
+            else:
+                multiplexed.append(f"{a} (must reach: #{i})")
+        nl = "\n"
+        print(f"""
+INSTRUCTIONS:
+
+1 - Configure your subject router according to accept/router_benchmark/conf/router.toml")
+    If using openwrt, an easy way to do that is to install the bmtools.ipk package. In addition,
+    bmtools includes two microbenchmarks: scion-coremark and scion-mmbm. Those will run
+    automatically and the results will be used to improve the benchmark report.
+
+    Optinal: If you did not install bmtools.ipk, install and run those microbenchmark and make a
+    note of the results: (scion-coremark; scion-mmbm).
+
+2 - Configure the following interfaces on your router (The procedure depends on your router
+    UI):
+    - One physical interface with addresses: {", ".join(multiplexed)}
+{nl.join(['    - One physical interface with address: ' + s for s in exclusives])}
+
+    IMPORTANT: if you're using a partitioned network (eg. multiple switches or no switches),
+    the "must reach" annotation matters. The 'h' number is the order in which the corresponding host
+    interface must be given on the command line in step 7.
+
+3 - Connect the corresponding ports into your test switch (best if dedicated for the test).
+
+4 - Restart the scion-router service.
+
+5 - Pick the same number of physical interfaces on the system where you are running this
+    script. Make sure that these interface aren't used for anything else and have no assigned
+    IP addresses. Make a note of their names and, if using a partitioned network, associate each
+    with one of the numbers from step 2.
+
+6 - Connect the corresponding ports into your test switch. If using a partitioned network, make
+    sure that port is reachable by the corresponding subject router port.
+
+7 - Execute this script with arguments: --run <interfaces>, where <interfaces> is the list
+    of names you collected in step 5. If using a partitioned network, make sure to supply them
+    in the order indicated in step 2.
+
+    If coremak and mmbm values are available, the report will include a performance index.
+
+    If coremark and mmbm are not available from the test subject, you may supply them on the command
+    line. To that end, add the following arguments: "--coremark=<coremark>", "--mmbm=<mmbm>", where
+    <coremark> and <mmbm> are the results you optionally collected in step 1.
+
+8 - Be patient...
+
+9 - Read the report.
+""")
+
+    def main(self, *interfaces: str):
+        status = 1
+        try:
+            logging.basicConfig(level=self.log_level.upper())
+            if not self.doit:
+                self.instructions()
+                status = 0
+            else:
+                self.setup(list(interfaces))
+                results = self.run_bm(TEST_CASES)
+                # No CI_check. We have no particular expectations here.
+                # Output the performance in human-friendly form by default...
+                if self.json:
+                    print(results.as_json())
+                else:
+                    print(results.as_report())
+                status = 0
+        except KeyboardInterrupt:
+            logger.info("Bailing out...")
+        except Exception:
+            logger.error(traceback.format_exc())
+        except SystemExit:
+            pass
+        return status
+
+
+if __name__ == "__main__":
+    RouterBMTool()
diff --git a/acceptance/router_benchmark/benchmarklib.py b/acceptance/router_benchmark/benchmarklib.py
new file mode 100644
index 000000000..a4666b986
--- /dev/null
+++ b/acceptance/router_benchmark/benchmarklib.py
@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+
+# Copyright 2024 SCION Association
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Benchmarking code that is common to the CI benchmark test and the stand-alone
+# benchmark program.
+
+import json
+import logging
+
+from collections import namedtuple
+from http.client import HTTPConnection
+from plumbum import cmd
+from urllib.parse import urlencode
+
+logger = logging.getLogger(__name__)
+
+# TODO(jchugly): this is still a work in progress. There are two unknowns in the model.
+# M_COEF: the proportion by which memory performance contributes to throughput compared to
+# arithmetic performance. NIC_CONSTANT: the fixed cost of the kernel interacting with the hardware
+# to retrieve a frame. That one is hardware dependent and must be found by a third benchmark, so
+# it is not theoretically a constant, but keeping it here to not forget. Until then, our performance
+# index isn't really valid cross-hardware. M_COEF=400 gives roughly consistent results with the
+# hardware we have. So, using that until we know more.
+
+M_COEF = 400
+NIC_CONSTANT = 0
+
+# TODO(jiceatscion): get it from or give it to brload?
+BM_PACKET_LEN = 172
+
+# Intf: description of an interface configured for brload's use. Depending on context
+# mac and peermac may be unused. "mac" is the MAC address configured on the side of the subject
+# router. "peer_mac" is the MAC address that brload should use when simulating the peer router.
+# If peer_mac is unset, we'll let brload use the true MAC address on its side, which is the default.
+Intf = namedtuple("Intf", "name, mac, peer_mac")
+
+
+class Results:
+    """Stores and format benchmark results.
+    """
+
+    cores: int = 0
+    coremark: int = 0
+    mmbm: int = 0
+    cases: list[dict] = []
+    failed: list[dict] = []
+    checked: bool = False
+
+    def __init__(self, cores: int, coremark: int, mmbm: int):
+        self.cores = cores
+        self.coremark = coremark
+        self.mmbm = mmbm
+
+    def perf_index(self, rate: int) -> float:
+        # TODO(jiceatscion): The perf index assumes that line speed isn't the bottleneck.
+        # It almost never is, but ideally we'd need to run iperf3 to verify.
+        # mmbm is in mebiBytes/s, rate is in pkt/s
+        return rate * (1.0 / self.coremark +
+                       M_COEF * BM_PACKET_LEN / (self.mmbm * 1024 * 1024) +
+                       NIC_CONSTANT)
+
+    def add_case(self, name: str, rate: int, droppage: int):
+        dropRatio = round(float(droppage) / (rate + droppage), 2)
+        saturated = dropRatio > 0.03
+        perf = 0.0
+        if self.cores == 3 and self.coremark and self.mmbm:
+            perf = round(self.perf_index(rate), 1)
+        self.cases.append({"case": name,
+                           "perf": perf, "rate": rate, "drop": dropRatio, "full": saturated})
+
+    def CI_check(self, expectations: dict[str, int]):
+        self.checked = True
+        for tc in self.cases:
+            want = expectations.get(tc["case"])
+            if want is not None:
+                slow = tc["rate"] < want
+                unsaturated = not tc["full"]
+                if slow or unsaturated:
+                    self.failed.append({"case": tc["case"],
+                                        "expected": want, "slow": slow, "unsaturated": unsaturated})
+
+    def as_json(self) -> str:
+        return json.dumps({
+            "cores": self.cores,
+            "coremark": self.coremark,
+            "mmbm": self.mmbm,
+            "cases": self.cases,
+            "checked": self.checked,
+            "failed": self.failed,
+        }, indent=4)
+
+    def as_report(self) -> str:
+        res = (f"Benchmark Results\n\ncores: {self.cores}\n"
+               f"coremark: {self.coremark or 'N/A'}\nmmbm: {self.mmbm or 'N/A'}\n")
+        for tc in self.cases:
+            res += (f"{tc['case']}: perf_index={tc['perf'] or 'N/A'}"
+                    f" rate={tc['rate']} droppage={tc['drop']:.1%} saturated={tc['full']}\n")
+        res += "CI pass/fail: "
+        if not self.checked:
+            res += "N/A\n"
+            return res
+        res += "FAILED\n" if self.failed else "PASS\n"
+        if not self.failed:
+            return res
+        for failure in self.failed:
+            res += (f"{failure['case']} expected={failure['expected']}"
+                    f" slow={failure['slow']} unsaturated={failure['unsaturated']}")
+        return res
+
+
+class RouterBM():
+    """Evaluates the performance of an external router running the SCION reference implementation.
+
+    The performance is reported in terms of packets per available machine*second (i.e. one
+    accumulated second of all configured CPUs available to execute the router code).
+
+    The router can actually be anything that has compatible metrics scrapable by
+    prometheus. So that's presumably the reference implementation or a fork thereof.
+
+    This test runs against a single router. The outgoing packets are not observed, the incoming
+    packets are fed directly by the test driver (brload). The other routers in the topology are a
+    fiction, they exist only in the routers configuration.
+
+    The topology (./conf/topology.json) is the following:
+
+    AS2 (br2) ---+== (br1a) AS1 (br1b) ---- (br4) AS4
+                 |
+    AS3 (br3) ---+
+
+    Only br1a is executed and observed.
+
+    Pretend traffic is injected by brload's. See the test cases for details.
+
+    This class is a Mixin that borrows the following attributes from the host class:
+    * coremark: the coremark benchmark results.
+    * mmbm: the mmbm benchmark results.
+    * intf_map: the map "label->actual_interface" map to be passed to brload.
+    * brload: "localCmd" wraper for the brload executable (plumbum.machines.LocalCommand)
+    * brload_cpus: [int] cpus where it is acceptable to run brload ([] means any)
+    * artifacts: the data directory (passed to docker).
+    * prom_address: the address of the prometheus API a string in the form "host:port"
+    """
+
+    def exec_br_load(self, case: str, map_args: list[str], count: int) -> str:
+        # For num-streams, attempt to distribute uniformly on many possible number of cores.
+        # 840 is a multiple of 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 15, 20, 21, 24, 28, ...
+        brload_args = [
+            self.brload.executable,
+            "run",
+            "--artifacts", self.artifacts,
+            *map_args,
+            "--case", case,
+            "--num-packets", str(count),
+            "--num-streams", "840",
+        ]
+        if self.brload_cpus:
+            brload_args = [
+                "taskset", "-c", ",".join(map(str, self.brload_cpus)),
+            ] + brload_args
+
+        return cmd.sudo("-A", *brload_args)
+
+    def run_test_case(self, case: str, map_args: list[str]) -> (int, int):
+        logger.debug(f"==> Starting load {case}")
+
+        output = self.exec_br_load(case, map_args, 10000000)
+        beg = "0"
+        end = "0"
+        for line in output.splitlines():
+            if line.startswith("metricsBegin"):
+                _, beg, _, end = line.split()
+
+        logger.debug(f"==> Collecting {case} performance metrics...")
+
+        # The raw metrics are expressed in terms of core*seconds. We convert to machine*seconds
+        # which allows us to provide a projected packet/s; ...more intuitive than packets/core*s.
+        # We measure the rate over 10s. For best results we sample the end of the middle 10s of the
+        # run. "beg" is the start time of the real action and "end" is the end time.
+        sampleTime = (int(beg) + int(end) + 10) / 2
+        prom_query = urlencode({
+            'time': f'{sampleTime}',
+            'query': (
+                'sum by (instance, job) ('
+                f'  rate(router_output_pkts_total{{job="BR", type="{case}"}}[10s])'
+                ')'
+                '/ on (instance, job) group_left()'
+                'sum by (instance, job) ('
+                '  1 - (rate(process_runnable_seconds_total[10s])'
+                '       / go_sched_maxprocs_threads)'
+                ')'
+            )
+        })
+        conn = HTTPConnection(self.prom_address)
+        conn.request("GET", f"/api/v1/query?{prom_query}")
+        resp = conn.getresponse()
+        if resp.status != 200:
+            raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}")
+
+        # There's only one router, so whichever metric we get is the right one.
+        pld = json.loads(resp.read().decode("utf-8"))
+        processed = 0
+        results = pld["data"]["result"]
+        for result in results:
+            ts, val = result["value"]
+            processed = int(float(val))
+            break
+
+        # Collect dropped packets metrics, so we can verify that the router was well saturated.
+        # If not, the metrics aren't very useful.
+        prom_query = urlencode({
+            'time': f'{sampleTime}',
+            'query': (
+                'sum by (instance, job) ('
+                '  rate(router_dropped_pkts_total{job="BR", reason=~"busy_.*"}[10s])'
+                ')'
+                '/ on (instance, job) group_left()'
+                'sum by (instance, job) ('
+                '  1 - (rate(process_runnable_seconds_total[10s])'
+                '       / go_sched_maxprocs_threads)'
+                ')'
+            )
+        })
+        conn = HTTPConnection(self.prom_address)
+        conn.request("GET", f"/api/v1/query?{prom_query}")
+        resp = conn.getresponse()
+        if resp.status != 200:
+            raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}")
+
+        # There's only one router, so whichever metric we get is the right one.
+        pld = json.loads(resp.read().decode("utf-8"))
+        dropped = 0
+        results = pld["data"]["result"]
+        for result in results:
+            ts, val = result["value"]
+            dropped = int(float(val))
+            break
+
+        return processed, dropped
+
+    # Fetch and log the number of cores used by Go. This may inform performance
+    # modeling later.
+    def core_count(self) -> int:
+        logger.debug("==> Collecting number of cores...")
+        prom_query = urlencode({
+            'query': 'go_sched_maxprocs_threads{job="BR"}'
+        })
+
+        conn = HTTPConnection(self.prom_address)
+        conn.request("GET", f"/api/v1/query?{prom_query}")
+        resp = conn.getresponse()
+        if resp.status != 200:
+            raise RuntimeError(f"FAILED: Unexpected response: {resp.status} {resp.reason}")
+
+        pld = json.loads(resp.read().decode("utf-8"))
+        results = pld["data"]["result"]
+        if not results:
+            raise RuntimeError("FAILED: Got no results when querying the core count")
+        if len(results) > 1:
+            raise RuntimeError(f"FAILED: Found more than one subject router in results: {results}")
+
+        result = results[0]
+        _, val = result["value"]
+        return int(val)
+
+    def run_bm(self, test_cases: [str]) -> Results:
+        logger.info("Benchmarking...")
+
+        # Build the interface mapping arg (here, we do not override the brload side mac address)
+        map_args = []
+        for label, intf in self.intf_map.items():
+            if intf.peer_mac is not None:
+                map_args.extend(["--interface", f"{label}={intf.name},{intf.peer_mac}"])
+            else:
+                map_args.extend(["--interface", f"{label}={intf.name}"])
+
+        # Run one test (30% size) as warm-up to trigger any frequency scaling, else the first test
+        # can get much lower performance.
+        logger.debug("Warmup")
+        self.exec_br_load(test_cases[0], map_args, 3000000)
+
+        # Fetch the core count once. It doesn't change while the router is running.
+        # We can't get it until the router has done some work, but the warmup is enough.
+        cores = self.core_count()
+
+        # At long last, run the tests.
+        results = Results(cores, self.coremark, self.mmbm)
+        for test_case in test_cases:
+            logger.info(f"Case: {test_case}")
+            rate, droppage = self.run_test_case(test_case, map_args)
+            results.add_case(test_case, rate, droppage)
+
+        return results
+        logger.info("Benchmarked")
diff --git a/dist/BUILD.bazel b/dist/BUILD.bazel
index 21550979b..5422f1fae 100644
--- a/dist/BUILD.bazel
+++ b/dist/BUILD.bazel
@@ -189,7 +189,7 @@ scion_pkg_ipk(
 )
 
 scion_pkg_ipk(
-    name = "bmconfig_ipk",
+    name = "bmtools_ipk",
 )
 
 # multiplatform_filegroup creates two targets: <name> and <name>_all.
@@ -215,8 +215,7 @@ multiplatform_filegroup(
 multiplatform_filegroup(
     name = "openwrt_testing",
     srcs = [
-        "bmconfig_ipk",
-        "coremark_ipk",
+        "bmtools_ipk",
         "testconfig_ipk",
     ],
     target_platforms = OPENWRT_PLATFORMS,
diff --git a/dist/openwrt/BUILD.bazel b/dist/openwrt/BUILD.bazel
index 466429d7f..3c580c169 100644
--- a/dist/openwrt/BUILD.bazel
+++ b/dist/openwrt/BUILD.bazel
@@ -24,6 +24,7 @@ exports_files(
         "initds/daemon",
         "initds/dispatcher",
         "initds/persistdbs",
+        "initds/bmtools",
 
         # Directory itself. Indicates the common root of config files.
         "test_configs",
@@ -34,6 +35,9 @@ exports_files(
         "test_configs/keys/master1.key",
         "test_configs/router.toml",
         "test_configs/control.toml",
+
+        # Special case initd file for benchmarking
+        "test_configs/router-initd",
     ],
     visibility = ["//visibility:public"],
 )
diff --git a/dist/openwrt/BUILD.external.bazel b/dist/openwrt/BUILD.external.bazel
index dc101ec5a..8cad8b807 100644
--- a/dist/openwrt/BUILD.external.bazel
+++ b/dist/openwrt/BUILD.external.bazel
@@ -99,8 +99,19 @@ ipk_pkg(
     deps = ["tools_ipk"],  # Force sequential build; openwrt make isn't reentrant.
 )
 
+# A note about the strange config file: scion-router-initd: This is a modified
+# version of the the scion-router initd file that will be copied over the normal
+# one by the scion-bmtools initd script. This is so we do not need to create a
+# separate init override mechanism just for the unique case.
 ipk_pkg(
-    name = "bmconfig_ipk",
+    name = "bmtools_ipk",
+    configs = ["@@//dist/openwrt:test_configs/router-initd"],
+    configsroot = "@@//dist/openwrt:test_configs",
+    executables = [
+        "@@//tools/mmbm:mmbm",
+        "@@//tools/coremark:coremark",
+    ],  # No _compressed; these are tiny, not Go.
+    initds = ["@@//dist/openwrt:initds/bmtools"],
     overrides = [
         "@@//acceptance/router_benchmark:conf/keys/master0.key",
         "@@//acceptance/router_benchmark:conf/keys/master1.key",
@@ -108,18 +119,10 @@ ipk_pkg(
         "@@//acceptance/router_benchmark:conf/topology.json",
     ],
     overridesroot = "@@//acceptance/router_benchmark:conf",
-    pkg = "bmconfig",
+    pkg = "bmtools",
     visibility = ["//visibility:public"],
     deps = ["testconfig_ipk"],  # Force sequential build; openwrt make isn't reentrant.
 )
 
-ipk_pkg(
-    name = "coremark_ipk",
-    executables = ["@@//tools/coremark:coremark"],  # No _compressed; it's small, not Go.
-    pkg = "coremark",
-    visibility = ["//visibility:public"],
-    deps = ["bmconfig_ipk"],  # Force sequential build; openwrt make isn't reentrant.
-)
-
 # Assemble a bazel toolchain out of the openwrt sdk binaries.
 musl_cc_toolchain(target_arch = "x86_64")
diff --git a/dist/openwrt/initds/bmtools b/dist/openwrt/initds/bmtools
new file mode 100644
index 000000000..d11f69f00
--- /dev/null
+++ b/dist/openwrt/initds/bmtools
@@ -0,0 +1,47 @@
+#!/bin/sh /etc/rc.common
+# "new(er)" style init script
+# Look at /lib/functions/service.sh on a running system for explanations of what other SERVICE_
+# options you can use, and when you might want them.
+
+# This is a run-once script. It executes a couple of trivial benchmarks and publishes the results
+# to an easy to scrape url.
+
+START=99
+STOP=0
+APP=scion-control
+
+start() {
+    u=$$
+    # coremark
+    /usr/bin/scion-coremark > /tmp/coremark.$u.out
+    if grep -q "Correct operation validated" /tmp/coremark.$u.out; then
+        coremark=$(grep -s "CoreMark [0-9.]* :" /tmp/coremark.$u.out | awk '{ print $4 }' 2>/dev/null)
+    fi
+    if [ -z "$coremark" ]; then
+        coremark=0
+    fi
+    rm /tmp/coremark.$u.out
+
+    # mmbm
+    mmbm=$(/usr/bin/scion-mmbm 2>/dev/null)
+    if [ -z "$mmbm" ]; then
+        mmbm="\"mmbm\": 0"
+    fi
+
+    # Publish
+    cat <<EOF > /www/horsepower.json
+{
+"coremark": ${coremark},
+${mmbm}
+}
+EOF
+
+    # Replace the scion-router initd script with a slightly modified one...
+    # (We need GOMAXPROCS set to a specific value)
+    cp -f /etc/scion/router-initd.default /etc/init.d/scion-router
+    chmod 755 /etc/init.d/scion-router
+}
+
+stop() {
+    true
+}
diff --git a/doc/dev/design/index.rst b/doc/dev/design/index.rst
index d2a249802..1eb308560 100644
--- a/doc/dev/design/index.rst
+++ b/doc/dev/design/index.rst
@@ -40,6 +40,7 @@ Active
    uri
    grpc
    BorderRouter
+   router-perf-model
 
 .. _design-docs-completed:
 
diff --git a/doc/dev/design/router-perf-model.rst b/doc/dev/design/router-perf-model.rst
new file mode 100644
index 000000000..7ccb63804
--- /dev/null
+++ b/doc/dev/design/router-perf-model.rst
@@ -0,0 +1,350 @@
+**************************************************
+Router benchmark observations and predictive model
+**************************************************
+
+-  Author(s): Jean-Christophe Hugly
+-  Last updated: 2024-04-22
+-  Discussion at: :issue:`4408`
+
+TL;DR
+=====
+
+For any hardware platform, given:
+
+-  :math:`L`: the router benchmark packet length (in bytes)
+-  coremark: The coremark benchmark result on the platform
+   (dimensionless)
+-  :math:`C`: The memmove benchmark result of the platform expressed (in bytes/s)
+-  rate: The benchmark result of the router on the platform, using three cores (in pkts/s)
+-  :math:`M`: an experimentally determined constant of value 400
+-  :math:`I`: a dimensionless number reflecting the router code’s performance
+
+At least for :math:`L` = 172, the following relationship is true:
+
+.. math::
+
+  I ~= (1 / coremark + (M \times L / C)) \times rate
+
+Introduction
+============
+
+The benchmark was ran on three configurations, the br_transit test case
+produced the following data points:
+
+-  An APU2 running openwrt:
+
+   -  4 amd cores 1GHz
+   -  Coremark (per core) 3044
+   -  memmove benchmark 248 MiB/s (1.9 Gib/s)
+   -  line rate 70K pkts/s
+   -  Observed throughput: 45377
+
+-  A lenovo laptop running the CI benchmark:
+
+   -  10 i7 cores 4GHz
+   -  Coremark (per core) 29861
+   -  memmove benchmark 3928 MiB/s (31 Gib/s)
+   -  line rate 1.4M pkts/s (veth)
+   -  Observed throuhgput: 639764 pkts/s
+
+-  Our CI system running the CI benchmark:
+
+   -  4 xeon cores 3.5 GHz
+   -  Coremark (per core) 28659
+   -  memmove benchmark 8102 MiB/s (64 Gib/s)
+   -  line rate unobserved (veth, assumed similar to laptop)
+   -  Observed throuhgput: 736357 pkts/s
+
+Important: all benchmarks have been run with only 3 cores assigned to
+the router. The cores are chosen by the benchmarking program to be of
+the same type (i.e. no mix of performance and energy-efficient cores).
+Hyperthreading is not used.
+
+See “The problem with the influence of core count”.
+
+Where might the time go
+=======================
+
+With a single core, the time processing a packet is presumed spend as follows...
+
+Variables:
+
+-  let :math:`b` = benchmark packet size (152)
+-  let :math:`e` = empty packet size (34)
+-  let :math:`m` = max packet size (1520)
+-  let :math:`po` = packet overhead time in seconds
+-  let :math:`L` = the length of a given packet [bytes]
+-  let :math:`x(L)` = packet-length dependent xmit time [s]
+-  let :math:`t(L)` = total packet transmission time for :math:`L` [s]
+-  let :math:`R` = nominal wire rate in decoded [bit/s]
+-  let :math:`X` = line slowness [seconds/byte]
+-  let :math:`bm(L)` = total benchmark packet processing time (including transmission and pkt overhead) [s]
+-  let :math:`ro` = router overhead [time added by processing one packet in the router] [s]
+-  let :math:`p(L)` = packet-length dependent router processing time for :math:`L` [s]
+-  let :math:`r(L)` = total processing time in router for :math:`L` [s]
+-  let :math:`C` = nominal copy rate in [bit/s]
+-  let :math:`Y` = copy slowness [seconds/byte]
+
+Transmission:
+
+-  assume :math:`x(L) = X \times L` [X is s/byte]
+-  assume :math:`t(L) = po + x(L)`
+-  assume :math:`X = 8/R`
+-  po is unknown
+
+Router:
+
+-  assume :math:`p(L) = Y \times L` [Y is s/byte]
+-  assume :math:`r(L) = ro + p(L)`
+-  assume :math:`Y = 8/C` [that is, the length-dependent processing time
+   is ~copying the packet]
+-  ro is unknown
+
+Aggregate:
+
+-  assume :math:`bm(L) = MAX(r(L), t(L))`
+
+What can be inferred from observations
+======================================
+
+APU2
+----
+
+Observed:
+
+-  :math:`t(m) = 1s/70K` (106 Mbyte/s, iperf3 with fast machine/nic -- assuming full packets)
+-  :math:`bm(b) = 1s/45377` (benchmark run)
+-  :math:`R = 1Gb/s` (nominal NIC rate)
+-  :math:`C = 1.9Gb/s` (mmbm - Go memmove small packets)
+
+Therefore:
+
+-  :math:`po = t(L) - x(L) = t(m) - x(m) = 1/70K - X \times m = 1/70K - 1520 \times 8 / 1G`
+   :math:`= 0.000002126 s`
+-  :math:`t(b) = po + x(b) = po + X \times b = 0.000002126 + 172 \times 8 / 1G = 0.000003502 s`
+-  :math:`bm(b) = 0.000024 s`
+
+Since :math:`bm(b) > t(b)`, we can conclude that the router isn’t processing at line speed,
+so :math:`bm(b) = r(b)`. That is :math:`bm(b)` reflects the router’s code performance.
+
+Therefore:
+
+-  :math:`r(b) = bm(b)`
+-  :math:`ro = r(b) - p(b)`
+-  :math:`ro = bm(b) - Y \times b = 1s / 45377 - 8 \times 172 / 1.9G = 0.00002131 s`
+
+Laptop local test
+-----------------
+
+Observed:
+
+-  :math:`t(m) = 1s/1.4M` (iperf3 on non-loopback ethernet interface)
+-  :math:`bm(b) = 1s/639764` (benchmark run)
+-  :math:`R = 17Gb/s` (same iperf3 run as t(m). Assuming po is neglictible)
+-  :math:`C = 31Gb/s` (mmbm - Go memmove small packets)
+
+Therefore:
+
+-  :math:`po = t(L) - x(L) = t(m) - x(m) = 1/1.4M - X \times m = 1/1.4M - 1520 \times 8 / 17G`
+   :math:`~= 0` (expected since we neglected po to derive R)
+-  :math:`t(b) = po + x(b) = 0 + X \times b = 172 \times 8 / 17G = 0.00000008 s`
+-  :math:`bm(b) = 0.000001563`
+
+Since :math:`bm(b) > t(b)` we can conclude that the router isn’t processing at line speed,
+so :math:`bm(b) = r(b)`. That is :math:`bm(b)` reflects the router’s code performance.
+
+Therefore:
+
+-  :math:`r(b) = bm(b)`
+-  :math:`ro = r(b) - p(b)`
+-  :math:`ro = bm(b) - Y \times b = 1s/639764 - 8 \times 172 / 31G = 0.000001519 s`
+
+Assumption of less-than-line-rate
+---------------------------------
+
+The case where the line rate is low enough to be the bottleneck isn’t
+very interresting. Because it is so clearly not the case with the
+hardware available to us, we will consider only the case where the wire
+is faster than the router.
+
+Tentative predictive model
+==========================
+
+Initial assumptions
+-------------------
+
+Variables:
+
+-  let :math:`pbm(L)` = predicted benchmark processing time for length :math:`L`
+-  let :math:`pt(L)` = predicted total transmission time for length :math:`L`
+-  let :math:`pp(L)` = predicted lenght-dependent processing time for :math:`L`
+-  let :math:`pro` = predicted router per-packet overhead
+-  let :math:`N` = The number of cores devote to packet processing
+-  let :math:`I` = The router’s code performance index; a measure of the code’s efficiency
+
+We assume that the length-independent time spent processing is inverse
+proportional to:
+
+-  coremark
+-  :math:`N`
+-  :math:`I`
+
+In all likelyness each packet is processed by one core, but we do not
+known that for sure. So, for now, we’re assuming that N packets are
+going to be processed by N cores as fast as 1 packet by one core (the
+difference, if any, would be an effect of the code’s quality and so
+reflected by the performance index. So, the statistical processing time
+per packet is 1/N that of single core processing.
+
+We assume that the length-dependent time processing the packet is proportional to:
+
+-  The length of the packet
+
+...and inverse proportional to:
+
+-  :math:`I` (in this case, reflecting how little copying is done)
+-  :math:`C`, The memcpy rate.
+
+Model
+-----
+
+From our assumptions, (and single I simplification) we have:
+
+-  :math:`pro = 1 / (I \times coremark)`
+-  :math:`pp(L) = 8 \times L / (I \times C)`
+-  :math:`pbm(L) = pro + pp(L)`
+   :math:`= 1 / (I \times coremark) + 8 \times L / (I \times C)`
+   :math:`= (1 / I) \times (1 / coremark) + (8 \times L / C))`
+-  :math:`I = (1 / coremark + (8 \times L / C)) / pbm(L)`
+
+Since we ran the same router on both benchmarking platform, we should be
+able to infer the same I from the benchmark result and hadrware
+characteristics. (or, at least, close).
+
+If so, that’s our platform independent performance index. That is, given
+a benchmark on any platform, with known C and coremark, we can find this
+number and use it to predict the performance of the same code on any
+other platform with known C and coremark.
+
+The influence of caches and TLBs
+--------------------------------
+
+Efforts to improve the precision of the mmbm benchmark (which attempts
+to measure :math:`C` showed that trying to model the behavior of caches
+and TLBs is extremely challenging. For example:
+
+-  The cache and TLB benefits do not disappear as the working set
+   exceeds the cache/TLB size. Nor are those benefits fully realized
+   while the working set remains within the cache size. The precise
+   behavior varies by CPU model.
+-  Page table walks polute the cache for some CPU models, but others
+   have a hidden cache exclusiveley for page table entries.
+-  Copying many 1 byte packets within a 128 pages arena takes
+   consistently more time than within an 8192 pages arena in at least
+   one CPU model. The reason for this remains a mystery.
+
+Multiway associative caches, combined with mostly secret replacement
+policies, undocumented caches, and intractable activity by the OS and
+the Go runtime make it extremely challenging to predict how a given
+workload (even a synthetic one) will perform on a given CPU. So far,
+this has not been achieved.
+
+As a distant second best, the mmbm benchmark only measures the copy
+speed in a handful of cases. For the purpose of modeling we use one case
+that is modeled after what the reference router does under benchmark. It
+is expected to cause non-temporal TLB access but a 100% hit rate on the
+L2 data cache, but that is not reflected reliably in the router’s
+benchmark results.
+
+By relying on these simple macroscopic metric and by tuning empirical
+parameters, we may be able to better approximate a cross-hardware router
+performance index. To that end we need to find or tune two additional
+parameters M and N such that:
+
+:math:`I = (1 / coremark + M * (8 \times L / C) + N) / pbm(L)`
+
+M represents the proportion in wich memory performance and arithmetic
+performance contribute to throughput. Such a ratio needs to exist for a
+translatable performance index.
+
+N represents a fixed hardware-dependent per-packet cost that, judging by
+the above results the model has failed to anticipate. We could speculate
+that this cost is related to interacting with network interfaces. If
+that is the case, a measure of it needs to be incorporated in our suite
+of microbenchmarks.
+
+Given a known N for three different systems, we could probably infer M.
+However, N is a property of each platform. It has to be measured.
+
+Inferred router performance index
+=================================
+
+The performance index has to be such that pbm(L) is equal to bm(L) as
+observed in at least two experiments. Note that only one type of
+forwarding is looked at - br_transit. Routers might have different
+performance indices for different packet types (although only small
+variations are expected).
+
+If we ignore M and N for now, (i.e. M = 1, N = 0), we have the
+following:
+
+.. _apu2-1:
+
+APU2
+----
+
+-  :math:`coremark = 3044`
+-  :math:`C = 1.9Gb/s`
+-  :math:`L = 172`
+-  :math:`pbm(L) = 1s/45377`
+-  :math:`I = (1 / coremark + (8 \times L / C)) / pbm(L)`
+   :math:`= (1 / 3044 + (8 \times 172 / 1.9G)) \times 45377`
+   :math:`~= 14.93`
+
+Laptop
+------
+
+-  :math:`coremark = 29851`
+-  :math:`C = 31Gb/s`
+-  :math:`L = 172`
+-  :math:`pbm(L) = 1s/639764`
+-  :math:`I = (1 / coremark + (8 \times L / C)) / pbm(L)`
+   :math:`= (1 / 29851 + (8 \times 172 / 31G)) \times 639864`
+   :math:`~= 21.46`
+
+CI system
+---------
+
+-  :math:`coremark = 28659`
+-  :math:`C = 64Gb/s`
+-  :math:`L = 172`
+-  :math:`pbm(L) = 1s/736357`
+-  :math:`I = (1 / coremark + (8 \times L / C)) / pbm(L)`
+   :math:`= (1 / 28659 + (8 \times 172 / 64G)) \times 736357`
+   :math:`~= 25.7`
+
+...Not close.
+
+The CI system has a coremark similar to that of the laptop but much
+faster memory copy. So memory copy may have a greater influence than
+coremark. In other words, the hypothetical M value (see the previous
+section) is much greater than 1. For example, assuming N=0, a value of
+400 for M yields performance indices of 28, 33 and 32 for APU2, the
+laptop, and the CI system, respectively. So, tuning is at least
+possible. This is purely speculative, though. We do not know if N can be
+neglected.
+
+Application to router improvement
+---------------------------------
+
+Some of the lessons learned during the benchmarking effort:
+
+-  The router performance is probably dominated by TLB misses.
+-  The router keeps 9K buffers, which makes 3/4 of all buffers
+   missaligned. That extra 1k is paid with an extra TLB miss.
+-  Small packets are dispersed over large buffers. Each small packet
+   requires at least one page access.
+-  At steady state, all buffers are eventually occupied, therefore we
+   cycle through the whole set. Since that set is much larger than the
+   cache, we loose all temporal locality: all packets get evicted from
+   the cache, possibly more than once, during processing.
diff --git a/tools/mmbm/BUILD.bazel b/tools/mmbm/BUILD.bazel
new file mode 100644
index 000000000..4b50a79bd
--- /dev/null
+++ b/tools/mmbm/BUILD.bazel
@@ -0,0 +1,15 @@
+load("//tools/lint:go.bzl", "go_library")
+load("//:scion.bzl", "scion_go_binary")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/scionproto/scion/tools/mmbm",
+    visibility = ["//visibility:private"],
+)
+
+scion_go_binary(
+    name = "mmbm",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
diff --git a/tools/mmbm/main.go b/tools/mmbm/main.go
new file mode 100644
index 000000000..43ced1f7b
--- /dev/null
+++ b/tools/mmbm/main.go
@@ -0,0 +1,175 @@
+// Copyright 2024 SCION Association
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// mmbm measures the performance of memory copy, as performed by the Go runtime.
+//
+// The go implementation of memcpy/memmove isn't necessarily the highest performing, but it is close
+// to that of glibc's and it has the advantages of being independent from any given libc
+// implementation. musl_libc's implementation, for example performs much worse.
+//
+// The copy speed is strongly influenced by caching effects, tlb effects, and the size of
+// the blocks being copied.
+// The CPUs available for benchmarking have resisted all attempts at analysing the TLB and
+// cache behaviour with any kind of reliability. So we have to be content with collecting some
+// empirical data that is relevant to the router benchmark and leave finer grain modeling for later.
+//
+// Therefore this benchmark measures the speed at which we can copy small packets with a working set
+// of a size similar to that of the router, plus a couple of other significant cases in the
+// hope that they can be used to make some not-too-incorrrect inferrences.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"testing"
+)
+
+// The arena that we play with.
+const allBufs = 8192
+const cycleStep = 5 // Must not divide any working set size, must not be 1.
+
+// Block is a packet buffer representation arranged to make it easy to copy 1, 172 or bufSize
+// bytes. It is exported to make sure go cannot optimize fields out.
+type Block struct {
+	packet [172]uint8
+	tail   [4096 - 172]uint8
+}
+
+var buf [allBufs]Block
+
+// We arrange to cycle through all the blocks (much like a steady state saturated router would.
+// Treat the buffers as a ring, with source and destination on opposite sides. Adjacent buffers
+// are used as far apart in time as possible (cycleStep != 1). No buffer is left unused (cycleStep
+// not a divisor).
+var last int = 0
+
+func nextPair(max int) (int, int) {
+	last = (last + max - cycleStep) % max
+	return last, (last + max/2) % max
+}
+
+// writeBuf: prevents go from taking possible advantage of the buffers being all zero.
+// This also allocates the memory pages and primes the cache as if in steady state.
+func writeBuf(numBufs int, cpSize int) {
+	for i := 0; i < numBufs; i++ {
+		for j := 0; j < 172; j++ {
+			buf[i].packet[j] = uint8(j % 256)
+		}
+		for j := 172; j < cpSize; j++ {
+			buf[i].tail[j-172] = uint8(j % 256)
+		}
+	}
+}
+
+// benchmarkCopyByte copies one byte from a block to another
+func benchmarkCopyByte(N int, numBufs int) {
+	for i := N; i > 0; i-- {
+		dst, src := nextPair(numBufs)
+		buf[dst].packet[0] = buf[src].packet[0]
+	}
+}
+
+// benchmarkCopyPacket copies a short packet from a block to another
+func benchmarkCopyPacket(N int, numBufs int) {
+	for i := N; i > 0; i-- {
+		dst, src := nextPair(numBufs)
+		buf[dst].packet = buf[src].packet
+	}
+}
+
+// benchmarkCopyBlock copies a short packet from a block to another
+func benchmarkCopyBlock(N int, numBufs int) {
+	for i := N; i > 0; i-- {
+		dst, src := nextPair(numBufs)
+		buf[dst] = buf[src]
+	}
+}
+
+// tc benchmarks one type of copy or another depending on the requested copySize and
+// using a working set of numBuf blocks. It returns a number of mebi bytes per second and
+// a number of million packets per second.
+func tc(name string, numBufs int, copySize int) (float64, float64) {
+	var res testing.BenchmarkResult
+	writeBuf(numBufs, copySize)
+	switch copySize {
+	case 1:
+		res = testing.Benchmark(func(b *testing.B) {
+			benchmarkCopyByte(b.N, numBufs)
+		})
+	case 172:
+		res = testing.Benchmark(func(b *testing.B) {
+			benchmarkCopyPacket(b.N, numBufs)
+		})
+	case 4096:
+		res = testing.Benchmark(func(b *testing.B) {
+			benchmarkCopyBlock(b.N, numBufs)
+		})
+	default:
+		panic("Size not supported")
+	}
+	bytes := uint64(res.N) * uint64(copySize)
+	megaBytes := float64(bytes) / (1024 * 1024)
+
+	mbps := megaBytes / res.T.Seconds()
+	mpps := float64(res.N) / float64(res.T.Microseconds())
+
+	return mbps, mpps
+}
+
+func main() {
+	testing.Init()
+	flag.VisitAll(func(f *flag.Flag) {
+		if f.Name == "test.benchtime" {
+			err := f.Value.Set("5s") // below that random outliers show-up.
+			if err != nil {
+				panic(err)
+			}
+		}
+	})
+
+	// In the following we assume an L2 TLB size between 1K and 2K, and an L2 cache size
+	// of 2M to 4M (32K to 64K lines).
+
+	// Per packet: 172b copy. No misses.
+	smallNoMiss, _ := tc("smallNoMiss", 128, 172)
+
+	// Per packet: 172b copy, TLB misses, no cache misses
+	// This is the router's default config under the standard benchmark run.
+	smallTlbMiss, _ := tc("smallTlbMiss", 3549, 172)
+
+	// "Per packet: 172b copy, Cache misses, no TLB misses" is not feasible.
+	// "Per packet, 172b copy, Cache misses, and TLB misses" is not realistic.
+
+	// Per packet: 4k copy. No misses.
+	largeNoMiss, _ := tc("largeNoMiss", 128, 4096)
+
+	// "Per packet: 4k copy, TLB misses, no cache misses" is not feasible.
+
+	// Per packet: 4k copy, Cache misses, no TLB misses
+	// (Works unless we have a CPU with the smallest TLB and the largeest cache).
+	largeCacheMiss, _ := tc("largeCacheMiss", 1024, 4096)
+
+	// Per packet, 4k copy, Cache misses, and TLB misses
+	largeAllMiss, _ := tc("largeAllMiss", 8192, 4096)
+
+	// All the results here. Best avoid printfs before. It tickles the gc.
+
+	fmt.Printf("\"mmbm\": %.2f,\n", smallTlbMiss) // Look at this one by default
+
+	fmt.Printf("\"mmbm_small_no_miss\": %.2f,\n", smallNoMiss)
+	fmt.Printf("\"mmbm_small_tlbmiss\": %.2f,\n", smallTlbMiss)
+	fmt.Printf("\"mmbm_large_no_miss\": %.2f,\n", largeNoMiss)
+	fmt.Printf("\"mmbm_large_cachemiss\": %.2f,\n", largeCacheMiss)
+	fmt.Printf("\"mmbm_large_allmiss\": %.2f\n", largeAllMiss)
+}


Test patch:
diff --git a/acceptance/router_benchmark/test.py b/acceptance/router_benchmark/test.py
index 366ec545b..b7b51c715 100644
--- a/acceptance/router_benchmark/test.py
+++ b/acceptance/router_benchmark/test.py
@@ -15,24 +15,21 @@
 # limitations under the License.
 
 
+import ipaddress
+import json
+import logging
 import shutil
 import time
 
+from acceptance.common import base
+from benchmarklib import Intf, RouterBM
 from collections import defaultdict, namedtuple
 from plumbum import cli
-from plumbum.cmd import docker, whoami, lscpu
 from plumbum import cmd
+from plumbum.cmd import docker, whoami, lscpu, taskset
+from plumbum.machines import LocalCommand
 from random import randint
 
-from acceptance.common import base
-
-import logging
-import json
-import ipaddress
-
-from http.client import HTTPConnection
-from urllib.parse import urlencode
-
 logger = logging.getLogger(__name__)
 
 # Those values are valid expectations only when running in the CI environment.
@@ -44,6 +41,9 @@ TEST_CASES = {
     "br_transit": 720000,
 }
 
+# Convenience types to carry interface request params.
+IntfReq = namedtuple("IntfReq", "label, prefix_len, ip, peer_ip, exclusive")
+
 
 def sudo(*args: [str]) -> str:
     # -A, --askpass makes sure command is failing and does not wait for
@@ -51,11 +51,6 @@ def sudo(*args: [str]) -> str:
     return cmd.sudo("-A", *args)
 
 
-# Convenience types to carry interface params.
-IntfReq = namedtuple("IntfReq", "label, prefixLen, ip, peerIp, exclusive")
-Intf = namedtuple("Intf", "name, mac, peerMac")
-
-
 # Make-up an eth mac address as unique as the given IP.
 # Assumes ips in a /16 or smaller block (i.e. The last two bytes are unique within the test).
 def mac_for_ip(ip: str) -> str:
@@ -162,7 +157,7 @@ def choose_cpus_from_best_cores(caches: list[int], cores: list[int]) -> list[int
     return chosen[0:4]
 
 
-class RouterBMTest(base.TestBase):
+class RouterBMTest(base.TestBase, RouterBM):
     """
     Tests that the implementation of a router has sufficient performance in terms of packets
     per available machine*second (i.e. one accumulated second of all configured CPUs available
@@ -183,9 +178,20 @@ class RouterBMTest(base.TestBase):
     Pretend traffic is injected by brload's. See the test cases for details.
     """
 
+    # TODO(jiceatscion): We construct intf_map during setup and we use it later, during
+    # _run(). As a result, running setup, run, at teardown separately is not possible for
+    # this test. May be it would be possible to reconstruct the map without actually setup the
+    # interfaces, assuming brload isn't being changed in-between.
+
     router_cpus: list[int] = [0]
+
+    # Used by the RouterBM mixin:
+    coremark: int = 0
+    mmbm: int = 0
+    intf_map: dict[str, Intf] = {}
+    brload: LocalCommand = None
     brload_cpus: list[int] = [0]
-    intfMap: dict[str, Intf] = {}
+    prom_address: str = "localhost:9999"
 
     ci = cli.Flag(
         "ci",
@@ -197,6 +203,7 @@ class RouterBMTest(base.TestBase):
         """Collects information about vcpus/cores/caches layout."""
 
         super().init()
+        self.brload = self.get_executable("brload")
         self.choose_cpus()
 
     def choose_cpus(self):
@@ -206,8 +213,6 @@ class RouterBMTest(base.TestBase):
         only as a last resort
         """
 
-        logger.info(f"CPUs summary BEGINS\n{cmd.lscpu('--extended')}\nCPUs summary ENDS")
-
         caches = defaultdict(list)  # cache -> [vcpu]
         cores = defaultdict(list)  # core -> [vcpu]
 
@@ -249,7 +254,7 @@ class RouterBMTest(base.TestBase):
             self.brload_cpus = chosen
         else:
             self.router_cpus = chosen[:-1]
-            self.brload_cpus = [chosen[-1]]
+            self.brload_cpus = chosen[-1:]
 
         logger.info(f"router cpus: {self.router_cpus}")
         logger.info(f"brload cpus: {self.brload_cpus}")
@@ -258,27 +263,27 @@ class RouterBMTest(base.TestBase):
         """Creates a pair of virtual interfaces, with one end in the given network namespace and the
         other in the host stack.
 
-        The outcome is the pair of interfaces and a record in intfMap, which associates the given
+        The outcome is the pair of interfaces and a record in intf_map, which associates the given
         label with the network interface's host-side name and two mac addresses; one for each end
         of the pair. The mac addresses, if they can be chosen, are not chosen by the invoker, but
         by this function. When we can choose, we follow a convention to facilitate debugging.
         Otherwise the values don't matter. brload has no expectations.
 
         We do not:
-          sudo("ip", "addr", "add", f"{req.peerIp}/{req.prefixLen}", "dev", hostIntf)
+          sudo("ip", "addr", "add", f"{req.peer_ip}/{req.prefix_len}", "dev", hostIntf)
 
         It causes trouble: if an IP is assigned, the kernel responds with "unbound port" icmp
         messages to the router traffic, which breaks the bound UDP connections that the router uses
         for external interfaces.
 
         We do not:
-          sudo("ip", "netns", "exec", ns, "ip", "neigh", "add", req.peerIp,
-               "lladdr", peerMac, "nud", "permanent", "dev", brIntf)
+          sudo("ip", "netns", "exec", ns, "ip", "neigh", "add", req.peer_ip,
+               "lladdr", peer_mac, "nud", "permanent", "dev", brIntf)
 
         This isn't need because brload now responds to arp requests.
 
         We do not:
-          sudo("ip", "link", "set", hostIntf, "address", peerMac)
+          sudo("ip", "link", "set", hostIntf, "address", peer_mac)
 
         If we do that, the interface address matches the dst addr in router->brload packets. This
         might seem desirable, even necessary, but is neither: since we're using veth pairs, the
@@ -288,65 +293,89 @@ class RouterBMTest(base.TestBase):
         use.
 
         Args:
-          IntfReq: A requested router-side network interface. It comprises:
-                   * A label by which brload identifies that interface.
-                   * The IP address to be assigned to that interface.
-                   * The IP address of one neighbor.
+          req: A requested router-side network interface. It comprises:
+            * A label by which brload identifies that interface.
+            * The IP address to be assigned to that interface.
+            * The IP address of one neighbor.
           ns: The network namespace where that interface must exist.
 
         """
 
-        physlabel = req.label if req.exclusive == "true" else "mx"
-        hostIntf = f"veth_{physlabel}_host"
-        brIntf = f"veth_{physlabel}"
+        phys_label = req.label if req.exclusive == "true" else "mx"
+        host_intf = f"veth_{phys_label}_host"
+        br_intf = f"veth_{phys_label}"
 
         # We do multiplex most requested router interfaces onto one physical interface pairs, so, we
         # must check that we haven't already created the physical pair.
-        for i in self.intfMap.values():
-            if i.name == hostIntf:
-                peerMac = i.peerMac
+        for i in self.intf_map.values():
+            if i.name == host_intf:
+                peer_mac = i.peer_mac
                 mac = i.mac
                 break
         else:
-            peerMac = mac_for_ip(req.peerIp)
+            peer_mac = mac_for_ip(req.peer_ip)
             mac = mac_for_ip(req.ip)
-            sudo("ip", "link", "add", hostIntf, "type", "veth", "peer", "name", brIntf)
-            sudo("ip", "link", "set", hostIntf, "mtu", "8000")
-            sudo("ip", "link", "set", hostIntf, "arp", "off")  # Make sure the real addr isn't used
+            sudo("ip", "link", "add", host_intf, "type", "veth", "peer", "name", br_intf)
+            sudo("ip", "link", "set", host_intf, "mtu", "8000")
+            sudo("ip", "link", "set", host_intf, "arp", "off")  # Make sure the real addr isn't used
 
             # Do not assign the host addresses but create one link-local addr.
             # Brload needs some src IP to send arp requests.
             sudo("ip", "addr", "add", f"169.254.{randint(0, 255)}.{randint(0, 255)}/16",
                  "broadcast", "169.254.255.255",
-                 "dev", hostIntf, "scope", "link")
+                 "dev", host_intf, "scope", "link")
 
-            sudo("sysctl", "-qw", f"net.ipv6.conf.{hostIntf}.disable_ipv6=1")
-            sudo("ethtool", "-K", brIntf, "rx", "off", "tx", "off")
-            sudo("ip", "link", "set", brIntf, "mtu", "8000")
-            sudo("ip", "link", "set", brIntf, "address", mac)
+            sudo("sysctl", "-qw", f"net.ipv6.conf.{host_intf}.disable_ipv6=1")
+            sudo("ethtool", "-K", br_intf, "rx", "off", "tx", "off")
+            sudo("ip", "link", "set", br_intf, "mtu", "8000")
+            sudo("ip", "link", "set", br_intf, "address", mac)
 
             # The network namespace
-            sudo("ip", "link", "set", brIntf, "netns", ns)
+            sudo("ip", "link", "set", br_intf, "netns", ns)
             sudo("ip", "netns", "exec", ns,
-                 "sysctl", "-qw", f"net.ipv6.conf.{brIntf}.disable_ipv6=1")
+                 "sysctl", "-qw", f"net.ipv6.conf.{br_intf}.disable_ipv6=1")
             sudo("ip", "netns", "exec", ns,
                  "sysctl", "-qw", "net.ipv4.conf.all.rp_filter=0")
             sudo("ip", "netns", "exec", ns,
-                 "sysctl", "-qw", f"net.ipv4.conf.{brIntf}.rp_filter=0")
+                 "sysctl", "-qw", f"net.ipv4.conf.{br_intf}.rp_filter=0")
 
         # Add the router side IP addresses (even if we're multiplexing on an existing interface).
         sudo("ip", "netns", "exec", ns,
-             "ip", "addr", "add", f"{req.ip}/{req.prefixLen}",
+             "ip", "addr", "add", f"{req.ip}/{req.prefix_len}",
              "broadcast",
-             ipaddress.ip_network(f"{req.ip}/{req.prefixLen}", strict=False).broadcast_address,
-             "dev", brIntf)
+             ipaddress.ip_network(f"{req.ip}/{req.prefix_len}", strict=False).broadcast_address,
+             "dev", br_intf)
 
         # Fit for duty.
-        sudo("ip", "link", "set", hostIntf, "up")
-        sudo("ip", "netns", "exec", ns, "ip", "link", "set", brIntf, "up")
+        sudo("ip", "link", "set", host_intf, "up")
+        sudo("ip", "netns", "exec", ns, "ip", "link", "set", br_intf, "up")
 
         # Ship it.
-        self.intfMap[req.label] = Intf(hostIntf, mac, peerMac)
+        self.intf_map[req.label] = Intf(host_intf, mac, peer_mac)
+
+    def fetch_horsepower(self):
+        try:
+            coremark_exe = self.get_executable("coremark")
+            output = taskset("-c", self.router_cpus[0], coremark_exe.executable)
+            line = output.splitlines()[-1]
+            if line.startswith("CoreMark "):
+                elems = line.split(" ")
+                if len(elems) >= 4:
+                    self.coremark = round(float(elems[3]))
+        except Exception as e:
+            logger.info(e)
+
+        try:
+            mmbm_exe = self.get_executable("mmbm")
+            output = taskset("-c", self.router_cpus[0], mmbm_exe.executable)
+            for line in output.splitlines():
+                if line.startswith("\"mmbm\": "):
+                    elems = line.strip(",").split(" ")
+                    if len(elems) >= 2:
+                        self.mmbm = round(float(elems[1]))
+                    break
+        except Exception as e:
+            logger.info(e)
 
     def setup_prepare(self):
         super().setup_prepare()
@@ -389,8 +418,7 @@ class RouterBMTest(base.TestBase):
         # The router uses one end and the test uses the other end to feed it with (and possibly
         # capture) traffic.
         # We supply the label->(host-side-name,mac,peermac) mapping to brload when we start it.
-        brload = self.get_executable("brload")
-        output = brload("show-interfaces")
+        output = self.brload("show-interfaces")
 
         for line in output.splitlines():
             elems = line.split(",")
@@ -406,7 +434,7 @@ class RouterBMTest(base.TestBase):
         docker("run",
                "-v", f"{self.artifacts}/conf:/etc/scion",
                "-d",
-               "-e", "GOMAXPROCS=3",
+               "-e", f"GOMAXPROCS={len(self.router_cpus)}",
                "--network", "container:prometheus",
                "--name", "router",
                "--cpuset-cpus", ",".join(map(str, self.router_cpus)),
@@ -414,6 +442,10 @@ class RouterBMTest(base.TestBase):
 
         time.sleep(2)
 
+        # Collect the horsepower microbenchmark numbers if we can.
+        # They'll be used to produce a performance index.
+        self.fetch_horsepower()
+
     def teardown(self):
         docker["logs", "router"].run_fg(retcode=None)
         docker("rm", "-f", "prometheus")
@@ -421,170 +453,21 @@ class RouterBMTest(base.TestBase):
         docker("network", "rm", "benchmark")  # veths are deleted automatically
         sudo("chown", "-R", whoami().strip(), self.artifacts)
 
-    def exec_br_load(self, case: str, mapArgs: list[str], count: int) -> str:
-        brload = self.get_executable("brload")
-        # For num-streams, attempt to distribute uniformly on many possible number of cores.
-        # 840 is a multiple of 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 15, 20, 21, 24, 28, ...
-        return sudo("taskset", "-c", ",".join(map(str, self.brload_cpus)),
-                    brload.executable,
-                    "run",
-                    "--artifacts", self.artifacts,
-                    *mapArgs,
-                    "--case", case,
-                    "--num-packets", str(count),
-                    "--num-streams", "840")
-
-    def run_test_case(self, case: str, mapArgs: list[str]) -> (int, int):
-        logger.info(f"==> Starting load {case}")
-
-        output = self.exec_br_load(case, mapArgs, 10000000)
-        for line in output.splitlines():
-            if line.startswith("metricsBegin"):
-                _, beg, _, end = line.split()
-
-        logger.info(f"==> Collecting {case} performance metrics...")
-
-        # The raw metrics are expressed in terms of core*seconds. We convert to machine*seconds
-        # which allows us to provide a projected packet/s; ...more intuitive than packets/core*s.
-        # We measure the rate over 10s. For best results we sample the end of the middle 10s of the
-        # run. "beg" is the start time of the real action and "end" is the end time.
-        sampleTime = (int(beg) + int(end) + 10) / 2
-        promQuery = urlencode({
-            'time': f'{sampleTime}',
-            'query': (
-                'sum by (instance, job) ('
-                f'  rate(router_output_pkts_total{{job="BR", type="{case}"}}[10s])'
-                ')'
-                '/ on (instance, job) group_left()'
-                'sum by (instance, job) ('
-                '  1 - (rate(process_runnable_seconds_total[10s])'
-                '       / go_sched_maxprocs_threads)'
-                ')'
-            )
-        })
-        conn = HTTPConnection("localhost:9999")
-        conn.request("GET", f"/api/v1/query?{promQuery}")
-        resp = conn.getresponse()
-        if resp.status != 200:
-            raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}")
-
-        # There's only one router, so whichever metric we get is the right one.
-        pld = json.loads(resp.read().decode("utf-8"))
-        processed = 0
-        results = pld["data"]["result"]
-        for result in results:
-            ts, val = result["value"]
-            processed = int(float(val))
-            break
-
-        # Collect dropped packets metrics, so we can verify that the router was well saturated.
-        # If not, the metrics aren't very useful.
-        promQuery = urlencode({
-            'time': f'{sampleTime}',
-            'query': (
-                'sum by (instance, job) ('
-                '  rate(router_dropped_pkts_total{job="BR", reason=~"busy_.*"}[10s])'
-                ')'
-                '/ on (instance, job) group_left()'
-                'sum by (instance, job) ('
-                '  1 - (rate(process_runnable_seconds_total[10s])'
-                '       / go_sched_maxprocs_threads)'
-                ')'
-            )
-        })
-        conn = HTTPConnection("localhost:9999")
-        conn.request("GET", f"/api/v1/query?{promQuery}")
-        resp = conn.getresponse()
-        if resp.status != 200:
-            raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}")
-
-        # There's only one router, so whichever metric we get is the right one.
-        pld = json.loads(resp.read().decode("utf-8"))
-        dropped = 0
-        results = pld["data"]["result"]
-        for result in results:
-            ts, val = result["value"]
-            dropped = int(float(val))
-            break
-
-        return processed, dropped
-
-    # Fetch and log the number of cores used by Go. This may inform performance
-    # modeling later.
-    def log_core_counts(self):
-        logger.info("==> Collecting number of cores...")
-        promQuery = urlencode({
-            'query': 'go_sched_maxprocs_threads{job="BR"}'
-        })
-
-        conn = HTTPConnection("localhost:9999")
-        conn.request("GET", f"/api/v1/query?{promQuery}")
-        resp = conn.getresponse()
-        if resp.status != 200:
-            raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}")
-
-        pld = json.loads(resp.read().decode("utf-8"))
-        results = pld["data"]["result"]
-        for result in results:
-            instance = result["metric"]["instance"]
-            _, val = result["value"]
-            logger.info(f"Router Cores for {instance}: {int(val)}")
-
     def _run(self):
-        # Build the interface mapping arg
-        mapArgs = []
-        for label, intf in self.intfMap.items():
-            mapArgs.extend(["--interface", f"{label}={intf.name},{intf.peerMac}"])
-
-        # Run one test (30% size) as warm-up to trigger the frequency scaling, else the first test
-        # gets much lower performance.
-        self.exec_br_load(list(TEST_CASES)[0], mapArgs, 3000000)
-
-        # At long last, run the tests
-        rateMap = {}
-        droppageMap = {}
-        for testCase in TEST_CASES:
-            processed, dropped = self.run_test_case(testCase, mapArgs)
-            rateMap[testCase] = processed
-            droppageMap[testCase] = dropped
-
-        self.log_core_counts()
-
-        # Log and check the performance...
-        # If this is used as a CI test. Make sure that the performance is within the expected
-        # ballpark.
-        rateTooLow = []
-        for tt, exp in TEST_CASES.items():
-            if self.ci and exp != 0:
-                logger.info(f"Packets/(machine*s) for {tt}: {rateMap[tt]} expected: {exp}")
-                if rateMap[tt] < 0.8 * exp:
-                    rateTooLow.append(tt)
-            else:
-                logger.info(f"Packets/(machine*s) for {tt}: {rateMap[tt]}")
-
-        if len(rateTooLow) != 0:
-            raise RuntimeError(f"Insufficient performance for: {rateTooLow}")
-
-        # Log and check the saturation...
-        # If this is used as a CI test. Make sure that the saturation is within the expected
-        # ballpark: that should manifest some measurable amount of loss due to queue overflow.
-        notSaturated = []
-        for tt in TEST_CASES:
-            total = rateMap[tt] + droppageMap[tt]
-            if total == 0:
-                logger.info(f"Droppage ratio unavailable for {tt}")
-            else:
-                ratio = float(droppageMap[tt]) / total
-                exp = 0.03
-                if self.ci:
-                    logger.info(f"Droppage ratio for {tt}: {ratio:.1%} expected: {exp:.1%}")
-                    if ratio < exp:
-                        notSaturated.append(tt)
-                else:
-                    logger.info(f"Droppage ratio for {tt}: {ratio:.1%}")
-
-        if len(notSaturated) != 0:
-            raise RuntimeError(f"Insufficient saturation for: {notSaturated}")
+        results = self.run_bm(list(TEST_CASES.keys()))
+        if results.cores != len(self.router_cpus):
+            raise RuntimeError("Wrong number of cores used by the router; "
+                               f"planned: {len(self.router_cpus)}), observed: {results.cores}")
+
+        if self.ci:
+            results.CI_check(TEST_CASES)
+
+        # Output results as json for easier post-processing
+        logger.info(results.as_json())
+
+        if self.ci:
+            if results.failed:
+                raise RuntimeError("CI check failed")
 
 
 if __name__ == "__main__":
diff --git a/dist/openwrt/test_configs/router-initd b/dist/openwrt/test_configs/router-initd
new file mode 100644
index 000000000..603df1975
--- /dev/null
+++ b/dist/openwrt/test_configs/router-initd
@@ -0,0 +1,19 @@
+#!/bin/sh /etc/rc.common
+# "new(er)" style init script
+# Look at /lib/functions/service.sh on a running system for explanations of what other SERVICE_
+# options you can use, and when you might want them.
+
+START=99
+STOP=0
+APP=scion-router
+SERVICE_WRITE_PID=1
+SERVICE_DAEMONIZE=1
+
+start() {
+    export GOMAXPROCS=3
+    service_start /usr/bin/$APP --config /etc/scion/router.toml
+}
+
+stop() {
+    service_stop /usr/bin/$APP
+}
diff --git a/dist/test/openwrt_test.sh b/dist/test/openwrt_test.sh
index 7082254f6..cfd4ee946 100755
--- a/dist/test/openwrt_test.sh
+++ b/dist/test/openwrt_test.sh
@@ -44,15 +44,19 @@ docker exec -i openwrt-x86_64 /bin/ash <<'EOF'
     cd /openwrt
 
     # check that the pki files are all here (avoid cryptic error from opk)
-    for c in persistdbs testconfig router control dispatcher daemon ip-gateway tools coremark; do
+    for c in persistdbs testconfig router control dispatcher daemon ip-gateway tools bmtools; do
     	ls /openwrt/scion-${c}_*_${arch}.ipk > /dev/null
     done
 
     # Start with the easy stuff. Install and run coremark.
-    opkg install scion-coremark_*_${arch}.ipk
+    opkg install scion-bmtools_*_${arch}.ipk
     /usr/bin/scion-coremark > /tmp/coremark.out
     cat /tmp/coremark.out
 
+    # Uninstall bmtooling. It contains benchmark-only configs that can get in
+    # the way.
+    opkg remove scion-bmtools
+
     # Now the real stuff...
 
     # Install the persistdbs and testconfig packages. It's just basic config