Repository: aai-institute/nnbench

Issue:
Add a run name to the BenchmarkRecord (and JSON representations)
Benchmark records should get a name to identify them with. This doesn't have to be overengineered (something like a UUID or prefix + machine name should be fine), but it can be useful e.g. as a primary key for database streaming or an extra index for a table-based report.

Steps:

- [ ] Add the name as a slot to the `BenchmarkRecord` class.
- [ ] Add a name to the `BenchmarkRunner.run()` method and pipe it through to the resulting record.
- [ ] Inline the name into each result in `BenchmarkRecord.to_list()` (beware, the "name" field is already taken to mean the benchmark name - so maybe we have to use a different key like "run-name".)

Discussion:

Patch:
diff --git a/src/nnbench/runner.py b/src/nnbench/runner.py
index ace830a..ae37729 100644
--- a/src/nnbench/runner.py
+++ b/src/nnbench/runner.py
@@ -5,8 +5,10 @@ import contextlib
 import inspect
 import logging
 import os
+import platform
 import sys
 import time
+import uuid
 import warnings
 from collections.abc import Callable, Generator, Sequence
 from dataclasses import asdict
@@ -22,14 +24,6 @@ from nnbench.util import import_file_as_module, ismodule
 logger = logging.getLogger(__name__)
 
 
-def iscontainer(s: Any) -> bool:
-    return isinstance(s, tuple | list)
-
-
-def isdunder(s: str) -> bool:
-    return s.startswith("__") and s.endswith("__")
-
-
 def qualname(fn: Callable) -> str:
     if fn.__name__ == fn.__qualname__:
         return fn.__name__
@@ -251,12 +245,13 @@ class BenchmarkRunner:
 
         # iterate through the module dict members to register
         for k, v in module.__dict__.items():
-            if isdunder(k):
+            if k.startswith("__") and k.endswith("__"):
+                # dunder names are ignored.
                 continue
             elif isinstance(v, self.benchmark_type):
                 if not tags or set(tags) & set(v.tags):
                     self.benchmarks.append(v)
-            elif iscontainer(v):
+            elif isinstance(v, list | tuple | set | frozenset):
                 for bm in v:
                     if isinstance(bm, self.benchmark_type):
                         if not tags or set(tags) & set(bm.tags):
@@ -268,6 +263,7 @@ class BenchmarkRunner:
         params: dict[str, Any] | Parameters | None = None,
         tags: tuple[str, ...] = (),
         context: Sequence[ContextProvider] = (),
+        name: str | None = None,
     ) -> BenchmarkRecord:
         """
         Run a previously collected benchmark workload.
@@ -287,14 +283,18 @@ class BenchmarkRunner:
             Additional context to log with the benchmark in the output JSON record. Useful for
             obtaining environment information and configuration, like CPU/GPU hardware info,
             ML model metadata, and more.
+        name: str | None
+            A name for the currently started run. If None, a name will be automatically generated.
 
         Returns
         -------
         BenchmarkRecord
-            A JSON output representing the benchmark results. Has two top-level keys, "context"
-            holding the context information, and "benchmarks", holding an array with the
-            benchmark results.
+            A JSON output representing the benchmark results. Has three top-level keys,
+            "name" giving the benchmark run name, "context" holding the context information,
+            and "benchmarks", holding an array with the benchmark results.
         """
+        name = name or "nnbench-" + platform.node() + "-" + uuid.uuid1().hex[:8]
+
         if not self.benchmarks:
             self.collect(path_or_module, tags)
 
@@ -313,7 +313,7 @@ class BenchmarkRunner:
         # if we didn't find any benchmarks, warn and return an empty record.
         if not self.benchmarks:
             warnings.warn(f"No benchmarks found in path/module {str(path_or_module)!r}.")
-            return BenchmarkRecord(context=ctx, benchmarks=[])
+            return BenchmarkRecord(name=name, context=ctx, benchmarks=[])
 
         for bm in self.benchmarks:
             family_sizes[bm.fn.__name__] += 1
@@ -370,6 +370,7 @@ class BenchmarkRunner:
                 results.append(res)
 
         return BenchmarkRecord(
+            name=name,
             context=ctx,
             benchmarks=results,
         )
diff --git a/src/nnbench/types/benchmark.py b/src/nnbench/types/benchmark.py
index c734f94..2738343 100644
--- a/src/nnbench/types/benchmark.py
+++ b/src/nnbench/types/benchmark.py
@@ -29,6 +29,7 @@ def NoOp(state: State, params: Mapping[str, Any] = MappingProxyType({})) -> None
 
 @dataclass(frozen=True)
 class BenchmarkRecord:
+    name: str
     context: dict[str, Any]
     benchmarks: list[dict[str, Any]]
 
@@ -51,8 +52,8 @@ class BenchmarkRecord:
         results = []
         for b in self.benchmarks:
             bc = copy.deepcopy(b)
-            # TODO: Give an option to elide (or process) the context?
             bc["context"] = self.context
+            bc["run-name"] = self.name
             results.append(bc)
         return results
 
@@ -81,16 +82,20 @@ class BenchmarkRecord:
 
             benchmarks = bms["benchmarks"]
             context = bms.get("context", {})
+            name = bms.get("name", "")
         else:
+            name = ""
             context = {}
             benchmarks = bms
             for b in benchmarks:
-                # Safeguard if the context is not in the record,
-                # for example if it came from a DB query.
+                # TODO(nicholasjng): This does not do the right thing if the list contains
+                #  data from multiple benchmark runs, e.g. from a DB query.
+                if "run-name" in b:
+                    name = b.pop("run-name")
                 if "context" in b:
                     # TODO: Log context key/value disagreements
                     context |= b.pop("context", {})
-        return cls(benchmarks=benchmarks, context=context)
+        return cls(name=name, benchmarks=benchmarks, context=context)
 
 
 @dataclass(frozen=True)


Test patch:
diff --git a/tests/test_fileio.py b/tests/test_fileio.py
index 8a43d8d..0d5dfb5 100644
--- a/tests/test_fileio.py
+++ b/tests/test_fileio.py
@@ -15,6 +15,7 @@ def test_fileio_writes_no_compression_inline(tmp_path: Path, ext: str) -> None:
     f = FileReporter()
 
     rec = BenchmarkRecord(
+        name="my-run",
         context={"a": "b", "s": 1, "b.c": 1.0},
         benchmarks=[{"name": "foo", "value": 1}, {"name": "bar", "value": 2}],
     )