Repository: ethereum/consensus-specs

Issue:
Add diagnostics to spec test output
We collect some diagnostics about the spec test generation in this function: https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py#L41

This information is logged to STDOUT but is generally not persisted anywhere.

It would be helpful for various reasons to write the output of the run to a file along w/ the test artifacts.

Given that generators are generally separate (and very likely run in parallel), I suggest two steps:

1) Just write a local file with the diagnostics to the top-level generated directory for the particular generator currently running.
2) Consolidate all diagnostics into one accumulator (possibly writing to a shared file w/ a file lock like https://py-filelock.readthedocs.io/en/latest/ or something)

With this information, we can do things like ensure we don't elide any previously established tests between runs of the generator. We can also provide the expected number of tests which facilitates consumers working w/ the test artifacts.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -1041,7 +1041,7 @@ def run(self):
     extras_require={
         "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"],
         "lint": ["flake8==3.7.7", "mypy==0.812", "pylint==2.12.2"],
-        "generator": ["python-snappy==0.5.4"],
+        "generator": ["python-snappy==0.5.4", "filelock"],
     },
     install_requires=[
         "eth-utils>=1.3.0,<2",


Test patch:
diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
--- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
+++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
@@ -3,7 +3,9 @@
 import shutil
 import argparse
 from pathlib import Path
+from filelock import FileLock
 import sys
+import json
 from typing import Iterable, AnyStr, Any, Callable
 import traceback
 
@@ -111,6 +113,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     collected_test_count = 0
     generated_test_count = 0
     skipped_test_count = 0
+    test_identifiers = []
+
     provider_start = time.time()
     for tprov in test_providers:
         if not collect_only:
@@ -123,12 +127,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
                 / Path(test_case.runner_name) / Path(test_case.handler_name)
                 / Path(test_case.suite_name) / Path(test_case.case_name)
             )
-            incomplete_tag_file = case_dir / "INCOMPLETE"
-
             collected_test_count += 1
-            if collect_only:
-                print(f"Collected test at: {case_dir}")
-                continue
+            print(f"Collected test at: {case_dir}")
+
+            incomplete_tag_file = case_dir / "INCOMPLETE"
 
             if case_dir.exists():
                 if not args.force and not incomplete_tag_file.exists():
@@ -198,6 +200,15 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]):
                     shutil.rmtree(case_dir)
                 else:
                     generated_test_count += 1
+                    test_identifier = "::".join([
+                        test_case.preset_name,
+                        test_case.fork_name,
+                        test_case.runner_name,
+                        test_case.handler_name,
+                        test_case.suite_name,
+                        test_case.case_name
+                    ])
+                    test_identifiers.append(test_identifier)
                     # Only remove `INCOMPLETE` tag file
                     os.remove(incomplete_tag_file)
             test_end = time.time()
@@ -216,6 +227,28 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]):
         if span > TIME_THRESHOLD_TO_PRINT:
             summary_message += f" in {span} seconds"
         print(summary_message)
+    diagnostics = {
+        "collected_test_count": collected_test_count,
+        "generated_test_count": generated_test_count,
+        "skipped_test_count": skipped_test_count,
+        "test_identifiers": test_identifiers,
+        "durations": [f"{span} seconds"],
+    }
+    diagnostics_path = Path(os.path.join(output_dir, "diagnostics.json"))
+    diagnostics_lock = FileLock(os.path.join(output_dir, "diagnostics.json.lock"))
+    with diagnostics_lock:
+        diagnostics_path.touch(exist_ok=True)
+        if os.path.getsize(diagnostics_path) == 0:
+            with open(diagnostics_path, "w+") as f:
+                json.dump(diagnostics, f)
+        else:
+            with open(diagnostics_path, "r+") as f:
+                existing_diagnostics = json.load(f)
+                for k, v in diagnostics.items():
+                    existing_diagnostics[k] += v
+            with open(diagnostics_path, "w+") as f:
+                json.dump(existing_diagnostics, f)
+        print(f"wrote diagnostics to {diagnostics_path}")
 
 
 def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML):