Repository: google/fuzzbench

Issue:
Move/publish reports of experimental experiments under fuzzbench.com/reports/experimental
Experimental experiments are experiments with fuzzers not in this list:
https://github.com/google/fuzzbench/blob/master/service/core-fuzzers.yaml


Discussion:
Actually i can knock this out.

Resolving patch:
diff --git a/experiment/reporter.py b/experiment/reporter.py
--- a/experiment/reporter.py
+++ b/experiment/reporter.py
@@ -26,8 +26,9 @@
 from analysis import generate_report
 from analysis import data_utils
 
-CORE_FUZZERS_YAML = os.path.join(os.path.dirname(__file__), '..', 'service',
-                                 'core-fuzzers.yaml')
+CORE_FUZZERS_YAML = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', 'service',
+                 'core-fuzzers.yaml'))
 
 logger = logs.Logger('reporter')  # pylint: disable=invalid-name
 
@@ -37,18 +38,29 @@ def get_reports_dir():
     return exp_path.path('reports')
 
 
+def get_core_fuzzers():
+    """Return list of core fuzzers to be used for merging experiment data."""
+    return yaml_utils.read(CORE_FUZZERS_YAML)['fuzzers']
+
+
 def output_report(experiment_config: dict,
                   in_progress=False,
                   coverage_report=False):
     """Generate the HTML report and write it to |web_bucket|."""
     experiment_name = experiment_utils.get_experiment_name()
-    web_filestore_path = posixpath.join(experiment_config['report_filestore'],
-                                        experiment_name)
-
     reports_dir = get_reports_dir()
 
-    core_fuzzers = yaml_utils.read(CORE_FUZZERS_YAML)['fuzzers']
-    fuzzers = sorted(set(experiment_config['fuzzers']).union(set(core_fuzzers)))
+    core_fuzzers = set(get_core_fuzzers())
+    experiment_fuzzers = set(experiment_config['fuzzers'])
+    fuzzers = experiment_fuzzers.union(core_fuzzers)
+
+    # Calculate path to store report files in filestore.
+    web_filestore_path = experiment_config['report_filestore']
+    if not fuzzers.issubset(core_fuzzers):
+        # This means that we are running an experimental report with fuzzers
+        # not in the core list. So, store these in |experimental| sub-directory.
+        web_filestore_path = os.path.join(web_filestore_path, 'experimental')
+    web_filestore_path = posixpath.join(web_filestore_path, experiment_name)
 
     # Don't merge with nonprivate experiments until the very end as doing it
     # while the experiment is in progress will produce unusable realtime


Test patch:
diff --git a/experiment/test_reporter.py b/experiment/test_reporter.py
--- a/experiment/test_reporter.py
+++ b/experiment/test_reporter.py
@@ -14,30 +14,54 @@
 """Tests for reporter.py."""
 
 import os
-from unittest import mock
 
-import yaml
+from unittest import mock
+import pytest
 
+from common import yaml_utils
 from experiment import reporter
 from test_libs import utils as test_utils
 
 # pylint: disable=unused-argument,invalid-name
 
 
-def test_output_report_filestore(fs, experiment):
-    """Test that output_report writes the report and rsyncs it to the report
-    filestore."""
-    config_filepath = os.path.join(os.path.dirname(__file__), '..', 'service',
-                                   'core-fuzzers.yaml')
-    fs.add_real_file(config_filepath)
+def _setup_experiment_files(fs):
+    """Set up experiment config and core-fuzzers files and return experiment
+    config yaml."""
+    fs.add_real_file(reporter.CORE_FUZZERS_YAML)
 
-    # Get the config.
     config_filepath = os.path.join(os.path.dirname(__file__), 'test_data',
                                    'experiment-config.yaml')
     fs.add_real_file(config_filepath)
+    experiment_config = yaml_utils.read(config_filepath)
 
-    with open(config_filepath) as file_handle:
-        experiment_config = yaml.load(file_handle, yaml.SafeLoader)
+    return experiment_config
+
+
+@pytest.mark.parametrize(
+    ('experiment_fuzzers', 'expected_merged_fuzzers', 'expected_report_url'),
+    [  # Tests the following usecases:
+        # If fuzzer variants are provided, then we merge them with the core
+        # fuzzers and put them in the |experimental| sub-directory.
+        (['fuzzer-a', 'fuzzer-b'
+         ], set(['fuzzer-a', 'fuzzer-b'] + reporter.get_core_fuzzers()),
+         'gs://web-reports/experimental/test-experiment'),
+        # If we provide the core fuzzers (e.g. in an official experiment),
+        # then they show up in the root reports directory.
+        (reporter.get_core_fuzzers(), set(
+            reporter.get_core_fuzzers()), 'gs://web-reports/test-experiment'),
+        # If we provide a subset of the core fuzzers, then they are merged with
+        # the other core fuzzers from older experiments and still show up in
+        # the root reports directory.
+        (reporter.get_core_fuzzers()[:2], set(
+            reporter.get_core_fuzzers()), 'gs://web-reports/test-experiment')
+    ])
+def test_output_report_filestore(experiment_fuzzers, expected_merged_fuzzers,
+                                 expected_report_url, fs, experiment):
+    """Test that output_report writes the report and rsyncs it to the report
+    filestore."""
+    experiment_config = _setup_experiment_files(fs)
+    experiment_config['fuzzers'] = experiment_fuzzers
 
     with test_utils.mock_popen_ctx_mgr() as mocked_popen:
         with mock.patch('analysis.generate_report.generate_report'
@@ -46,28 +70,14 @@ def test_output_report_filestore(fs, experiment):
             reports_dir = os.path.join(os.environ['WORK'], 'reports')
             assert mocked_popen.commands == [[
                 'gsutil', '-h', 'Cache-Control:public,max-age=0,no-transform',
-                'rsync', '-r', reports_dir, 'gs://web-reports/test-experiment'
+                'rsync', '-r', reports_dir, expected_report_url
             ]]
             experiment_name = os.environ['EXPERIMENT']
             mocked_generate_report.assert_called_with(
                 [experiment_name],
                 reports_dir,
                 report_name=experiment_name,
-                fuzzers=[
-                    'afl',
-                    'aflfast',
-                    'aflplusplus_optimal',
-                    'aflsmart',
-                    'entropic',
-                    'fairfuzz',
-                    'fuzzer-a',
-                    'fuzzer-b',
-                    'honggfuzz',
-                    'lafintel',
-                    'libaflfuzzer',
-                    'libfuzzer',
-                    'mopt',
-                ],
+                fuzzers=expected_merged_fuzzers,
                 in_progress=False,
                 merge_with_clobber_nonprivate=False,
                 coverage_report=False)