Repository: google/fuzzbench

Issue:
afl++ shows up on openssl_x509 benchmark in 2020-03-11 report even though build is broken
See https://www.fuzzbench.com/reports/2020-03-11/index.html
As i reported in #110 the afl++/openssl_x509 failed.
I assume FuzzBench used old docker image. I need to change things to not run trials for a fuzzer,benchmark pair when the build fails.


Discussion:

Resolving patch:
diff --git a/experiment/build/builder.py b/experiment/build/builder.py
--- a/experiment/build/builder.py
+++ b/experiment/build/builder.py
@@ -68,15 +68,16 @@ def build_measurer(benchmark: str) -> bool:
 
 
 def build_all_measurers(benchmarks: List[str]) -> List[str]:
-    """Build measurers for benchmarks."""
+    """Build measurers for each benchmark in |benchmarks| in parallel
+    Returns a list of benchmarks built successfully."""
     logger.info('Building measurers.')
     filesystem.recreate_directory(build_utils.get_coverage_binaries_dir())
-    benchmarks = [(benchmark,) for benchmark in benchmarks]
-    results = retry_build_loop(build_measurer, benchmarks)
+    build_measurer_args = [(benchmark,) for benchmark in benchmarks]
+    successful_calls = retry_build_loop(build_measurer, build_measurer_args)
     logger.info('Done building measurers.')
     # Return list of benchmarks (like the list we were passed as an argument)
     # instead of returning a list of tuples each containing a benchmark.
-    return [result[0] for result in results]
+    return [successful_call[0] for successful_call in successful_calls]
 
 
 def split_successes_and_failures(inputs: List,
@@ -96,8 +97,9 @@ def split_successes_and_failures(inputs: List,
 
 
 def retry_build_loop(build_func: Callable, inputs: List[Tuple]) -> List:
-    """Call |build_func| concurrently on |inputs|. Repeat on failures up to
-    |NUM_BUILD_RETRIES| times."""
+    """Calls |build_func| in parallel on |inputs|. Repeat on failures up to
+    |NUM_BUILD_RETRIES| times. Returns the list of inputs that |build_func| was
+    called successfully on."""
     successes = []
     with mp_pool.ThreadPool(MAX_CONCURRENT_BUILDS) as pool:
         for _ in range(NUM_BUILD_RETRIES):
@@ -122,7 +124,8 @@ def retry_build_loop(build_func: Callable, inputs: List[Tuple]) -> List:
 
 def build_fuzzer_benchmark(fuzzer: str, benchmark: str) -> bool:
     """Wrapper around buildlib.build_fuzzer_benchmark that logs and catches
-    exceptions."""
+    exceptions. buildlib.build_fuzzer_benchmark builds an image for |fuzzer|
+    to fuzz |benchmark|."""
     logger.info('Building benchmark: %s, fuzzer: %s.', benchmark, fuzzer)
     try:
         buildlib.build_fuzzer_benchmark(fuzzer, benchmark)
@@ -136,15 +139,17 @@ def build_fuzzer_benchmark(fuzzer: str, benchmark: str) -> bool:
 
 def build_all_fuzzer_benchmarks(fuzzers: List[str],
                                 benchmarks: List[str]) -> List[str]:
-    """Call buildlib.build_fuzzer_benchmark on each fuzzer,benchmark pair
-    concurrently."""
+    """Build fuzzer,benchmark images for all pairs of |fuzzers| and |benchmarks|
+    in parallel. Returns a list of fuzzer,benchmark pairs that built
+    successfully."""
     logger.info('Building all fuzzer benchmarks.')
-    product = list(itertools.product(fuzzers, benchmarks))
+    build_fuzzer_benchmark_args = list(itertools.product(fuzzers, benchmarks))
     # TODO(metzman): Use an asynchronous unordered map variant to schedule
     # eagerly.
-    results = retry_build_loop(build_fuzzer_benchmark, product)
+    successful_calls = retry_build_loop(build_fuzzer_benchmark,
+                                        build_fuzzer_benchmark_args)
     logger.info('Done building fuzzer benchmarks.')
-    return results
+    return successful_calls
 
 
 def main():
diff --git a/experiment/dispatcher.py b/experiment/dispatcher.py
--- a/experiment/dispatcher.py
+++ b/experiment/dispatcher.py
@@ -16,7 +16,6 @@
 configuration, spawns a runner VM for each benchmark-fuzzer combo, and then
 records coverage data received from the runner VMs."""
 
-import itertools
 import multiprocessing
 import os
 import posixpath
@@ -48,8 +47,7 @@ def create_work_subdirs(subdirs: List[str]):
 
 
 def _initialize_experiment_in_db(experiment: str, git_hash: str,
-                                 benchmarks: List[str], fuzzers: List[str],
-                                 num_trials: int):
+                                 trials: List[models.Trial]):
     """Initializes |experiment| in the database by creating the experiment
     entity and entities for each trial in the experiment."""
     db_utils.add_all([
@@ -58,12 +56,6 @@ def _initialize_experiment_in_db(experiment: str, git_hash: str,
                                git_hash=git_hash)
     ])
 
-    trials_args = itertools.product(sorted(benchmarks), range(num_trials),
-                                    sorted(fuzzers))
-    trials = [
-        models.Trial(fuzzer=fuzzer, experiment=experiment, benchmark=benchmark)
-        for benchmark, _, fuzzer in trials_args
-    ]
     # TODO(metzman): Consider doing this without sqlalchemy. This can get
     # slow with SQLalchemy (it's much worse with add_all).
     db_utils.bulk_save(trials)
@@ -75,22 +67,44 @@ class Experiment:
     def __init__(self, experiment_config_filepath: str):
         self.config = yaml_utils.read(experiment_config_filepath)
 
-        benchmarks = self.config['benchmarks'].split(',')
-        self.benchmarks = builder.build_all_measurers(benchmarks)
+        self.benchmarks = self.config['benchmarks'].split(',')
 
         self.fuzzers = [
             fuzzer_config_utils.get_fuzzer_name(filename) for filename in
             os.listdir(fuzzer_config_utils.get_fuzzer_configs_dir())
         ]
-
-        _initialize_experiment_in_db(self.config['experiment'],
-                                     self.config['git_hash'], self.benchmarks,
-                                     self.fuzzers, self.config['trials'])
+        self.num_trials = self.config['trials']
+        self.experiment_name = self.config['experiment']
+        self.git_hash = self.config['git_hash']
 
         self.web_bucket = posixpath.join(self.config['cloud_web_bucket'],
                                          experiment_utils.get_experiment_name())
 
 
+def build_images_for_trials(fuzzers: List[str], benchmarks: List[str],
+                            num_trials: int) -> List[models.Trial]:
+    """Builds the images needed to run |experiment| and returns a list of trials
+    that can be run for experiment. This is the number of trials specified in
+    experiment times each pair of fuzzer+benchmark that builds successfully."""
+    # This call will raise an exception if the images can't be built which will
+    # halt the experiment.
+    builder.build_base_images()
+
+    # Only build fuzzers for benchmarks whose measurers built successfully.
+    benchmarks = builder.build_all_measurers(benchmarks)
+    build_successes = builder.build_all_fuzzer_benchmarks(fuzzers, benchmarks)
+    experiment_name = experiment_utils.get_experiment_name()
+    trials = []
+    for fuzzer, benchmark in build_successes:
+        fuzzer_benchmark_trials = [
+            models.Trial(fuzzer=fuzzer,
+                         experiment=experiment_name,
+                         benchmark=benchmark) for _ in range(num_trials)
+        ]
+        trials.extend(fuzzer_benchmark_trials)
+    return trials
+
+
 def dispatcher_main():
     """Do the experiment and report results."""
     logs.info('Starting experiment.')
@@ -102,13 +116,13 @@ def dispatcher_main():
     if os.getenv('LOCAL_EXPERIMENT'):
         models.Base.metadata.create_all(db_utils.engine)
 
-    builder.build_base_images()
-
     experiment_config_file_path = os.path.join(fuzzer_config_utils.get_dir(),
                                                'experiment.yaml')
     experiment = Experiment(experiment_config_file_path)
-    builder.build_all_fuzzer_benchmarks(experiment.fuzzers,
-                                        experiment.benchmarks)
+    trials = build_images_for_trials(experiment.fuzzers, experiment.benchmarks,
+                                     experiment.num_trials)
+    _initialize_experiment_in_db(experiment.experiment_name,
+                                 experiment.git_hash, trials)
 
     create_work_subdirs(['experiment-folders', 'measurement-folders'])
 


Test patch:
diff --git a/experiment/test_dispatcher.py b/experiment/test_dispatcher.py
--- a/experiment/test_dispatcher.py
+++ b/experiment/test_dispatcher.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Tests for dispatcher.py."""
+import itertools
 import os
 from unittest import mock
 
@@ -27,7 +28,7 @@
 TEST_DATA_PATH = os.path.join(os.path.dirname(__file__), 'test_data')
 SANCOV_DIR = '/sancov'
 
-# pylint: disable=invalid-name,redefined-outer-name,unused-argument
+# pylint: disable=invalid-name,redefined-outer-name,unused-argument,protected-access
 
 
 def get_test_data_path(*subpaths):
@@ -51,6 +52,7 @@ def mock_split_successes_and_failures(inputs, results):
             mock_split_successes_and_failures)
 def dispatcher_experiment(fs, db, experiment):
     """Creates a dispatcher.Experiment object."""
+    fs.create_dir(os.environ['WORK'])
     experiment_config_filepath = get_test_data_path('experiment-config.yaml')
     fs.add_real_file(experiment_config_filepath)
     for fuzzer in FUZZERS:
@@ -67,6 +69,23 @@ def test_experiment(dispatcher_experiment):
     assert dispatcher_experiment.fuzzers == FUZZERS
     assert (
         dispatcher_experiment.web_bucket == 'gs://web-reports/test-experiment')
+
+
+def test_initialize_experiment_in_db(dispatcher_experiment):
+    """Tests that _initialize_experiment_in_db adds the right things to the
+    database."""
+    trials_args = itertools.product(dispatcher_experiment.benchmarks,
+                                    range(dispatcher_experiment.num_trials),
+                                    dispatcher_experiment.fuzzers)
+    trials = [
+        models.Trial(fuzzer=fuzzer,
+                     experiment=dispatcher_experiment.experiment_name,
+                     benchmark=benchmark)
+        for benchmark, _, fuzzer in trials_args
+    ]
+    dispatcher._initialize_experiment_in_db(
+        dispatcher_experiment.experiment_name, dispatcher_experiment.git_hash,
+        trials)
     db_experiments = db_utils.query(models.Experiment).all()
     assert len(db_experiments) == 1
     db_experiment = db_experiments[0]
@@ -78,3 +97,105 @@ def test_experiment(dispatcher_experiment):
                                       ('benchmark-1', 'fuzzer-b')] *
                                      4) + [('benchmark-2', 'fuzzer-a'),
                                            ('benchmark-2', 'fuzzer-b')] * 4
+
+
+@mock.patch('experiment.build.builder.build_base_images', side_effect=Exception)
+def test_build_images_for_trials_base_images_fail(dispatcher_experiment):
+    """Tests that build_for_trial raises an exception when base images can't be
+    built. This is important because the experiment should not proceed."""
+    with pytest.raises(Exception):
+        dispatcher.build_images_for_trials(dispatcher_experiment.fuzzers,
+                                           dispatcher_experiment.benchmarks,
+                                           dispatcher_experiment.num_trials)
+
+
+@mock.patch('experiment.build.builder.build_base_images')
+def test_build_images_for_trials_build_success(_, dispatcher_experiment):
+    """Tests that build_for_trial returns all trials we expect to run in an
+    experiment when builds are successful."""
+    fuzzer_benchmarks = list(
+        itertools.product(dispatcher_experiment.fuzzers,
+                          dispatcher_experiment.benchmarks))
+    with mock.patch('experiment.build.builder.build_all_measurers',
+                    return_value=dispatcher_experiment.benchmarks):
+        with mock.patch('experiment.build.builder.build_all_fuzzer_benchmarks',
+                        return_value=fuzzer_benchmarks):
+            trials = dispatcher.build_images_for_trials(
+                dispatcher_experiment.fuzzers, dispatcher_experiment.benchmarks,
+                dispatcher_experiment.num_trials)
+    trial_fuzzer_benchmarks = [
+        (trial.fuzzer, trial.benchmark) for trial in trials
+    ]
+    expected_trial_fuzzer_benchmarks = [
+        fuzzer_benchmark for fuzzer_benchmark in fuzzer_benchmarks
+        for _ in range(dispatcher_experiment.num_trials)
+    ]
+    assert (sorted(expected_trial_fuzzer_benchmarks) == sorted(
+        trial_fuzzer_benchmarks))
+
+
+@mock.patch('experiment.build.builder.build_base_images')
+def test_build_images_for_trials_benchmark_fail(_, dispatcher_experiment):
+    """Tests that build_for_trial doesn't return trials or try to build fuzzers
+    for a benchmark whose coverage build failed."""
+    successful_benchmark = 'benchmark-1'
+
+    def mocked_build_all_fuzzer_benchmarks(fuzzers, benchmarks):
+        assert benchmarks == [successful_benchmark]
+        return list(itertools.product(fuzzers, benchmarks))
+
+    with mock.patch('experiment.build.builder.build_all_measurers',
+                    return_value=[successful_benchmark]):
+        with mock.patch('experiment.build.builder.build_all_fuzzer_benchmarks',
+                        side_effect=mocked_build_all_fuzzer_benchmarks):
+            # Sanity check this test so that we know we are actually testing
+            # behavior when benchmarks fail.
+            assert len(set(dispatcher_experiment.benchmarks)) > 1
+            trials = dispatcher.build_images_for_trials(
+                dispatcher_experiment.fuzzers, dispatcher_experiment.benchmarks,
+                dispatcher_experiment.num_trials)
+    for trial in trials:
+        assert trial.benchmark == successful_benchmark
+
+
+@mock.patch('experiment.build.builder.build_base_images')
+def test_build_images_for_trials_fuzzer_fail(_, dispatcher_experiment):
+    """Tests that build_for_trial doesn't return trials a fuzzer whose build
+    failed on a benchmark."""
+    successful_fuzzer = 'fuzzer-a'
+    fail_fuzzer = 'fuzzer-b'
+    fuzzers = [successful_fuzzer, fail_fuzzer]
+    successful_benchmark_for_fail_fuzzer = 'benchmark-1'
+    fail_benchmark_for_fail_fuzzer = 'benchmark-2'
+    benchmarks = [
+        successful_benchmark_for_fail_fuzzer, fail_benchmark_for_fail_fuzzer
+    ]
+    successful_builds = [(successful_fuzzer, fail_benchmark_for_fail_fuzzer),
+                         (successful_fuzzer,
+                          successful_benchmark_for_fail_fuzzer),
+                         (fail_fuzzer, successful_benchmark_for_fail_fuzzer)]
+    num_trials = 10
+
+    def mocked_build_all_fuzzer_benchmarks(fuzzers, benchmarks):
+        # Sanity check this test so that we know we are actually testing
+        # behavior when fuzzers fail.
+        assert sorted(fuzzers) == sorted([successful_fuzzer, fail_fuzzer])
+        assert successful_benchmark_for_fail_fuzzer in benchmarks
+        return successful_builds
+
+    with mock.patch('experiment.build.builder.build_all_measurers',
+                    return_value=benchmarks):
+        with mock.patch('experiment.build.builder.build_all_fuzzer_benchmarks',
+                        side_effect=mocked_build_all_fuzzer_benchmarks):
+            trials = dispatcher.build_images_for_trials(fuzzers, benchmarks,
+                                                        num_trials)
+
+    trial_fuzzer_benchmarks = [
+        (trial.fuzzer, trial.benchmark) for trial in trials
+    ]
+    expected_trial_fuzzer_benchmarks = [
+        fuzzer_benchmark for fuzzer_benchmark in successful_builds
+        for _ in range(num_trials)
+    ]
+    assert (sorted(expected_trial_fuzzer_benchmarks) == sorted(
+        trial_fuzzer_benchmarks))