Repository: google/fuzzbench

Issue:
master cannot pass "make presubmit"
Current master branch has one failed at "``make presubmit``".

To reproduce: ``git clone https://github.com/google/fuzzbench ; cd fuzzbench ; git submodule update --init ; make install-dependencies ; source ./vene/bin/activate ; make presubmit``

https://github.com/google/fuzzbench/blob/41aa9b4267907185bd51514a6fb335ff70f4990d/experiment/test_measurer.py#L347

E            assert 0 == 3798

Reference: https://google.github.io/fuzzbench/getting-started/prerequisites/


Discussion:
Can you post the output. It doens't reproduce on my desktop (the same one I use for developing).
Ah OK it makes sense why this test would fail. I'll disable it by default.
It seems relate to the TODO: https://github.com/google/fuzzbench/blob/41aa9b4267907185bd51514a6fb335ff70f4990d/experiment/test_measurer.py#L340
I suspected something else. Could you run this command and post the output?
`./fuzzbench/experiment/test_data/test_measure_snapshot_coverage/freetype2-2017-coverage`
My guess is that this test relies on running a binary that won't run on every machine where make install-deps succeeds.
./fuzzbench/experiment/test_data/test_measure_snapshot_coverage/freetype2-2017-coverage: error while loading shared libraries: libarchive.so.13: cannot open shared object file: No such file or directory

`sudo apt-get install libarchive13` will solve this issue. Thanks for double checking.

Resolving patch:
diff --git a/conftest.py b/conftest.py
--- a/conftest.py
+++ b/conftest.py
@@ -73,9 +73,9 @@ def environ():
 
 
 @pytest.fixture
-def experiment(tmp_path, environ):  # pylint: disable=redefined-outer-name,unused-argument
+def experiment(environ):  # pylint: disable=redefined-outer-name,unused-argument
     """Mock an experiment."""
-    os.environ['WORK'] = str(tmp_path)
+    os.environ['WORK'] = '/work'
     os.environ['EXPERIMENT'] = 'test-experiment'
     os.environ['CLOUD_EXPERIMENT_BUCKET'] = 'gs://experiment-data'
     os.environ['CLOUD_WEB_BUCKET'] = 'gs://web-bucket'


Test patch:
diff --git a/common/test_filesystem.py b/common/test_filesystem.py
--- a/common/test_filesystem.py
+++ b/common/test_filesystem.py
@@ -25,10 +25,10 @@
 # pylint: disable=invalid-name,unused-argument
 
 
-def test_recreate_directory_existing(tmp_path):
+def test_recreate_directory_existing(fs):
     """Tests that recreate_directory recreates a directory that already
     exists."""
-    new_directory = os.path.join(tmp_path, 'new-directory')
+    new_directory = 'new-directory'
     os.mkdir(new_directory)
     new_file = os.path.join(new_directory, 'file')
     with open(new_file, 'w') as file_handle:
@@ -39,10 +39,10 @@ def test_recreate_directory_existing(tmp_path):
     assert not os.path.exists(new_file)
 
 
-def test_recreate_directory_not_existing(tmp_path):
+def test_recreate_directory_not_existing(fs):
     """Tests that recreate_directory creates a directory that does not already
     exist."""
-    new_directory = os.path.join(tmp_path, 'new-directory')
+    new_directory = 'new-directory'
     filesystem.recreate_directory(new_directory)
     assert os.path.exists(new_directory)
 
diff --git a/experiment/build/test_builder.py b/experiment/build/test_builder.py
--- a/experiment/build/test_builder.py
+++ b/experiment/build/test_builder.py
@@ -19,6 +19,7 @@
 
 import pytest
 
+from common import utils
 from experiment.build import builder
 
 SRC_ROOT = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
@@ -65,9 +66,10 @@ def get_benchmarks_or_fuzzers(benchmarks_or_fuzzers_directory, filename,
 @mock.patch('time.sleep')
 @pytest.mark.parametrize('build_measurer_return_value', [True, False])
 def test_build_all_measurers(_, mocked_build_measurer,
-                             build_measurer_return_value, experiment):
+                             build_measurer_return_value, experiment, fs):
     """Tests that build_all_measurers works as intendend when build_measurer
     calls fail."""
+    fs.add_real_directory(utils.ROOT_DIR)
     mocked_build_measurer.return_value = build_measurer_return_value
     benchmarks = get_regular_benchmarks()
     result = builder.build_all_measurers(benchmarks)
diff --git a/experiment/test_measurer.py b/experiment/test_measurer.py
--- a/experiment/test_measurer.py
+++ b/experiment/test_measurer.py
@@ -280,28 +280,23 @@ def get_test_data_path(*subpaths):
     return os.path.join(TEST_DATA_PATH, *subpaths)
 
 
-@pytest.fixture
-def create_measurer(experiment):
-    """Fixture that provides a function for creating a SnapshotMeasurer."""
-    os.mkdir(build_utils.get_coverage_binaries_dir())
-
-    def _create_measurer(fuzzer, benchmark, trial_num):
-        return measurer.SnapshotMeasurer(fuzzer, benchmark, trial_num,
-                                         SNAPSHOT_LOGGER)
-
-    yield _create_measurer
-
-
 # pylint: disable=no-self-use
 
 
 class TestIntegrationMeasurement:
     """Integration tests for measurement."""
 
+    # TODO(metzman): Get this test working everywhere by using docker or a more
+    # portable binary.
+    @pytest.mark.skipif(not os.getenv('FUZZBENCH_TEST_INTEGRATION'),
+                        reason='Not running integration tests.')
     @mock.patch('experiment.measurer.SnapshotMeasurer.is_cycle_unchanged')
     def test_measure_snapshot_coverage(  # pylint: disable=too-many-locals
-            self, mocked_is_cycle_unchanged, create_measurer, db, experiment):
+            self, mocked_is_cycle_unchanged, db, experiment, tmp_path):
         """Integration test for measure_snapshot_coverage."""
+        # WORK is set by experiment to a directory that only makes sense in a
+        # fakefs.
+        os.environ['WORK'] = str(tmp_path)
         mocked_is_cycle_unchanged.return_value = False
         # Set up the coverage binary.
         benchmark = 'freetype2-2017'
@@ -324,8 +319,9 @@ def test_measure_snapshot_coverage(  # pylint: disable=too-many-locals
                              experiment=os.environ['EXPERIMENT'])
         db_utils.add_all([trial])
 
-        snapshot_measurer = create_measurer(trial.fuzzer, trial.benchmark,
-                                            trial.id)
+        snapshot_measurer = measurer.SnapshotMeasurer(trial.fuzzer,
+                                                      trial.benchmark, trial.id,
+                                                      SNAPSHOT_LOGGER)
 
         # Set up the snapshot archive.
         cycle = 1
diff --git a/experiment/test_run_coverage.py b/experiment/test_run_coverage.py
--- a/experiment/test_run_coverage.py
+++ b/experiment/test_run_coverage.py
@@ -31,14 +31,14 @@ def _get_test_data_dir(directory):
 
 
 def _make_crashes_dir(parent_path):
-    """Makes a crashes dir in |tmp_path| and returns it."""
+    """Makes a crashes dir in |parent_path| and returns it."""
     crashes_dir = os.path.join(str(parent_path), 'crashes')
     os.mkdir(crashes_dir)
     return crashes_dir
 
 
 def _make_sancov_dir(parent_path):
-    """Makes a sancov dir in |tmp_path| and returns it."""
+    """Makes a sancov dir in |parent_path| and returns it."""
     sancov_dir = os.path.join(str(parent_path), 'sancov')
     os.mkdir(sancov_dir)
     return sancov_dir
diff --git a/experiment/test_runner.py b/experiment/test_runner.py
--- a/experiment/test_runner.py
+++ b/experiment/test_runner.py
@@ -298,10 +298,10 @@ def test_integration_runner(self, mocked_error, tmp_path, environ):
         mocked_error.assert_not_called()
 
 
-def test_clean_seed_corpus(tmp_path, fs):
+def test_clean_seed_corpus(fs):
     """Test that seed corpus files are moved to root directory and deletes files
     exceeding 1 MB limit."""
-    seed_corpus_dir = tmp_path / 'seeds'
+    seed_corpus_dir = '/seeds'
     fs.create_dir(seed_corpus_dir)
 
     fs.create_file(os.path.join(seed_corpus_dir, 'a', 'abc'), contents='abc')