Repository: google/clusterfuzz

Issue:
Implement proper corpus eviction.
Right now, the only way we deal with our corpora filling up disk is to call shell.clear_data_directories and exit with a log_error.

For cases where there are many targets which can run on a single bot, we reach this state easily. We should have a non-fatal way of handling this by cleaning up corpora we don't need.


Discussion:
Actually there is another related issue here -- we don't clear FUZZ_DATA. For some deployments the libFuzzer and AFL entities won't have a data bundle set so it'll default to FUZZ_DATA. This leads to a buildup that's never cleaned up properly.
As per Oliver, not a priority now since he fixed the FUZZ_DATA issue in https://github.com/google/clusterfuzz/pull/896
Per #1543 this issue is coming back so we should fix this.
We should probably also take the chance to clean up FUZZ_DATA logic -- it makes cleaning it difficult as web tests can live there. 
Re corpora filling up disk, maybe we can even do a rolling model where we store at most N corpora on disk. When we try to set up the N+1'th corpora, we clear the oldest. 

For small OSS-Fuzz projects with a small number of targets, this would ensure we don't waste time constantly re-syncing corpora. For larger deployments with hundreds/thousands, this would stop the buildup. 

Resolving patch:
diff --git a/src/python/bot/tasks/fuzz_task.py b/src/python/bot/tasks/fuzz_task.py
--- a/src/python/bot/tasks/fuzz_task.py
+++ b/src/python/bot/tasks/fuzz_task.py
@@ -504,7 +504,7 @@ def sync_from_gcs(self):
 
     # Check if the corpus was recently synced. If yes, set a flag so that we
     # don't sync it again and save some time.
-    if last_sync_time:
+    if last_sync_time and os.path.exists(self._corpus_directory):
       last_update_time = storage.last_updated(self.gcs_corpus.get_gcs_url())
       if last_update_time and last_sync_time > last_update_time:
         logs.log('Corpus for target %s has no new updates, skipping rsync.' %
diff --git a/src/python/bot/tasks/setup.py b/src/python/bot/tasks/setup.py
--- a/src/python/bot/tasks/setup.py
+++ b/src/python/bot/tasks/setup.py
@@ -45,6 +45,7 @@
 from system import shell
 
 _BOT_DIR = 'bot'
+_DATA_BUNDLE_CACHE_COUNT = 10
 _DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS = 6 * 60 * 60
 _DATA_BUNDLE_LOCK_INTERVAL_IN_SECONDS = 3 * 60 * 60
 _SYNC_FILENAME = '.sync'
@@ -356,6 +357,27 @@ def _fetch_lock_for_data_bundle_update(data_bundle):
       by_zone=True)
 
 
+def _clear_old_data_bundles_if_needed():
+  """Clear old data bundles so as to keep the disk cache restricted to
+  |_DATA_BUNDLE_CACHE_COUNT| data bundles and prevent potential out-of-disk
+  spaces."""
+  data_bundles_directory = environment.get_value('DATA_BUNDLES_DIR')
+
+  dirs = []
+  for filename in os.listdir(data_bundles_directory):
+    file_path = os.path.join(data_bundles_directory, filename)
+    if not os.path.isdir(file_path):
+      continue
+    dirs.append(file_path)
+
+  dirs_to_remove = sorted(
+      dirs, key=os.path.getmtime, reverse=True)[_DATA_BUNDLE_CACHE_COUNT:]
+  for dir_to_remove in dirs_to_remove:
+    logs.log('Removing data bundle directory to keep disk cache small: %s' %
+             dir_to_remove)
+    shell.remove_directory(dir_to_remove)
+
+
 def update_data_bundle(fuzzer, data_bundle):
   """Updates a data bundle to the latest version."""
   # This module can't be in the global imports due to appengine issues
@@ -513,6 +535,8 @@ def update_fuzzer_and_data_bundles(fuzzer_name):
     revisions.write_revision_to_revision_file(version_file, fuzzer.revision)
     logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)
 
+  _clear_old_data_bundles_if_needed()
+
   # Setup data bundles associated with this fuzzer.
   data_bundles = ndb_utils.get_all_from_query(
       data_types.DataBundle.query(


Test patch:
diff --git a/src/python/tests/core/bot/fuzzers/builtin_fuzzers_test.py b/src/python/tests/core/bot/fuzzers/builtin_fuzzers_test.py
--- a/src/python/tests/core/bot/fuzzers/builtin_fuzzers_test.py
+++ b/src/python/tests/core/bot/fuzzers/builtin_fuzzers_test.py
@@ -61,6 +61,8 @@ def setUp(self):
     test_utils.set_up_pyfakefs(self)
     self.fs.create_dir('/input')
     self.fs.create_dir('/output')
+    self.fs.create_dir('/data-bundles')
+    environment.set_value('DATA_BUNDLES_DIR', '/data-bundles')
 
     helpers.patch(self, [
         'bot.fuzzers.libFuzzer.fuzzer.LibFuzzer.run',
diff --git a/src/python/tests/core/bot/tasks/setup_test.py b/src/python/tests/core/bot/tasks/setup_test.py
--- a/src/python/tests/core/bot/tasks/setup_test.py
+++ b/src/python/tests/core/bot/tasks/setup_test.py
@@ -12,9 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Tests for setup."""
+from builtins import range
+from builtins import str
 
+import os
 import unittest
 
+from pyfakefs import fake_filesystem_unittest
+
 from bot.tasks import setup
 from datastore import data_types
 from system import environment
@@ -154,3 +159,35 @@ def test_afl_job_for_variant_task(self):
         '%TESTCASE%',
         setup._get_application_arguments(self.testcase,
                                          'afl_asan_chrome_variant', 'variant'))
+
+
+# pylint: disable=protected-access
+class ClearOldDataBundlesIfNeededTest(fake_filesystem_unittest.TestCase):
+  """Tests _clear_old_data_bundles_if_needed."""
+
+  def setUp(self):
+    test_utils.set_up_pyfakefs(self)
+    helpers.patch_environ(self)
+
+    self.data_bundles_dir = '/data-bundles'
+    os.mkdir(self.data_bundles_dir)
+    environment.set_value('DATA_BUNDLES_DIR', self.data_bundles_dir)
+
+  def test_evict(self):
+    """Tests that eviction works when more than certain number of bundles."""
+    for i in range(1, 15):
+      os.mkdir(os.path.join(self.data_bundles_dir, str(i)))
+
+    setup._clear_old_data_bundles_if_needed()
+    self.assertEqual([str(i) for i in range(5, 15)],
+                     sorted(os.listdir(self.data_bundles_dir), key=int))
+
+  def test_no_evict(self):
+    """Tests that no eviction is required when less than certain number of
+    bundles."""
+    for i in range(1, 5):
+      os.mkdir(os.path.join(self.data_bundles_dir, str(i)))
+
+    setup._clear_old_data_bundles_if_needed()
+    self.assertEqual([str(i) for i in range(1, 5)],
+                     sorted(os.listdir(self.data_bundles_dir), key=int))