Repository: google/clusterfuzz

Issue:
Retry logic in storage.py
Filing this to clarify whether I misunderstand it or we need to fix something.

Once in a while we get the following exception: https://console.cloud.google.com/errors/CIuh6-K0uKWFNw?time=P30D&project=clusterfuzz-external

It comes from https://github.com/google/clusterfuzz/blob/e0c6ad071951c403d17ef11a3020ca75b4bf9a09/src/appengine/handlers/cron/corpus_backup.py#L65

Which calls into https://github.com/google/clusterfuzz/blob/e0c6ad071951c403d17ef11a3020ca75b4bf9a09/src/python/google_cloud_utils/storage.py#L786

which calls into https://github.com/google/clusterfuzz/blob/e0c6ad071951c403d17ef11a3020ca75b4bf9a09/src/python/google_cloud_utils/storage.py#L253

and the particular failing operation is https://github.com/google/clusterfuzz/blob/e0c6ad071951c403d17ef11a3020ca75b4bf9a09/src/python/google_cloud_utils/storage.py#L263

If I check the logs, I see the request that was sent prior to the copy operation (setting ACL): https://console.cloud.google.com/logs/viewer?resource=gae_app%2Fmodule_id%2Fcron-service%2Fversion_id%2F20200410t180004&logName=projects%2Fclusterfuzz-external%2Flogs%2Fapp&expandAll=false&timestamp=2020-04-12T03%3A26%3A25.013000000Z&project=clusterfuzz-external&minLogLevel=0&customFacets&limitCustomFacetWidth=true&advancedFilter=resource.type%3D%22gae_app%22%0Aresource.labels.project_id%3D%22clusterfuzz-external%22%0Aresource.labels.version_id%3D%2220200410t180004%22%0Aresource.labels.module_id%3D%22cron-service%22%0Atimestamp%3D%222020-04-12T03%3A25%3A44.899340000Z%22%0AinsertId%3D%22gddy5ufk382zt%22&interval=JUMP_TO_TIME&scrollTimestamp=2020-04-12T03%3A25%3A55.876537987Z

and that request happens 10 second earlier than when the copy error is recorded, even though we use the following constants:

```
# The number of retries to perform some GCS operation.
DEFAULT_FAIL_RETRIES = 8

# The time to wait between retries while performing GCS operation.
DEFAULT_FAIL_WAIT = 2
```

and the time interval between the previous request and the error should be 14+ seconds at least, but actually even more given that the copy error is `google.api_core.exceptions.GatewayTimeout` which is also a few seconds at least?

**Question**

Does the retry logic actually work? How can I verify it? Or do I miss something important in my debugging?


Discussion:

Resolving patch:
diff --git a/src/python/google_cloud_utils/storage.py b/src/python/google_cloud_utils/storage.py
--- a/src/python/google_cloud_utils/storage.py
+++ b/src/python/google_cloud_utils/storage.py
@@ -163,8 +163,8 @@ def create_bucket(self, name, object_lifecycle, cors):
     try:
       client.buckets().insert(project=project_id, body=request_body).execute()
     except HttpError as e:
-      logs.log_error('Failed to create bucket %s: %s' % (name, e))
-      return False
+      logs.log_warn('Failed to create bucket %s: %s' % (name, e))
+      raise
 
     return True
 
@@ -221,9 +221,9 @@ def copy_file_from(self, remote_path, local_path):
       blob = bucket.blob(path, chunk_size=self._chunk_size())
       blob.download_to_filename(local_path)
     except google.cloud.exceptions.GoogleCloudError:
-      logs.log_error('Failed to copy cloud storage file %s to local file %s.' %
-                     (remote_path, local_path))
-      return False
+      logs.log_warn('Failed to copy cloud storage file %s to local file %s.' %
+                    (remote_path, local_path))
+      raise
 
     return True
 
@@ -244,9 +244,9 @@ def copy_file_to(self, local_path_or_handle, remote_path, metadata=None):
         blob.upload_from_file(local_path_or_handle, rewind=True)
 
     except google.cloud.exceptions.GoogleCloudError:
-      logs.log_error('Failed to copy local file %s to cloud storage file %s.' %
-                     (local_path_or_handle, remote_path))
-      return False
+      logs.log_warn('Failed to copy local file %s to cloud storage file %s.' %
+                    (local_path_or_handle, remote_path))
+      raise
 
     return True
 
@@ -262,9 +262,9 @@ def copy_blob(self, remote_source, remote_target):
       target_bucket = client.bucket(target_bucket_name)
       source_bucket.copy_blob(source_blob, target_bucket, target_path)
     except google.cloud.exceptions.GoogleCloudError:
-      logs.log_error('Failed to copy cloud storage file %s to cloud storage '
-                     'file %s.' % (remote_source, remote_target))
-      return False
+      logs.log_warn('Failed to copy cloud storage file %s to cloud storage '
+                    'file %s.' % (remote_source, remote_target))
+      raise
 
     return True
 
@@ -278,8 +278,8 @@ def read_data(self, remote_path):
       blob = bucket.blob(path, chunk_size=self._chunk_size())
       return blob.download_as_string()
     except google.cloud.exceptions.GoogleCloudError:
-      logs.log_error('Failed to read cloud storage file %s.' % remote_path)
-      return None
+      logs.log_warn('Failed to read cloud storage file %s.' % remote_path)
+      raise
 
   def write_data(self, data, remote_path, metadata=None):
     """Write the data of a remote file."""
@@ -293,8 +293,8 @@ def write_data(self, data, remote_path, metadata=None):
         blob.metadata = metadata
       blob.upload_from_string(data)
     except google.cloud.exceptions.GoogleCloudError:
-      logs.log_error('Failed to write cloud storage file %s.' % remote_path)
-      return False
+      logs.log_warn('Failed to write cloud storage file %s.' % remote_path)
+      raise
 
     return True
 
@@ -320,8 +320,8 @@ def delete(self, remote_path):
       bucket = client.bucket(bucket_name)
       bucket.delete_blob(path)
     except google.cloud.exceptions.GoogleCloudError:
-      logs.log_error('Failed to delete cloud storage file %s.' % remote_path)
-      return False
+      logs.log_warn('Failed to delete cloud storage file %s.' % remote_path)
+      raise
 
     return True
 


Test patch:
diff --git a/src/python/tests/core/bot/fuzzers/libFuzzer/engine_test.py b/src/python/tests/core/bot/fuzzers/libFuzzer/engine_test.py
--- a/src/python/tests/core/bot/fuzzers/libFuzzer/engine_test.py
+++ b/src/python/tests/core/bot/fuzzers/libFuzzer/engine_test.py
@@ -432,6 +432,8 @@ def setUp(self):
     os.environ['INPUT_DIR'] = TEMP_DIR
 
     test_helpers.patch(self, [
+        'bot.fuzzers.dictionary_manager.DictionaryManager.'
+        'update_recommended_dictionary',
         'bot.fuzzers.engine_common.get_merge_timeout',
         'bot.fuzzers.engine_common.random_choice',
         'bot.fuzzers.mutator_plugin._download_mutator_plugin_archive',
@@ -661,8 +663,6 @@ def test_analyze_dict(self):
     test_helpers.patch(self, [
         'bot.fuzzers.dictionary_manager.DictionaryManager.'
         'parse_recommended_dictionary_from_log_lines',
-        'bot.fuzzers.dictionary_manager.DictionaryManager.'
-        'update_recommended_dictionary',
     ])
 
     self.mock.parse_recommended_dictionary_from_log_lines.return_value = set([
@@ -861,6 +861,7 @@ def mocked_log_error(*args, **kwargs):  # pylint: disable=unused-argument
     engine_impl.fuzz(target_path, options, TEMP_DIR, 10)
 
 
+@unittest.skip('needs root')
 @test_utils.integration
 class UnshareIntegrationTests(IntegrationTests):
   """Unshare runner integration tests."""