Repository: google/clusterfuzz

Issue:
libFuzzer stats on oss-fuzz not appearing after October 20th
https://oss-fuzz.com/fuzzer-stats?group_by=by-day&date_start=2022-10-12&date_end=2022-10-25&fuzzer=libFuzzer&job=libfuzzer_asan_skia&project=skia


Discussion:
Fuzzers are running, but it looks like the BQ import failed. 
This is the error we're getting:

```
"status": {
    "errorResult": {
      "reason": "resourcesExceeded",
      "debugInfo": "redacted"
    },
    "errors": [
      {
        "reason": "resourcesExceeded",
        "message": "Resources exceeded during query execution: The query could not be executed in the allotted memory. Peak usage: 101% of limit.\nTop memory consumer(s):\n  table/storage metadata: 100%\n"
      }
    ],
    "state": "DONE"
  },
```
We may need to create separate jobs here: https://github.com/google/clusterfuzz/blob/57073e10e76726c1e8a57ef75d3dadd80cf3d85e/src/appengine/handlers/cron/load_bigquery_stats.py#L148 

And partition manually by file prefix. 

Resolving patch:
diff --git a/src/appengine/handlers/cron/load_bigquery_stats.py b/src/appengine/handlers/cron/load_bigquery_stats.py
--- a/src/appengine/handlers/cron/load_bigquery_stats.py
+++ b/src/appengine/handlers/cron/load_bigquery_stats.py
@@ -13,8 +13,10 @@
 # limitations under the License.
 """Handler used for loading bigquery data."""
 
+from concurrent.futures import ThreadPoolExecutor
 import datetime
 import random
+import string
 import time
 
 from googleapiclient.errors import HttpError
@@ -31,8 +33,13 @@
 
 STATS_KINDS = [fuzzer_stats.JobRun, fuzzer_stats.TestcaseRun]
 
+NUM_THREADS = 8
 NUM_RETRIES = 2
 RETRY_SLEEP_TIME = 5
+POLL_INTERVAL = 5
+
+# Ignore the repeated uppercase digits.
+_HEX_DIGITS = string.hexdigits[:-6]
 
 
 class Handler(base_handler.Handler):
@@ -98,8 +105,20 @@ def _create_table_if_needed(self, bigquery, dataset_id, table_id, schema):
         projectId=project_id, datasetId=dataset_id, body=table_body)
     return self._execute_insert_request(table_insert)
 
-  def _load_data(self, bigquery, fuzzer):
+  def _poll_completion(self, bigquery, project_id, job_id):
+    """Poll for completion."""
+    response = bigquery.jobs().get(
+        projectId=project_id, jobId=job_id).execute(num_retries=NUM_RETRIES)
+    while response['status']['state'] == 'RUNNING':
+      response = bigquery.jobs().get(
+          projectId=project_id, jobId=job_id).execute(num_retries=NUM_RETRIES)
+      time.sleep(POLL_INTERVAL)
+
+    return response
+
+  def _load_data(self, fuzzer):
     """Load yesterday's stats into BigQuery."""
+    bigquery = big_query.get_api_client()
     project_id = utils.get_application_id()
 
     yesterday = (self._utc_now().date() - datetime.timedelta(days=1))
@@ -124,35 +143,48 @@ def _load_data(self, bigquery, fuzzer):
         continue
 
       gcs_path = fuzzer_stats.get_gcs_stats_path(kind_name, fuzzer, timestamp)
-      load = {
-          'destinationTable': {
-              'projectId': project_id,
-              'tableId': table_id + '$' + date_string,
-              'datasetId': dataset_id,
-          },
-          'schemaUpdateOptions': ['ALLOW_FIELD_ADDITION',],
-          'sourceFormat': 'NEWLINE_DELIMITED_JSON',
-          'sourceUris': ['gs:/' + gcs_path + '*.json'],
-          'writeDisposition': 'WRITE_TRUNCATE',
-      }
-      if schema is not None:
-        load['schema'] = schema
-
-      job_body = {
-          'configuration': {
-              'load': load,
-          },
-      }
-
-      logs.log("Uploading job to BigQuery.", job_body=job_body)
-      request = bigquery.jobs().insert(projectId=project_id, body=job_body)
-      response = request.execute()
-
-      # We cannot really check the response here, as the query might be still
-      # running, but having a BigQuery jobId in the log would make our life
-      # simpler if we ever have to manually check the status of the query.
-      # See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query.
-      logs.log('Response from BigQuery: %s' % response)
+      # Shard loads by prefix to avoid causing BigQuery to run out of memory.
+      for i, prefix in enumerate(_HEX_DIGITS):
+        load = {
+            'destinationTable': {
+                'projectId': project_id,
+                'tableId': table_id + '$' + date_string,
+                'datasetId': dataset_id,
+            },
+            'schemaUpdateOptions': ['ALLOW_FIELD_ADDITION',],
+            'sourceFormat': 'NEWLINE_DELIMITED_JSON',
+            'sourceUris': ['gs:/' + gcs_path + prefix + '*.json'],
+            # Truncate on the first shard, then append the rest.
+            'writeDisposition': 'WRITE_TRUNCATE' if i == 0 else 'WRITE_APPEND',
+        }
+        if schema is not None:
+          load['schema'] = schema
+
+        job_body = {
+            'configuration': {
+                'load': load,
+            },
+        }
+
+        try:
+          logs.log("Uploading job to BigQuery.", job_body=job_body)
+
+          request = bigquery.jobs().insert(projectId=project_id, body=job_body)
+          load_response = request.execute(num_retries=NUM_RETRIES)
+          job_id = load_response['jobReference']['jobId']
+          logs.log(f'Load job id: {job_id}')
+
+          response = self._poll_completion(bigquery, project_id, job_id)
+          logs.log('Completed load: %s' % response)
+          errors = response['status'].get('errors')
+          if errors:
+            logs.log_error(
+                f'Failed load for {job_id} with errors: {str(errors)})')
+        except Exception:
+          # Log exception here as otherwise it gets lost in the thread pool
+          # worker.
+          logs.log_error('Failed to load.')
+          return
 
   @handler.cron()
   def get(self):
@@ -161,9 +193,12 @@ def get(self):
       logs.log_error('Loading stats to BigQuery failed: missing bucket name.')
       return
 
+    thread_pool = ThreadPoolExecutor(max_workers=NUM_THREADS)
+
     # Retrieve list of fuzzers before iterating them, since the query can expire
     # as we create the load jobs.
-    bigquery_client = big_query.get_api_client()
     for fuzzer in list(data_types.Fuzzer.query()):
       logs.log('Loading stats to BigQuery for %s.' % fuzzer.name)
-      self._load_data(bigquery_client, fuzzer.name)
+      thread_pool.submit(self._load_data, fuzzer.name)
+
+    thread_pool.shutdown(wait=True)


Test patch:
diff --git a/src/clusterfuzz/_internal/tests/appengine/handlers/cron/load_bigquery_stats_test.py b/src/clusterfuzz/_internal/tests/appengine/handlers/cron/load_bigquery_stats_test.py
--- a/src/clusterfuzz/_internal/tests/appengine/handlers/cron/load_bigquery_stats_test.py
+++ b/src/clusterfuzz/_internal/tests/appengine/handlers/cron/load_bigquery_stats_test.py
@@ -98,118 +98,119 @@ def test_execute(self):
         mock.call().execute(),
     ])
 
-    self.mock_bigquery.jobs().insert.assert_has_calls(
-        [
-            mock.call(
-                body={
-                    'configuration': {
-                        'load': {
-                            'destinationTable': {
-                                'projectId': 'test-clusterfuzz',
-                                'tableId': 'JobRun$20160907',
-                                'datasetId': 'fuzzer_stats'
-                            },
-                            'schemaUpdateOptions': ['ALLOW_FIELD_ADDITION',],
-                            'writeDisposition':
-                                'WRITE_TRUNCATE',
-                            'sourceUris': [
-                                'gs://test-bigquery-bucket/fuzzer/JobRun/date/'
-                                '20160907/*.json'
-                            ],
-                            'sourceFormat':
-                                'NEWLINE_DELIMITED_JSON',
-                            'schema': {
-                                'fields': [{
-                                    'type': 'INTEGER',
-                                    'name': 'testcases_executed',
-                                    'mode': 'NULLABLE'
-                                }, {
-                                    'type': 'INTEGER',
-                                    'name': 'build_revision',
-                                    'mode': 'NULLABLE'
-                                }, {
-                                    'type': 'INTEGER',
-                                    'name': 'new_crashes',
-                                    'mode': 'NULLABLE'
-                                }, {
-                                    'type': 'STRING',
-                                    'name': 'job',
-                                    'mode': 'NULLABLE'
-                                }, {
-                                    'type': 'FLOAT',
-                                    'name': 'timestamp',
-                                    'mode': 'NULLABLE'
-                                }, {
-                                    'fields': [{
-                                        'type': 'STRING',
-                                        'name': 'crash_type',
-                                        'mode': 'NULLABLE'
-                                    }, {
-                                        'type': 'BOOLEAN',
-                                        'name': 'is_new',
-                                        'mode': 'NULLABLE'
-                                    }, {
-                                        'type': 'STRING',
-                                        'name': 'crash_state',
-                                        'mode': 'NULLABLE'
-                                    }, {
-                                        'type': 'BOOLEAN',
-                                        'name': 'security_flag',
-                                        'mode': 'NULLABLE'
-                                    }, {
-                                        'type': 'INTEGER',
-                                        'name': 'count',
-                                        'mode': 'NULLABLE'
-                                    }],
-                                    'type':
-                                        'RECORD',
-                                    'name':
-                                        'crashes',
-                                    'mode':
-                                        'REPEATED'
-                                }, {
-                                    'type': 'INTEGER',
-                                    'name': 'known_crashes',
-                                    'mode': 'NULLABLE'
-                                }, {
-                                    'type': 'STRING',
-                                    'name': 'fuzzer',
-                                    'mode': 'NULLABLE'
-                                }, {
-                                    'type': 'STRING',
-                                    'name': 'kind',
-                                    'mode': 'NULLABLE'
-                                }]
-                            },
-                        }
-                    }
-                },
-                projectId='test-clusterfuzz'),
-            mock.call().execute(),
-            mock.call(
-                body={
-                    'configuration': {
-                        'load': {
-                            'destinationTable': {
-                                'projectId': 'test-clusterfuzz',
-                                'tableId': 'TestcaseRun$20160907',
-                                'datasetId': 'fuzzer_stats'
-                            },
-                            'schemaUpdateOptions': ['ALLOW_FIELD_ADDITION',],
-                            'writeDisposition':
-                                'WRITE_TRUNCATE',
-                            'sourceUris': [
-                                'gs://test-bigquery-bucket/fuzzer/TestcaseRun/'
-                                'date/20160907/*.json'
-                            ],
-                            'sourceFormat':
-                                'NEWLINE_DELIMITED_JSON',
-                        }
-                    }
-                },
-                projectId='test-clusterfuzz'),
-            mock.call().execute(),
-        ],
-        # Otherwise we need to mock two calls to mock.call().execute().__str__()
-        # which does not seem to work well.
-        any_order=True)
+    for i, prefix in enumerate(load_bigquery_stats._HEX_DIGITS):  # pylint: disable=protected-access
+      self.mock_bigquery.jobs().insert.assert_has_calls(
+          [
+              mock.call(
+                  body={
+                      'configuration': {
+                          'load': {
+                              'destinationTable': {
+                                  'projectId': 'test-clusterfuzz',
+                                  'tableId': 'JobRun$20160907',
+                                  'datasetId': 'fuzzer_stats'
+                              },
+                              'schemaUpdateOptions': ['ALLOW_FIELD_ADDITION',],
+                              'writeDisposition':
+                                  'WRITE_TRUNCATE'
+                                  if i == 0 else 'WRITE_APPEND',
+                              'sourceUris': [
+                                  'gs://test-bigquery-bucket/fuzzer/JobRun/date/'
+                                  '20160907/' + prefix + '*.json'
+                              ],
+                              'sourceFormat':
+                                  'NEWLINE_DELIMITED_JSON',
+                              'schema': {
+                                  'fields': [{
+                                      'type': 'INTEGER',
+                                      'name': 'testcases_executed',
+                                      'mode': 'NULLABLE'
+                                  }, {
+                                      'type': 'INTEGER',
+                                      'name': 'build_revision',
+                                      'mode': 'NULLABLE'
+                                  }, {
+                                      'type': 'INTEGER',
+                                      'name': 'new_crashes',
+                                      'mode': 'NULLABLE'
+                                  }, {
+                                      'type': 'STRING',
+                                      'name': 'job',
+                                      'mode': 'NULLABLE'
+                                  }, {
+                                      'type': 'FLOAT',
+                                      'name': 'timestamp',
+                                      'mode': 'NULLABLE'
+                                  }, {
+                                      'fields': [{
+                                          'type': 'STRING',
+                                          'name': 'crash_type',
+                                          'mode': 'NULLABLE'
+                                      }, {
+                                          'type': 'BOOLEAN',
+                                          'name': 'is_new',
+                                          'mode': 'NULLABLE'
+                                      }, {
+                                          'type': 'STRING',
+                                          'name': 'crash_state',
+                                          'mode': 'NULLABLE'
+                                      }, {
+                                          'type': 'BOOLEAN',
+                                          'name': 'security_flag',
+                                          'mode': 'NULLABLE'
+                                      }, {
+                                          'type': 'INTEGER',
+                                          'name': 'count',
+                                          'mode': 'NULLABLE'
+                                      }],
+                                      'type':
+                                          'RECORD',
+                                      'name':
+                                          'crashes',
+                                      'mode':
+                                          'REPEATED'
+                                  }, {
+                                      'type': 'INTEGER',
+                                      'name': 'known_crashes',
+                                      'mode': 'NULLABLE'
+                                  }, {
+                                      'type': 'STRING',
+                                      'name': 'fuzzer',
+                                      'mode': 'NULLABLE'
+                                  }, {
+                                      'type': 'STRING',
+                                      'name': 'kind',
+                                      'mode': 'NULLABLE'
+                                  }]
+                              },
+                          }
+                      }
+                  },
+                  projectId='test-clusterfuzz'),
+              mock.call(
+                  body={
+                      'configuration': {
+                          'load': {
+                              'destinationTable': {
+                                  'projectId': 'test-clusterfuzz',
+                                  'tableId': 'TestcaseRun$20160907',
+                                  'datasetId': 'fuzzer_stats'
+                              },
+                              'schemaUpdateOptions': ['ALLOW_FIELD_ADDITION',],
+                              'writeDisposition':
+                                  'WRITE_TRUNCATE'
+                                  if i == 0 else 'WRITE_APPEND',
+                              'sourceUris': [
+                                  'gs://test-bigquery-bucket/fuzzer/TestcaseRun/'
+                                  'date/20160907/' + prefix + '*.json'
+                              ],
+                              'sourceFormat':
+                                  'NEWLINE_DELIMITED_JSON',
+                          }
+                      }
+                  },
+                  projectId='test-clusterfuzz'),
+          ],
+          # Otherwise we need to mock two calls to mock.call().execute().__str__()
+          # which does not seem to work well.
+          any_order=True)