Repository: google/osv.dev

Issue:
More heuristics in case of bisection failure (for OSS-Fuzz)
Some times project maintainers leave references to OSS-Fuzz bug or testcase IDs in the commit message. We can use them. 


Discussion:

Resolving patch:
diff --git a/docker/worker/worker.py b/docker/worker/worker.py
--- a/docker/worker/worker.py
+++ b/docker/worker/worker.py
@@ -19,6 +19,7 @@
 import logging
 import os
 import math
+import re
 import resource
 import shutil
 import subprocess
@@ -42,6 +43,7 @@
 MAX_LEASE_DURATION = 6 * 60 * 60  # 4 hours.
 
 OSS_FUZZ_ISSUE_URL = 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id='
+OSS_FUZZ_SOURCE_PREFIX = 'oss-fuzz:'
 
 # Large projects which take way too long to build.
 # TODO(ochang): Don't hardcode this.
@@ -294,6 +296,59 @@ def _set_result_attributes(oss_fuzz_dir, message, entity):
     entity.timestamp = timestamp
 
 
+def find_oss_fuzz_fix_via_commit(repo, start_commit, end_commit, source_id,
+                                 issue_id):
+  """Find fix commit by checking commit messages."""
+  if not source_id.startswith(OSS_FUZZ_SOURCE_PREFIX):
+    return None
+
+  # Walk through start_commit..end_commit
+  try:
+    walker = repo.walk(end_commit, pygit2.GIT_SORT_TOPOLOGICAL)
+  except KeyError:
+    logging.error('Failed to walk repo with invalid commit: %s', end_commit)
+    return None
+
+  walker.hide(start_commit)
+
+  testcase_id = source_id.split(':')[1]
+  oss_fuzz_pattern = re.compile(r'oss-?fuzz', re.IGNORECASE)
+  has_oss_fuzz_in_message = []
+  has_testcase_id_in_message = []
+  has_issue_id_in_message = []
+
+  # Look for commits with (in order of decreasing priority):
+  # - "oss-?fuzz" and the issue ID in the message.
+  # - ClusterFuzz testcase ID in the message.
+  # - "oss-?fuzz" in the message.
+
+  for commit in walker:
+    commit_message = commit.message.lower()
+    has_oss_fuzz = False
+
+    if oss_fuzz_pattern.search(commit_message):
+      has_oss_fuzz = True
+      has_oss_fuzz_in_message.append(commit)
+
+    if testcase_id in commit_message:
+      has_testcase_id_in_message.append(commit)
+
+    if issue_id and issue_id in commit_message and has_oss_fuzz:
+      has_issue_id_in_message.append(commit)
+
+  if has_issue_id_in_message:
+    return str(has_issue_id_in_message[0].id)
+
+  if has_testcase_id_in_message:
+    return str(has_testcase_id_in_message[0].id)
+
+  if len(has_oss_fuzz_in_message) == 1:
+    # Only pick the commit if there is a single one that mentions oss-fuzz.
+    return str(has_oss_fuzz_in_message[0].id)
+
+  return None
+
+
 def process_bisect_task(oss_fuzz_dir, bisect_type, source_id, message):
   """Process a bisect task."""
   bisect_type = message.attributes['type']
@@ -415,8 +470,26 @@ def process_impact_task(source_id, message):
   if not repo_url:
     raise osv.ImpactError('No repo_url set')
 
-  result = osv.get_affected(repo_url, regress_result.commit, fix_result.commit)
-  logging.info('Found affected %s', ', '.join(result.tags))
+  issue_id = fix_result.issue_id or regress_result.issue_id
+  fix_commit = fix_result.commit
+
+  with tempfile.TemporaryDirectory() as tmp_dir:
+    repo = osv.clone_with_retries(repo_url, tmp_dir)
+
+    # If not a precise fix commit, try to find the exact one by going through
+    # commit messages (oss-fuzz only).
+    if source_id.startswith(OSS_FUZZ_SOURCE_PREFIX) and ':' in fix_commit:
+      start_commit, end_commit = fix_commit.split(':')
+      commit = find_oss_fuzz_fix_via_commit(repo, start_commit, end_commit,
+                                            source_id, issue_id)
+      if commit:
+        logging.info('Found exact fix commit %s via commit message (oss-fuzz)',
+                     commit)
+        fix_commit = commit
+
+    # Actually compute the affected commits/tags.
+    result = osv.get_affected(repo, regress_result.commit, fix_commit)
+    logging.info('Found affected %s', ', '.join(result.tags))
 
   # If the range resolved to a single commit, simplify it.
   if len(result.fix_commits) == 1:
@@ -424,15 +497,12 @@ def process_impact_task(source_id, message):
   elif not result.fix_commits:
     # Not fixed.
     fix_commit = ''
-  else:
-    fix_commit = fix_result.commit
 
   if len(result.regress_commits) == 1:
     regress_commit = result.regress_commits[0]
   else:
     regress_commit = regress_result.commit
 
-  issue_id = fix_result.issue_id or regress_result.issue_id
   project = fix_result.project or regress_result.project
   ecosystem = fix_result.ecosystem or regress_result.ecosystem
   summary = fix_result.summary or regress_result.summary
@@ -538,7 +608,7 @@ def get_source_id(message):
 
   testcase_id = message.attributes['testcase_id']
   if testcase_id:
-    return 'oss-fuzz:' + testcase_id
+    return OSS_FUZZ_SOURCE_PREFIX + testcase_id
 
   return None
 
diff --git a/lib/osv/impact.py b/lib/osv/impact.py
--- a/lib/osv/impact.py
+++ b/lib/osv/impact.py
@@ -94,65 +94,62 @@ def ranges(self):
     return ranges
 
 
-def get_affected(git_url, regress_commit_or_range, fix_commit_or_range):
+def get_affected(repo, regress_commit_or_range, fix_commit_or_range):
   """"Get list of affected tags and commits for a bug given regressed and fixed
   commits."""
   confidence = CONFIDENCE_FULL
 
-  with tempfile.TemporaryDirectory() as tmp_dir:
-    repo = clone_with_retries(git_url, tmp_dir)
-
-    # If multiple, assume any commit in the regression range cause the
-    # regression.
-    regress_commits = get_commit_range(repo, regress_commit_or_range)
-    if len(regress_commits) > COMMIT_RANGE_LIMIT:
-      raise ImpactError('Too many commits in regression range.')
-
-    # If multiple, assume all commits are necessary for fixing the regression.
-    fix_commits = get_commit_range(repo, fix_commit_or_range)
-    if len(fix_commits) > COMMIT_RANGE_LIMIT:
-      logging.warning('Too many commits in fix range.')
-      # Rather than bail out here and potentially leaving a Bug as "unfixed"
-      # indefinitely, we do the best we can here, by assuming the last
-      # COMMIT_RANGE_LIMIT commits fix the bug.
-      fix_commits = fix_commits[-COMMIT_RANGE_LIMIT:]
-      confidence -= CONFIDENCE_RANGE_REDUCTION
-
-    # For every extra commit in the range, reduce the confidence.
-    if len(regress_commits) > 1:
-      confidence -= CONFIDENCE_RANGE_REDUCTION
-      confidence -= (len(regress_commits) - 1) * CONFIDENCE_RANGE_REDUCTION_STEP
-
-    # Special case: unknown status for earlier revisions.
-    unknown_earlier_revisions = UNKNOWN_COMMIT in regress_commit_or_range
-    if unknown_earlier_revisions:
-      confidence -= CONFIDENCE_RANGE_REDUCTION
-
-    if len(fix_commits) > 1:
-      confidence -= CONFIDENCE_RANGE_REDUCTION
-      confidence -= (len(fix_commits) - 1) * CONFIDENCE_RANGE_REDUCTION_STEP
-
-    if confidence < 0:
-      confidence = 0
-
-    tags_with_bug = set()
-    for commit in regress_commits:
-      tags_with_bug.update(get_tags_with_commits(repo, [commit]))
-
-    tags_with_fix = get_tags_with_commits(repo, fix_commits)
-
-    affected_tags = list(tags_with_bug - tags_with_fix)
-    affected_tags.sort()
-
-    affected_commits, affected_ranges = get_affected_range(
-        repo, regress_commits, fix_commits)
-
-    if unknown_earlier_revisions:
-      # Include the unknown marker in resulting entities.
-      regress_commits.insert(0, UNKNOWN_COMMIT)
-
-    return AffectedResult(affected_tags, affected_commits, affected_ranges,
-                          regress_commits, fix_commits, confidence)
+  # If multiple, assume any commit in the regression range cause the
+  # regression.
+  regress_commits = get_commit_range(repo, regress_commit_or_range)
+  if len(regress_commits) > COMMIT_RANGE_LIMIT:
+    raise ImpactError('Too many commits in regression range.')
+
+  # If multiple, assume all commits are necessary for fixing the regression.
+  fix_commits = get_commit_range(repo, fix_commit_or_range)
+  if len(fix_commits) > COMMIT_RANGE_LIMIT:
+    logging.warning('Too many commits in fix range.')
+    # Rather than bail out here and potentially leaving a Bug as "unfixed"
+    # indefinitely, we do the best we can here, by assuming the last
+    # COMMIT_RANGE_LIMIT commits fix the bug.
+    fix_commits = fix_commits[-COMMIT_RANGE_LIMIT:]
+    confidence -= CONFIDENCE_RANGE_REDUCTION
+
+  # For every extra commit in the range, reduce the confidence.
+  if len(regress_commits) > 1:
+    confidence -= CONFIDENCE_RANGE_REDUCTION
+    confidence -= (len(regress_commits) - 1) * CONFIDENCE_RANGE_REDUCTION_STEP
+
+  # Special case: unknown status for earlier revisions.
+  unknown_earlier_revisions = UNKNOWN_COMMIT in regress_commit_or_range
+  if unknown_earlier_revisions:
+    confidence -= CONFIDENCE_RANGE_REDUCTION
+
+  if len(fix_commits) > 1:
+    confidence -= CONFIDENCE_RANGE_REDUCTION
+    confidence -= (len(fix_commits) - 1) * CONFIDENCE_RANGE_REDUCTION_STEP
+
+  if confidence < 0:
+    confidence = 0
+
+  tags_with_bug = set()
+  for commit in regress_commits:
+    tags_with_bug.update(get_tags_with_commits(repo, [commit]))
+
+  tags_with_fix = get_tags_with_commits(repo, fix_commits)
+
+  affected_tags = list(tags_with_bug - tags_with_fix)
+  affected_tags.sort()
+
+  affected_commits, affected_ranges = get_affected_range(
+      repo, regress_commits, fix_commits)
+
+  if unknown_earlier_revisions:
+    # Include the unknown marker in resulting entities.
+    regress_commits.insert(0, UNKNOWN_COMMIT)
+
+  return AffectedResult(affected_tags, affected_commits, affected_ranges,
+                        regress_commits, fix_commits, confidence)
 
 
 def get_affected_range(repo, regress_commits, fix_commits):


Test patch:
diff --git a/docker/worker/worker_test.py b/docker/worker/worker_test.py
--- a/docker/worker/worker_test.py
+++ b/docker/worker/worker_test.py
@@ -695,6 +695,51 @@ def test_mark_bug_invalid(self):
     self.assertEqual(0, len(commits))
 
 
+class FindOssFuzzFixViaCommitTest(unittest.TestCase):
+  """Test finding OSS-Fuzz fixes via commits."""
+
+  def setUp(self):
+    self.repo = pygit2.Repository('osv-test')
+
+  def test_has_issue_id(self):
+    """Test identifying the commit that has the issue ID."""
+    commit = worker.find_oss_fuzz_fix_via_commit(
+        self.repo, 'e1b045257bc5ca2a11d0476474f45ef77a0366c7',
+        '949f182716f037e25394bbb98d39b3295d230a29', 'oss-fuzz:133713371337',
+        '12345')
+    self.assertEqual('57e58a5d7c2bb3ce0f04f17ec0648b92ee82531f', commit)
+
+    commit = worker.find_oss_fuzz_fix_via_commit(
+        self.repo, 'e1b045257bc5ca2a11d0476474f45ef77a0366c7',
+        '25147a74d8aeb27b43665530ee121a2a1b19dc58', 'oss-fuzz:133713371337',
+        '12345')
+    self.assertEqual('25147a74d8aeb27b43665530ee121a2a1b19dc58', commit)
+
+  def test_has_testcase_id(self):
+    """Test identifying the commit that has the testcase ID."""
+    commit = worker.find_oss_fuzz_fix_via_commit(
+        self.repo, 'e1b045257bc5ca2a11d0476474f45ef77a0366c7',
+        '00514d6f244f696e750a37083163992c6a50cfd3', 'oss-fuzz:133713371337',
+        '12345')
+
+    self.assertEqual('90aa4127295b2c37b5f7fcf6a9772b12c99a5212', commit)
+
+  def test_has_oss_fuzz_reference(self):
+    """Test identifying the commit that has the testcase ID."""
+    commit = worker.find_oss_fuzz_fix_via_commit(
+        self.repo, 'e1b045257bc5ca2a11d0476474f45ef77a0366c7',
+        'b1fa81a5d59e9b4d6e276d82fc17058f3cf139d9', 'oss-fuzz:133713371337',
+        '12345')
+
+    self.assertEqual('3c5dcf6a5bec14baab3b247d369a7270232e1b83', commit)
+
+  def test_has_multiple_oss_fuzz_reference(self):
+    commit = worker.find_oss_fuzz_fix_via_commit(
+        self.repo, 'e1b045257bc5ca2a11d0476474f45ef77a0366c7',
+        '949f182716f037e25394bbb98d39b3295d230a29', 'oss-fuzz:7331', '54321')
+    self.assertIsNone(commit)
+
+
 if __name__ == '__main__':
   os.system('pkill -f datastore')
   ds_emulator = tests.start_datastore_emulator()