Repository: google/osv.dev

Issue:
Inaccurate impact analysis
https://github.com/google/oss-fuzz-vulns/pull/7#issuecomment-897529522


Discussion:

Resolving patch:
diff --git a/docker/worker/oss_fuzz.py b/docker/worker/oss_fuzz.py
--- a/docker/worker/oss_fuzz.py
+++ b/docker/worker/oss_fuzz.py
@@ -30,6 +30,12 @@
 OSS_FUZZ_ISSUE_URL = 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id='
 SOURCE_PREFIX = 'oss-fuzz:'
 
+COMMIT_RANGE_LIMIT = 4
+
+# Used in cases where an earlier commit in a regression range cannot be
+# determined.
+UNKNOWN_COMMIT = 'unknown'
+
 # Large projects which take way too long to build.
 # TODO(ochang): Don't hardcode this.
 PROJECT_DENYLIST = {
@@ -48,7 +54,7 @@ def format_commit_range(old_commit, new_commit):
   if old_commit == new_commit:
     return old_commit
 
-  return (old_commit or osv.UNKNOWN_COMMIT) + ':' + new_commit
+  return (old_commit or UNKNOWN_COMMIT) + ':' + new_commit
 
 
 def find_oss_fuzz_fix_via_commit(repo, start_commit, end_commit, source_id,
@@ -224,6 +230,39 @@ def set_bug_attributes(bug, regress_result, fix_result):
   bug.fixed = fix_result.commit or ''
 
 
+def _get_commit_range(repo, commit_or_range):
+  """Get a commit range."""
+  if not commit_or_range:
+    return []
+
+  if ':' not in commit_or_range:
+    return [commit_or_range]
+
+  start_commit, end_commit = commit_or_range.split(':')
+  if start_commit == UNKNOWN_COMMIT:
+    # Special case: No information about earlier builds. Assume the end_commit
+    # is the regressing commit as that's the best we can do.
+    return [end_commit]
+
+  commits, _ = osv.get_commit_and_tag_list(repo, start_commit, end_commit)
+  return commits
+
+
+def _get_commits(repo, regress_commit_or_range, fix_commit_or_range):
+  """Get commits for analysis."""
+  regress_commits = _get_commit_range(repo, regress_commit_or_range)
+  if len(regress_commits) > COMMIT_RANGE_LIMIT:
+    raise osv.ImpactError('Too many commits in regression range.')
+
+  fix_commits = _get_commit_range(repo, fix_commit_or_range)
+  if len(fix_commits) > COMMIT_RANGE_LIMIT:
+    logging.warning('Too many commits in fix range.')
+    # Rather than bail out here and potentially leaving a Bug as "unfixed"
+    # indefinitely, we continue.
+
+  return regress_commits, fix_commits
+
+
 def process_impact_task(source_id, message):
   """Process an impact task."""
   logging.info('Processing impact task for %s', source_id)
@@ -285,20 +324,43 @@ def process_impact_task(source_id, message):
 
     # Actually compute the affected commits/tags.
     repo_analyzer = osv.RepoAnalyzer()
-    result = repo_analyzer.get_affected(repo, regress_result.commit, fix_commit)
+    regress_commits, fix_commits = _get_commits(repo, regress_result.commit,
+                                                fix_commit)
+
+    # If multiple, assume the first commit in the regression range cause the
+    # regression.
+    if regress_commits:
+      regress_commit_to_analyze = regress_commits[0]
+    else:
+      regress_commit_to_analyze = None
+
+    # If multiple, assume the last commit is necessary for fixing the
+    # regression.
+    if fix_commits:
+      fix_commit_to_analyze = fix_commits[-1]
+    else:
+      fix_commit_to_analyze = None
+
+    result = repo_analyzer.get_affected(repo, [regress_commit_to_analyze],
+                                        [fix_commit_to_analyze])
     affected_tags = sorted(list(result.tags))
     logging.info('Found affected %s', ', '.join(affected_tags))
 
+    if len(regress_commits) > 1 or len(fix_commits) > 1:
+      # Don't return ranges if input regressed and fixed commits are not single
+      # commits.
+      result.affected_ranges.clear()
+
   # If the range resolved to a single commit, simplify it.
-  if len(result.fix_commits) == 1:
-    fix_commit = result.fix_commits[0]
-  elif not result.fix_commits:
+  if len(fix_commits) == 1:
+    fix_commit = fix_commits[0]
+  elif not fix_commits:
     # Not fixed.
     fix_commit = ''
 
-  if (len(result.regress_commits) == 1 and
-      osv.UNKNOWN_COMMIT not in regress_result.commit):
-    regress_commit = result.regress_commits[0]
+  if (len(regress_commits) == 1 and
+      UNKNOWN_COMMIT not in regress_result.commit):
+    regress_commit = regress_commits[0]
   else:
     regress_commit = regress_result.commit
 
@@ -316,8 +378,8 @@ def process_impact_task(source_id, message):
 
   # For the AffectedRange, use the first commit in the regress commit range, and
   # the last commit in the fix commit range.
-  introduced = result.regress_commits[0] if result.regress_commits else ''
-  fixed = result.fix_commits[-1] if result.fix_commits else ''
+  introduced = regress_commits[0] if regress_commits else ''
+  fixed = fix_commits[-1] if fix_commits else ''
   existing_bug.affected_ranges = [
       osv.AffectedRange(
           type='GIT', repo_url=repo_url, introduced=introduced, fixed=fixed),
diff --git a/lib/osv/impact.py b/lib/osv/impact.py
--- a/lib/osv/impact.py
+++ b/lib/osv/impact.py
@@ -29,17 +29,10 @@
 from . import models
 from . import vulnerability_pb2
 
-COMMIT_RANGE_LIMIT = 4
-
 TAG_PREFIX = 'refs/tags/'
 
-# Used in cases where an earlier commit in a regression range cannot be
-# determined.
-UNKNOWN_COMMIT = 'unknown'
-
-AffectedResult = collections.namedtuple(
-    'AffectedResult', 'tags commits affected_ranges '
-    'regress_commits fix_commits')
+AffectedResult = collections.namedtuple('AffectedResult',
+                                        'tags commits affected_ranges')
 
 AnalyzeResult = collections.namedtuple('AnalyzeResult', 'has_changes commits')
 
@@ -92,45 +85,15 @@ class RepoAnalyzer:
   def __init__(self, detect_cherrypicks=True):
     self.detect_cherrypicks = detect_cherrypicks
 
-  def get_affected(self, repo, regress_commit_or_range, fix_commit_or_range):
+  def get_affected(self, repo, regress_commits, fix_commits):
     """"Get list of affected tags and commits for a bug given regressed and
     fixed commits."""
-    regress_commits = _get_commit_range(repo, regress_commit_or_range)
-    if len(regress_commits) > COMMIT_RANGE_LIMIT:
-      raise ImpactError('Too many commits in regression range.')
-
-    # If multiple, assume the first commit in the regression range cause the
-    # regression.
-    if regress_commits:
-      regress_commit = regress_commits[0]
-    else:
-      regress_commit = None
-
-    fix_commits = _get_commit_range(repo, fix_commit_or_range)
-    if len(fix_commits) > COMMIT_RANGE_LIMIT:
-      logging.warning('Too many commits in fix range.')
-      # Rather than bail out here and potentially leaving a Bug as "unfixed"
-      # indefinitely, we continue.
-
-    # If multiple, assume the last commit is necessary for fixing the
-    # regression.
-    if fix_commits:
-      fix_commit = fix_commits[-1]
-    else:
-      fix_commit = None
-
     affected_commits, affected_ranges, tags = self._get_affected_range(
-        repo, regress_commit, fix_commit)
+        repo, regress_commits, fix_commits)
 
-    if len(regress_commits) > 1 or len(fix_commits) > 1:
-      # Don't return ranges if input regressed and fixed commits are not single
-      # commits.
-      affected_ranges = []
+    return AffectedResult(tags, affected_commits, affected_ranges)
 
-    return AffectedResult(tags, affected_commits, affected_ranges,
-                          regress_commits, fix_commits)
-
-  def _get_affected_range(self, repo, regress_commit, fix_commit):
+  def _get_affected_range(self, repo, regress_commits, fix_commits):
     """Get affected range."""
     range_collector = RangeCollector()
     commits = set()
@@ -138,50 +101,52 @@ def _get_affected_range(self, repo, regress_commit, fix_commit):
     tags = set()
     commits_to_tags = _get_commit_to_tag_mappings(repo)
 
-    is_complete_range = regress_commit and fix_commit
-    # Only actually detect cherrypicks if the input range includes both
-    # "introduced" and "fixed". Otherwise we may get false positives.
-    detect_cherrypicks = self.detect_cherrypicks and is_complete_range
-
     branches = []
-    if detect_cherrypicks:
+    if self.detect_cherrypicks:
       # Check all branches for cherry picked regress/fix commits (sorted for
       # determinism).
       branches = sorted(repo.branches.remote)
     else:
-      if fix_commit:
-        # If a fix commit is available, we don't really need to analyze any
-        # other branches.
-        branches = [repo.head.name.replace('refs/heads/', 'origin/')]
-      elif regress_commit:
+      if fix_commits:
+        # If not detecting cherry picks, take only branches that contain the fix
+        # commit. Otherwise we may have false positives.
+        for fix_commit in fix_commits:
+          branches.extend(_branches_with_commit(repo, fix_commit))
+      elif regress_commits:
         # If only a regress commit is available, we need to find all branches
         # that it reaches.
-        branches = _branches_with_commit(repo, regress_commit)
+        for regress_commit in regress_commits:
+          branches.extend(_branches_with_commit(repo, regress_commit))
 
     for branch in branches:
       ref = 'refs/remotes/' + branch
 
       # Get the earliest equivalent commit in the regression range.
-      if detect_cherrypicks:
+      equivalent_regress_commit = None
+      for regress_commit in regress_commits:
         logging.info('Finding equivalent regress commit to %s in %s',
                      regress_commit, ref)
         equivalent_regress_commit = self._get_equivalent_commit(
-            repo, ref, regress_commit, detect_cherrypicks=detect_cherrypicks)
-      else:
-        equivalent_regress_commit = regress_commit
+            repo,
+            ref,
+            regress_commit,
+            detect_cherrypicks=self.detect_cherrypicks)
+        if equivalent_regress_commit:
+          break
 
       # If regress_commits is provided, then we should find an equivalent.
-      if not equivalent_regress_commit and regress_commit:
+      if not equivalent_regress_commit and regress_commits:
         continue
 
       # Get the latest equivalent commit in the fix range.
-      if detect_cherrypicks:
+      equivalent_fix_commit = None
+      for fix_commit in fix_commits:
         logging.info('Finding equivalent fix commit to %s in %s', fix_commit,
                      ref)
         equivalent_fix_commit = self._get_equivalent_commit(
-            repo, ref, fix_commit, detect_cherrypicks=detect_cherrypicks)
-      else:
-        equivalent_fix_commit = fix_commit
+            repo, ref, fix_commit, detect_cherrypicks=self.detect_cherrypicks)
+        if equivalent_fix_commit:
+          break
 
       range_collector.add(equivalent_regress_commit, equivalent_fix_commit)
 
@@ -197,7 +162,7 @@ def _get_affected_range(self, repo, regress_commit, fix_commit):
         continue
 
       seen_commits.add((equivalent_regress_commit, end_commit))
-      cur_commits, cur_tags = _get_commit_and_tag_list(
+      cur_commits, cur_tags = get_commit_and_tag_list(
           repo,
           equivalent_regress_commit,
           end_commit,
@@ -253,24 +218,6 @@ def _get_equivalent_commit(self,
     return None
 
 
-def _get_commit_range(repo, commit_or_range):
-  """Get a commit range."""
-  if not commit_or_range:
-    return []
-
-  if ':' not in commit_or_range:
-    return [commit_or_range]
-
-  start_commit, end_commit = commit_or_range.split(':')
-  if start_commit == UNKNOWN_COMMIT:
-    # Special case: No information about earlier builds. Assume the end_commit
-    # is the regressing commit as that's the best we can do.
-    return [end_commit]
-
-  commits, _ = _get_commit_and_tag_list(repo, start_commit, end_commit)
-  return commits
-
-
 def _get_commit_to_tag_mappings(repo):
   """Get all commit to tag mappings"""
   mappings = {}
@@ -285,12 +232,12 @@ def _get_commit_to_tag_mappings(repo):
   return mappings
 
 
-def _get_commit_and_tag_list(repo,
-                             start_commit,
-                             end_commit,
-                             commits_to_tags=None,
-                             include_start=False,
-                             include_end=True):
+def get_commit_and_tag_list(repo,
+                            start_commit,
+                            end_commit,
+                            commits_to_tags=None,
+                            include_start=False,
+                            include_end=True):
   """Given a commit range, return the list of commits and tags in the range."""
   logging.info('Getting commits %s..%s', start_commit, end_commit)
   try:
@@ -412,10 +359,10 @@ def _analyze_git_ranges(repo_analyzer, checkout_path, vulnerability,
                         range_collectors, new_versions, commits):
   """Analyze git ranges."""
   package_repo_dir = tempfile.TemporaryDirectory()
-  package_repo_url = None
   package_repo = None
 
   try:
+    grouped_git_ranges = {}
     for affected_range in vulnerability.affects.ranges:
       if affected_range.type != vulnerability_pb2.AffectedRange.GIT:
         continue
@@ -425,38 +372,38 @@ def _analyze_git_ranges(repo_analyzer, checkout_path, vulnerability,
       fixed = affected_range.fixed or None
       range_collectors[affected_range.repo].add(introduced, fixed)
 
-    for affected_range in vulnerability.affects.ranges:
-      # Go through existing provided ranges to find additional ranges (via
-      # cherrypicks and branches).
-      if affected_range.type != vulnerability_pb2.AffectedRange.GIT:
-        continue
+      grouped_git_ranges.setdefault(affected_range.repo, []).append(
+          (introduced, fixed))
 
-      current_repo_url = affected_range.repo
+    for repo_url, affected_ranges in grouped_git_ranges.items():
       if checkout_path:
-        repo_name = os.path.basename(
-            current_repo_url.rstrip('/')).rstrip('.git')
+        repo_name = os.path.basename(repo_url.rstrip('/')).rstrip('.git')
         package_repo = repos.ensure_updated_checkout(
-            current_repo_url, os.path.join(checkout_path, repo_name))
+            repo_url, os.path.join(checkout_path, repo_name))
       else:
-        if current_repo_url != package_repo_url:
-          # Different repo from previous one.
-          package_repo_dir.cleanup()
-          package_repo_dir = tempfile.TemporaryDirectory()
-          package_repo_url = current_repo_url
-          package_repo = repos.clone_with_retries(package_repo_url,
-                                                  package_repo_dir.name)
+        package_repo_dir.cleanup()
+        package_repo_dir = tempfile.TemporaryDirectory()
+        package_repo = repos.clone_with_retries(repo_url, package_repo_dir.name)
+
+      all_introduced = []
+      all_fixed = []
+      for (introduced, fixed) in affected_ranges:
+        if introduced:
+          all_introduced.append(introduced)
+
+        if fixed:
+          all_fixed.append(fixed)
 
       try:
-        result = repo_analyzer.get_affected(package_repo,
-                                            affected_range.introduced,
-                                            affected_range.fixed)
+        result = repo_analyzer.get_affected(package_repo, all_introduced,
+                                            all_fixed)
       except ImpactError:
         logging.warning('Got error while analyzing git range: %s',
                         traceback.format_exc())
         continue
 
       for introduced, fixed in result.affected_ranges:
-        range_collectors[current_repo_url].add(introduced, fixed)
+        range_collectors[repo_url].add(introduced, fixed)
 
       new_versions.update(result.tags)
       commits.update(result.commits)


Test patch:
diff --git a/docker/worker/worker_test.py b/docker/worker/worker_test.py
--- a/docker/worker/worker_test.py
+++ b/docker/worker/worker_test.py
@@ -658,14 +658,21 @@ def test_not_fixed(self):
             'purl': None,
             'related': [],
             'withdrawn': None,
-            'affected':
-                ['branch-v0.1.1', 'branch-v0.1.1-with-fix', 'v0.1.1', 'v0.2'],
-            'affected_fuzzy': ['0-1-1', '0-1-1', '0-1-1', '0-2'],
+            'affected': [
+                'branch-v0.1.1', 'branch-v0.1.1-with-fix',
+                'branch_1_cherrypick_regress', 'v0.1.1', 'v0.2'
+            ],
+            'affected_fuzzy': ['0-1-1', '0-1-1', '1', '0-1-1', '0-2'],
             'affected_ranges': [{
                 'fixed': '',
                 'introduced': 'eefe8ec3f1f90d0e684890e810f3f21e8500a4cd',
                 'repo_url': 'https://repo.com/repo',
                 'type': 'GIT'
+            }, {
+                'fixed': '',
+                'introduced': 'febfac1940086bc1f6d3dc33fda0a1d1ba336209',
+                'repo_url': 'https://repo.com/repo',
+                'type': 'GIT'
             }],
             'fixed': '',
             'regressed': 'eefe8ec3f1f90d0e684890e810f3f21e8500a4cd',
@@ -712,6 +719,8 @@ def test_not_fixed(self):
         'b587c21c36a84e16cfc6b39eb68578d43b5281ad',
         '88e5ae3c40c85b702ba89a34c29f233048abb12b',
         '3ea6feea9bb853596c727abab309476cc07d1505',
+        'febfac1940086bc1f6d3dc33fda0a1d1ba336209',
+        'ff8cc32ba60ad9cbb3b23f0a82aad96ebe9ff76b',
     ], [commit.commit for commit in affected_commits])
 
 
@@ -1103,13 +1112,20 @@ def test_update_no_introduced(self):
             'purl': None,
             'related': [],
             'withdrawn': None,
-            'affected': ['v0.1', 'v0.1.1'],
-            'affected_fuzzy': ['0-1', '0-1-1'],
+            'affected': [
+                'branch-v0.1.1', 'branch_1_cherrypick_regress', 'v0.1', 'v0.1.1'
+            ],
+            'affected_fuzzy': ['0-1-1', '1', '0-1', '0-1-1'],
             'affected_ranges': [{
                 'fixed': '8d8242f545e9cec3e6d0d2e3f5bde8be1c659735',
                 'introduced': '',
                 'repo_url': 'https://osv-test/repo/url',
                 'type': 'GIT'
+            }, {
+                'fixed': 'b9b3fd4732695b83c3068b7b6a14bb372ec31f98',
+                'introduced': '',
+                'repo_url': 'https://osv-test/repo/url',
+                'type': 'GIT'
             }],
             'details': 'Blah blah blah\nBlah\n',
             'ecosystem': 'golang',
@@ -1147,6 +1163,17 @@ def test_update_no_introduced(self):
         'eefe8ec3f1f90d0e684890e810f3f21e8500a4cd',
         'a2ba949290915d445d34d0e8e9de2e7ce38198fc',
         'e1b045257bc5ca2a11d0476474f45ef77a0366c7',
+        '00514d6f244f696e750a37083163992c6a50cfd3',
+        '25147a74d8aeb27b43665530ee121a2a1b19dc58',
+        '3c5dcf6a5bec14baab3b247d369a7270232e1b83',
+        '4c155795426727ea05575bd5904321def23c03f4',
+        '57e58a5d7c2bb3ce0f04f17ec0648b92ee82531f',
+        '90aa4127295b2c37b5f7fcf6a9772b12c99a5212',
+        '949f182716f037e25394bbb98d39b3295d230a29',
+        'b1fa81a5d59e9b4d6e276d82fc17058f3cf139d9',
+        'f0cc40d8c3dabb27c2cfe26f1764305abc91a0b9',
+        'febfac1940086bc1f6d3dc33fda0a1d1ba336209',
+        'ff8cc32ba60ad9cbb3b23f0a82aad96ebe9ff76b',
     ], [commit.commit for commit in affected_commits])
 
   def test_update_new(self):