Repository: google/osv.dev

Issue:
Allow failed version expansions for non-existent packages.
Some advisories may refer to packages that no longer exist (e.g. malicious packages)


Discussion:

Resolving patch:
diff --git a/lib/osv/ecosystems.py b/lib/osv/ecosystems.py
--- a/lib/osv/ecosystems.py
+++ b/lib/osv/ecosystems.py
@@ -31,6 +31,10 @@
 deps_dev_api_key = ''
 
 
+class EnumerateError(Exception):
+  """Non-retryable version enumeration error."""
+
+
 class DepsDevMixin:
   """deps.dev mixin."""
 
@@ -48,6 +52,8 @@ def _deps_dev_enumerate(self, package, introduced, fixed, limits=None):
             'X-DepsDev-APIKey': deps_dev_api_key,
         })
 
+    if response.status_code == 404:
+      raise EnumerateError(f'Package {package} not found')
     if response.status_code != 200:
       raise RuntimeError(
           f'Failed to get {ecosystem} versions for {package} with: '
@@ -173,6 +179,9 @@ def sort_key(self, version):
   def enumerate_versions(self, package, introduced, fixed, limits=None):
     """Enumerate versions."""
     response = requests.get(self._API_PACKAGE_URL.format(package=package))
+
+    if response.status_code == 404:
+      raise EnumerateError(f'Package {package} not found')
     if response.status_code != 200:
       raise RuntimeError(
           f'Failed to get PyPI versions for {package} with: {response.text}')
@@ -218,6 +227,9 @@ def enumerate_versions(self, package, introduced, fixed, limits=None):
             f'Failed to get Maven versions for {package} with: {response.text}')
 
       response = response.json()['response']
+      if response['numFound'] == 0:
+        raise EnumerateError(f'Package {package} not found')
+
       for result in response['docs']:
         versions.append(result['v'])
 
@@ -242,6 +254,8 @@ def sort_key(self, version):
   def enumerate_versions(self, package, introduced, fixed, limits=None):
     """Enumerate versions."""
     response = requests.get(self._API_PACKAGE_URL.format(package=package))
+    if response.status_code == 404:
+      raise EnumerateError(f'Package {package} not found')
     if response.status_code != 200:
       raise RuntimeError(
           f'Failed to get RubyGems versions for {package} with: {response.text}'
@@ -268,6 +282,8 @@ def enumerate_versions(self, package, introduced, fixed, limits=None):
     """Enumerate versions."""
     url = self._API_PACKAGE_URL.format(package=package.lower())
     response = requests.get(url)
+    if response.status_code == 404:
+      raise EnumerateError(f'Package {package} not found')
     if response.status_code != 200:
       raise RuntimeError(
           f'Failed to get NuGet versions for {package} with: {response.text}')
diff --git a/lib/osv/impact.py b/lib/osv/impact.py
--- a/lib/osv/impact.py
+++ b/lib/osv/impact.py
@@ -483,9 +483,14 @@ def analyze(vulnerability,
         # Enumerate ECOSYSTEM and SEMVER ranges.
         ecosystem_helpers = ecosystems.get(affected_package.package.ecosystem)
         if ecosystem_helpers:
-          versions.extend(
-              enumerate_versions(affected_package.package.name,
-                                 ecosystem_helpers, affected_range))
+          try:
+            versions.extend(
+                enumerate_versions(affected_package.package.name,
+                                   ecosystem_helpers, affected_range))
+          except ecosystems.EnumerateError:
+            # Allow non-retryable enumeration errors to occur (e.g. if the
+            # package no longer exists).
+            pass
         else:
           logging.warning('No ecosystem helpers implemented for %s',
                           affected_package.package.ecosystem)


Test patch:
diff --git a/lib/osv/ecosystems_test.py b/lib/osv/ecosystems_test.py
--- a/lib/osv/ecosystems_test.py
+++ b/lib/osv/ecosystems_test.py
@@ -23,19 +23,26 @@ class GetNextVersionTest(unittest.TestCase):
   """get_next_version tests."""
 
   def test_pypi(self):
+    """Test PyPI."""
     ecosystem = ecosystems.get('PyPI')
     self.assertEqual('1.36.0rc1', ecosystem.next_version('grpcio', '1.35.0'))
     self.assertEqual('1.36.1', ecosystem.next_version('grpcio', '1.36.0'))
     self.assertEqual('0.3.0', ecosystem.next_version('grpcio', '0'))
+    with self.assertRaises(ecosystems.EnumerateError):
+      ecosystem.next_version('doesnotexist123456', '1')
 
   def test_maven(self):
+    """Test Maven."""
     ecosystem = ecosystems.get('Maven')
     self.assertEqual('1.36.0',
                      ecosystem.next_version('io.grpc:grpc-core', '1.35.1'))
     self.assertEqual('0.7.0', ecosystem.next_version('io.grpc:grpc-core', '0'))
+    with self.assertRaises(ecosystems.EnumerateError):
+      ecosystem.next_version('blah:doesnotexist123456', '1')
 
   @unittest.skipUnless(os.getenv('DEPS_DEV_API_KEY'), 'Requires API key')
   def test_maven_deps_dev(self):
+    """Test Maven using deps.dev."""
     ecosystems.use_deps_dev = True
     ecosystems.deps_dev_api_key = os.getenv('DEPS_DEV_API_KEY')
 
@@ -43,9 +50,13 @@ def test_maven_deps_dev(self):
     self.assertEqual('1.36.0',
                      ecosystem.next_version('io.grpc:grpc-core', '1.35.1'))
     self.assertEqual('0.7.0', ecosystem.next_version('io.grpc:grpc-core', '0'))
+    with self.assertRaises(ecosystems.EnumerateError):
+      ecosystem.next_version('blah:doesnotexist123456', '1')
+
     ecosystems.use_deps_dev = False
 
   def test_gems(self):
+    """Test RubyGems."""
     ecosystem = ecosystems.get('RubyGems')
     self.assertEqual('0.8.0', ecosystem.next_version('rails', '0'))
     self.assertEqual('0.9.5', ecosystem.next_version('rails', '0.9.4.1'))
@@ -54,8 +65,11 @@ def test_gems(self):
                      ecosystem.next_version('rails', '4.0.0.beta1'))
     self.assertEqual('5.0.0.racecar1',
                      ecosystem.next_version('rails', '5.0.0.beta4'))
+    with self.assertRaises(ecosystems.EnumerateError):
+      ecosystem.next_version('doesnotexist123456', '1')
 
   def test_nuget(self):
+    """Test NuGet."""
     ecosystem = ecosystems.get('NuGet')
     self.assertEqual('3.0.1',
                      ecosystem.next_version('NuGet.Server.Core', '3.0.0'))
@@ -65,8 +79,11 @@ def test_nuget(self):
                      ecosystem.next_version('Castle.Core', '3.0.0.4001'))
     self.assertEqual('2.1.0-dev-00668',
                      ecosystem.next_version('Serilog', '2.1.0-dev-00666'))
+    with self.assertRaises(ecosystems.EnumerateError):
+      ecosystem.next_version('doesnotexist123456', '1')
 
   def test_semver(self):
+    """Test SemVer."""
     ecosystem = ecosystems.get('Go')
     self.assertEqual('1.0.1-0', ecosystem.next_version('blah', '1.0.0'))
     self.assertEqual('1.0.0-pre.0', ecosystem.next_version('blah', '1.0.0-pre'))