Repository: google/osv.dev

Issue:
Normalize package versions for querying
Currently, for non-semver ecosystems, version numbers in queries only work if they are exactly what is listed on the package manager.

Some ecosystems allow for multiple ways to refer to a specific version e.g. in PyPI installing `django==4.2.0` will install [`Django==4.2`](https://pypi.org/project/Django/4.2/). (in fact, you can specify any number of trailing `.0`'s and it will install the same version)
Querying the OSV API for django 4.2.0 yields no results, even though django 4.2 does have vulnerabilities.

We need to normalize versions both when ingesting and on the API when querying so we can match these cases.


Discussion:
I suppose we can use https://packaging.pypa.io/en/latest/utils.html#packaging.utils.canonicalize_version for this.

@michaelkedar do you know of any other ecosystems that have this issue?

@cuixq may be an interesting starter project for osv.dev infra.
Per https://github.com/google/osv.dev/discussions/1673 this may be an issue for NuGet as well. 

Resolving patch:
diff --git a/gcp/api/server.py b/gcp/api/server.py
--- a/gcp/api/server.py
+++ b/gcp/api/server.py
@@ -36,6 +36,7 @@
 from grpc_health.v1 import health_pb2_grpc
 from grpc_reflection.v1alpha import reflection
 from packageurl import PackageURL
+from packaging.utils import canonicalize_version
 
 import osv
 from osv import ecosystems
@@ -773,40 +774,54 @@ def _query_by_generic_version(
     version: str,
 ):
   """Query by generic version."""
-  # Try without normalizing.
+
   results = []
-  query: ndb.Query = base_query.filter(osv.Bug.affected_fuzzy == version)
-  it: ndb.QueryIterator = query.iter(
-      # page_token can be the token for this query, or the token for the one
-      # below. If the token is used for the normalized query below, this query
-      # must have returned no results, so will still return no results, fall
-      # through to the query below again.
-      start_cursor=context.page_token)
-  cursor = None
 
-  while (yield it.has_next_async()):
-    if len(results) >= context.total_responses.page_limit():
-      cursor = it.cursor_after()
-      break
-    bug = it.next()
-    if _is_version_affected(bug.affected_packages, project, ecosystem, purl,
-                            version):
-      results.append(bug)
-      context.total_responses.add(1)
+  cursor = None
+  # Try without normalizing.
+  results, cursor = yield query_by_generic_helper(results, cursor, context,
+                                                  base_query, project,
+                                                  ecosystem, purl, version,
+                                                  False)
 
   if results:
     return results, cursor
 
+  # page_token can be the token for this query, or the token for the one
+  # below. If the token is used for the normalized query below, this query
+  # must have returned no results, so will still return no results, fall
+  # through to the query below again.
   # Try again after normalizing.
-  version = osv.normalize_tag(version)
-  query = base_query.filter(osv.Bug.affected_fuzzy == version)
-  it = query.iter(start_cursor=context.page_token)
+  results, cursor = yield query_by_generic_helper(results, cursor, context,
+                                                  base_query, project,
+                                                  ecosystem, purl,
+                                                  osv.normalize_tag(version),
+                                                  True)
+
+  if results:
+    return results, cursor
+
+  # Try again after canonicalizing + normalizing version.
+  results, cursor = yield query_by_generic_helper(results, cursor, context,
+                                                  base_query, project,
+                                                  ecosystem, purl,
+                                                  canonicalize_version(version),
+                                                  True)
+  return results, cursor
 
+
+@ndb.tasklet
+def query_by_generic_helper(results: list, cursor, context: QueryContext,
+                            base_query: ndb.Query, project: str, ecosystem: str,
+                            purl: PackageURL | None, version: str,
+                            is_normalized):
+  """Helper function for query_by_generic."""
+  query: ndb.Query = base_query.filter(osv.Bug.affected_fuzzy == version)
+  it: ndb.QueryIterator = query.iter(start_cursor=context.page_token)
   while (yield it.has_next_async()):
     if len(results) >= context.total_responses.page_limit():
       cursor = it.cursor_after()
       break
-
     bug = it.next()
     if _is_version_affected(
         bug.affected_packages,
@@ -814,10 +829,9 @@ def _query_by_generic_version(
         ecosystem,
         purl,
         version,
-        normalize=True):
+        normalize=is_normalized):
       results.append(bug)
       context.total_responses.add(1)
-
   return results, cursor
 
 


Test patch:
diff --git a/gcp/api/integration_tests.py b/gcp/api/integration_tests.py
--- a/gcp/api/integration_tests.py
+++ b/gcp/api/integration_tests.py
@@ -32,6 +32,7 @@
 _TIMEOUT = 10  # Timeout for HTTP(S) requests
 _LONG_TESTS = os.getenv('LONG_TESTS')
 _TEST_DATA_DIR = 'fixtures'
+_BASE_QUERY = '/v1/query'
 
 
 def _api():
@@ -195,7 +196,7 @@ def test_get_with_multiple(self):
   def test_query_commit(self):
     """Test querying by commit."""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'commit': '60e572dbf7b4ded66b488f54773f66aaf6184321',
         }),
@@ -205,7 +206,7 @@ def test_query_commit(self):
   def test_query_version(self):
     """Test querying by version."""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '2.1.2-rc',
             'package': {
@@ -217,7 +218,7 @@ def test_query_version(self):
     self.assert_results_equal({'vulns': [self._VULN_744]}, response.json())
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '2.1.2-rc',
             'package': {
@@ -236,7 +237,7 @@ def test_query_debian(self):
     dsa_2665_1 = self._get('DSA-710-1')
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '1.0.2-1',
             'package': {
@@ -248,7 +249,7 @@ def test_query_debian(self):
     self.assert_results_equal({'vulns': [dsa_2665_1]}, response.json())
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '1.0.2-1',
             'package': {
@@ -262,7 +263,7 @@ def test_query_debian(self):
     # The vulnerbility does not exist in 4.0 release, so this should return
     # with nothing
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '1.0.2-1',
             'package': {
@@ -274,7 +275,7 @@ def test_query_debian(self):
     self.assert_results_equal({}, response.json())
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '1.0.2-1',
             'package': {
@@ -307,7 +308,7 @@ def test_query_semver(self):
     # Test that a SemVer (believed to be vulnerable) version and an ecosystem
     # returns expected vulnerabilities.
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '1.1.4',
             'package': {
@@ -322,7 +323,7 @@ def test_query_semver(self):
     # ecosystem returns expected vulnerabilities to test the fallback logic to
     # try semver matching in the case that an ecosystem isn't specified.
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '1.1.4',
             'package': {
@@ -336,7 +337,7 @@ def test_query_semver(self):
   def test_query_invalid_ecosystem(self):
     """Test a query with an invalid ecosystem fails validation."""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '1.0.0',
             'package': {
@@ -354,7 +355,7 @@ def test_query_invalid_ecosystem(self):
   def test_query_unknown_purl_invalid_semver(self):
     """Test an unknown purl query with an invalid semver"""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'package': {
                 'purl':
@@ -367,7 +368,7 @@ def test_query_unknown_purl_invalid_semver(self):
     self.assert_results_equal({}, response.json())
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'package': {
                 'purl':
@@ -388,7 +389,7 @@ def test_query_semver_no_vulns(self):
     # ecosystem returns no vulnerabilities.
     # (This version does not exist)
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '1.1.1',
             'package': {
@@ -402,7 +403,7 @@ def test_query_semver_no_vulns(self):
   def test_query_semver_multiple_package(self):
     """Test query by SemVer (with multiple packages)."""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '2.4.0',
             'package': {
@@ -415,7 +416,7 @@ def test_query_semver_multiple_package(self):
     self.assert_results_equal({}, response.json())
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '2.4.0',
             'package': {
@@ -430,7 +431,7 @@ def test_query_semver_multiple_package(self):
                           [vuln['id'] for vuln in response_json['vulns']])
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '7.1.1',
             'package': {
@@ -453,7 +454,7 @@ def test_query_purl(self):
     ]
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'version': '0.8.6',
             'package': {
@@ -465,7 +466,7 @@ def test_query_purl(self):
     self.assert_results_equal({'vulns': expected}, response.json())
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps(
             {'package': {
                 'purl': 'pkg:cargo/crossbeam-utils@0.8.6',
@@ -476,7 +477,7 @@ def test_query_purl(self):
 
     another_expected = [self._get('GHSA-j8xg-fqg3-53r7')]
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({'package': {
             'purl': 'pkg:npm/word-wrap@1.2.2',
         }}),
@@ -487,7 +488,7 @@ def test_query_purl(self):
     expected_deb = [self._get('DLA-3203-1'), self._get('DSA-4921-1')]
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps(
             {'package': {
                 'purl': 'pkg:deb/debian/nginx@1.14.2-2+deb10u3',
@@ -498,7 +499,7 @@ def test_query_purl(self):
 
     # Source arch should return the same as above
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'package': {
                 'purl': 'pkg:deb/debian/nginx@1.14.2-2+deb10u3?arch=source',
@@ -510,7 +511,7 @@ def test_query_purl(self):
 
     # A non source arch should also return the same item
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'package': {
                 'purl': 'pkg:deb/debian/nginx@1.14.2-2+deb10u3?arch=x64',
@@ -522,7 +523,7 @@ def test_query_purl(self):
 
     # A non arch qualifier should be ignored
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'package': {
                 'purl': ('pkg:deb/debian/nginx@1.14.2-2+deb10u3?'
@@ -533,10 +534,59 @@ def test_query_purl(self):
 
     self.assert_results_equal({'vulns': expected_deb}, response.json())
 
+  def test_query_purl_with_version_trailing_zeroes(self):
+    """Test purl with trailing zeroes in version."""
+    expected = requests.post(
+        _api() + _BASE_QUERY,
+        data=json.dumps({'package': {
+            'purl': 'pkg:pypi/cryptography@3.1',
+        }}),
+        timeout=_TIMEOUT)
+
+    response = requests.post(
+        _api() + _BASE_QUERY,
+        data=json.dumps({'package': {
+            'purl': 'pkg:pypi/cryptography@3.1.0',
+        }}),
+        timeout=_TIMEOUT)
+
+    self.assert_results_equal(expected.json(), response.json())
+
+    expected = requests.post(
+        _api() + _BASE_QUERY,
+        data=json.dumps({'package': {
+            'purl': 'pkg:nuget/SkiaSharp@2.80.3',
+        }}),
+        timeout=_TIMEOUT)
+
+    response = requests.post(
+        _api() + _BASE_QUERY,
+        data=json.dumps({'package': {
+            'purl': 'pkg:nuget/SkiaSharp@2.80.3.0',
+        }}),
+        timeout=_TIMEOUT)
+
+    self.assert_results_equal(expected.json(), response.json())
+
+    expected = requests.post(
+        _api() + _BASE_QUERY,
+        data=json.dumps({'package': {
+            'purl': 'pkg:pypi/django@4.2',
+        }}),
+        timeout=_TIMEOUT)
+    response = requests.post(
+        _api() + _BASE_QUERY,
+        data=json.dumps({'package': {
+            'purl': 'pkg:pypi/django@4.2.0',
+        }}),
+        timeout=_TIMEOUT)
+
+    self.assert_results_equal(expected.json(), response.json())
+
   def test_query_with_redundant_ecosystem(self):
     """Test purl with redundant ecosystem raises error"""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             "package": {
                 "ecosystem": "PyPI",
@@ -553,7 +603,7 @@ def test_query_with_redundant_ecosystem(self):
   def test_query_with_redundant_version(self):
     """Test purl with redundant version raises error"""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             "version": "0.4.0",
             "package": {
@@ -570,7 +620,7 @@ def test_query_with_redundant_version(self):
   def test_query_with_redundant_package_name(self):
     """Test purl with redundant name raises error"""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps(
             {"package": {
                 "name": "mlflow",
@@ -683,7 +733,7 @@ def test_query_batch(self):
   def test_query_pagination(self):
     """Test query by package."""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps(
             {'package': {
                 'ecosystem': 'PyPI',
@@ -696,7 +746,7 @@ def test_query_pagination(self):
     self.assertIn('next_page_token', result)
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'package': {
                 'ecosystem': 'PyPI',
@@ -716,7 +766,7 @@ def test_query_pagination(self):
   def test_query_package_purl(self):
     """Test query by package (purl)."""
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({'package': {
             'purl': 'pkg:pypi/tensorflow',
         }}),
@@ -726,7 +776,7 @@ def test_query_package_purl(self):
     self.assertIn('next_page_token', result)
 
     response = requests.post(
-        _api() + '/v1/query',
+        _api() + _BASE_QUERY,
         data=json.dumps({
             'package': {
                 'purl': 'pkg:pypi/tensorflow',
@@ -799,7 +849,7 @@ def test_all_possible_queries(self):
     actual_lines = []
     for query in sorted(list(combined_product)):
       response = requests.post(
-          _api() + '/v1/query', data=query, timeout=_TIMEOUT)
+          _api() + _BASE_QUERY, data=query, timeout=_TIMEOUT)
 
       # No possible queries should cause a server error
       self.assertLess(response.status_code, 500)