Repository: google/osv.dev

Issue:
pypi bridge: notify on withdrawn entries



Discussion:

Resolving patch:
diff --git a/docker/worker/worker.py b/docker/worker/worker.py
--- a/docker/worker/worker.py
+++ b/docker/worker/worker.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 """OSV Worker."""
 import argparse
+import datetime
 import json
 import logging
 import os
@@ -167,6 +168,7 @@ def mark_bug_invalid(message):
     logging.error('Bug with source id %s does not exist.', source_id)
     return
 
+  bug.withdrawn = datetime.datetime.utcnow()
   bug.status = osv.BugStatus.INVALID
   bug.put()
 
diff --git a/gcp/functions/pypi/main.py b/gcp/functions/pypi/main.py
--- a/gcp/functions/pypi/main.py
+++ b/gcp/functions/pypi/main.py
@@ -46,8 +46,20 @@ def publish(event, context):
   private_key = serialization.load_pem_private_key(
       data=key_data['key'].encode(), password=None)
 
+  # TODO: Support multiple packages. Currently this only takes the first
+  # package.
+  package_name = None
+  events = []
+  versions = []
   for affected in vulnerability.affected:
-    events = []
+    if affected.package.ecosystem != 'PyPI':
+      continue
+
+    if package_name:
+      if affected.package.name != package_name:
+        continue
+    else:
+      package_name = affected.package.name
 
     for affected_range in affected.ranges:
       if affected_range.type != osv.vulnerability_pb2.Range.ECOSYSTEM:
@@ -59,15 +71,21 @@ def publish(event, context):
         elif evt.fixed:
           events.append({'fixed': evt.fixed})
 
-    request = json.dumps([{
-        'id': vulnerability.id,
-        'project': affected.package.name,
-        'versions': list(affected.versions),
-        'link': f'https://osv.dev/vulnerability/{vulnerability.id}',
-        'aliases': list(vulnerability.aliases),
-        'details': vulnerability.details,
-        'events': events,
-    }]).encode()
+    versions.extend(affected.versions)
+
+  if vulnerability.HasField('withdrawn'):
+    events = []
+    versions = []
+
+  request = json.dumps([{
+      'id': vulnerability.id,
+      'project': package_name,
+      'versions': versions,
+      'link': f'https://osv.dev/vulnerability/{vulnerability.id}',
+      'aliases': list(vulnerability.aliases),
+      'details': vulnerability.details,
+      'events': events,
+  }]).encode()
 
   signature = private_key.sign(
       data=request, signature_algorithm=ECDSA(algorithm=SHA256()))
diff --git a/lib/osv/models.py b/lib/osv/models.py
--- a/lib/osv/models.py
+++ b/lib/osv/models.py
@@ -406,6 +406,9 @@ def _pre_put_hook(self):
 
       self.key = ndb.Key(Bug, key_id)
 
+    if self.withdrawn:
+      self.status = bug.BugStatus.INVALID
+
   def update_from_vulnerability(self, vulnerability):
     """Set fields from vulnerability. Does not set the ID."""
     self.summary = vulnerability.summary
@@ -421,6 +424,8 @@ def update_from_vulnerability(self, vulnerability):
       self.timestamp = vulnerability.published.ToDatetime()
     if vulnerability.HasField('withdrawn'):
       self.withdrawn = vulnerability.withdrawn.ToDatetime()
+    else:
+      self.withdrawn = None
 
     self.aliases = list(vulnerability.aliases)
     self.related = list(vulnerability.related)
@@ -534,9 +539,6 @@ def to_vulnerability(self, include_source=False):
         affected.append(current)
 
     details = self.details
-    if self.status == bug.BugStatus.INVALID:
-      affected = None
-      details = 'INVALID'
 
     if self.last_modified:
       modified = timestamp_pb2.Timestamp()


Test patch:
diff --git a/gcp/api/integration_tests.py b/gcp/api/integration_tests.py
--- a/gcp/api/integration_tests.py
+++ b/gcp/api/integration_tests.py
@@ -82,18 +82,6 @@ class IntegrationTests(unittest.TestCase):
       'summary': 'Heap-double-free in mrb_default_allocf',
   }
 
-  _VULN_2258 = {
-      'published': '2020-12-11T00:00:45.856Z',
-      'schema_version': '1.2.0',
-      'details': 'INVALID',
-      'id': 'OSV-2020-2258',
-      'references': [{
-          'type': 'REPORT',
-          'url': 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=28505',
-      }],
-      'summary': 'Heap-buffer-overflow in grk::t1_part1::T1Part1::decompress',
-  }
-
   _VULN_GO_2020_0004 = {
       'schema_version':
           '1.2.0',
@@ -313,11 +301,6 @@ def test_get_with_multiple(self):
     response = requests.get(_api() + '/v1/vulns/GO-2020-0015')
     self.assert_vuln_equal(self._VULN_GO_2020_0015, response.json())
 
-  def test_get_invalid(self):
-    """Test getting an invalid vulnerability."""
-    response = requests.get(_api() + '/v1/vulns/OSV-2020-2258')
-    self.assert_vuln_equal(self._VULN_2258, response.json())
-
   def test_query_commit(self):
     """Test querying by commit."""
     response = requests.post(
diff --git a/gcp/functions/pypi/main_test.py b/gcp/functions/pypi/main_test.py
--- a/gcp/functions/pypi/main_test.py
+++ b/gcp/functions/pypi/main_test.py
@@ -35,6 +35,7 @@ def _load_test_data(name):
 
 
 _TEST_VULN = _load_test_data("test_vuln.json")
+_TEST_VULN_WITHDRAWN = _load_test_data("test_vuln_withdrawn.json")
 _FAKE_SECRET = _load_test_data("fake_secret.json")
 _FAKE_PUB_KEY = _load_test_data("fake_pub_key.pem")
 
@@ -79,7 +80,34 @@ def test_publish(self):
         b'"link": "https://osv.dev/vulnerability/PYSEC-2021-63", '
         b'"aliases": ["CVE-2020-36242"], '
         b'"details": "In the cryptography package before 3.3.2 for Python, certain sequences of update calls to symmetrically encrypt multi-GB values could result in an integer overflow and buffer overflow, as demonstrated by the Fernet class.", '
-        b'"events": [{"introduced": "3.1"}, {"fixed": "3.3.2"}]'
+        b'"events": [{"introduced": "3.1"}, {"fixed": "3.1.2"}, {"introduced": "3.2"}, {"fixed": "3.3.2"}]'
+        b'}]',
+        headers={
+            'VULN-PUBLIC-KEY-IDENTIFIER': '7ef88907d5bba4c0120f82bfd78386a9'
+                                          'd9328fb5d2d112c473ce52add3e4cd5b',
+            'VULN-PUBLIC-KEY-SIGNATURE': mock.ANY
+        })
+
+    request = self.mock_post.call_args.kwargs['data']
+    signature = self.mock_post.call_args.kwargs['headers'][
+        'VULN-PUBLIC-KEY-SIGNATURE']
+    self._verify_signature(request, signature)
+
+  def test_publish_withdrawn(self):
+    """Test publishing withdrawn vulnerability."""
+    event = {
+        'data': base64.b64encode(_TEST_VULN_WITHDRAWN),
+    }
+
+    main.publish(event, None)
+    self.mock_post.assert_called_once_with(
+        'https://pypi.org/_/vulnerabilities/osv/report',
+        data=b'[{"id": "PYSEC-2021-63", "project": "cryptography", '
+        b'"versions": [], '
+        b'"link": "https://osv.dev/vulnerability/PYSEC-2021-63", '
+        b'"aliases": ["CVE-2020-36242"], '
+        b'"details": "In the cryptography package before 3.3.2 for Python, certain sequences of update calls to symmetrically encrypt multi-GB values could result in an integer overflow and buffer overflow, as demonstrated by the Fernet class.", '
+        b'"events": []'
         b'}]',
         headers={
             'VULN-PUBLIC-KEY-IDENTIFIER': '7ef88907d5bba4c0120f82bfd78386a9'
diff --git a/gcp/functions/pypi/testdata/test_vuln.json b/gcp/functions/pypi/testdata/test_vuln.json
--- a/gcp/functions/pypi/testdata/test_vuln.json
+++ b/gcp/functions/pypi/testdata/test_vuln.json
@@ -38,14 +38,38 @@
               "introduced": "3.1"
             },
             {
-              "fixed": "3.3.2"
+              "fixed": "3.1.2"
             }
           ]
         }
       ],
       "versions": [
         "3.1",
-        "3.1.1",
+        "3.1.1"
+      ],
+      "database_specific": {
+        "source": "https://github.com/pypa/advisory-db/blob/main/vulns/cryptography/PYSEC-2021-63.yaml"
+      }
+    },
+    {
+      "package": {
+        "name": "cryptography",
+        "ecosystem": "PyPI"
+      },
+      "ranges": [
+        {
+          "type": "ECOSYSTEM",
+          "events": [
+            {
+              "introduced": "3.2"
+            },
+            {
+              "fixed": "3.3.2"
+            }
+          ]
+        }
+      ],
+      "versions": [
         "3.2",
         "3.2.1",
         "3.3",