Repository: google/osv.dev

Issue:
Support duplicate IDs across DBs
In some cases, IDs may not be unique across databases. 


Discussion:

Resolving patch:
diff --git a/docker/worker/worker.py b/docker/worker/worker.py
--- a/docker/worker/worker.py
+++ b/docker/worker/worker.py
@@ -359,7 +359,7 @@ def _do_update(self, source_repo, repo, vulnerability, relative_path,
         return
 
       bug = osv.Bug(
-          id=vulnerability.id,
+          db_id=vulnerability.id,
           source_id=f'{source_repo.name}:{relative_path}',
           timestamp=osv.utcnow(),
           status=osv.BugStatus.PROCESSED,
diff --git a/gcp/appengine/handlers.py b/gcp/appengine/handlers.py
--- a/gcp/appengine/handlers.py
+++ b/gcp/appengine/handlers.py
@@ -158,14 +158,14 @@ def process_results():
       counters[id_year] = counter
 
     try:
-      cur_id = '{}-{}'.format(counter.key.id(), counter.next_id)
-      logging.info('Allocating OSV-%s.', cur_id)
+      cur_id = 'OSV-{}-{}'.format(counter.key.id(), counter.next_id)
+      logging.info('Allocating %s.', cur_id)
       counter.next_id += 1
 
       # Create the Bug now to avoid races when this cron is run again before the
       # impact task finishes.
       bug = osv.Bug(
-          id=cur_id,
+          db_id=cur_id,
           timestamp=datetime.datetime.utcnow(),
           public=False,
           source_id=key_id,
diff --git a/lib/osv/models.py b/lib/osv/models.py
--- a/lib/osv/models.py
+++ b/lib/osv/models.py
@@ -148,6 +148,9 @@ class Bug(ndb.Model):
   # Very large fake version to use when there is no fix available.
   _NOT_FIXED_SEMVER = '999999.999999.999999'
 
+  # Display ID as used by the source database. The full qualified database that
+  # OSV tracks this as may be different.
+  db_id = ndb.StringProperty()
   # Other IDs this bug is known as.
   aliases = ndb.StringProperty(repeated=True)
   # Related IDs.
@@ -211,6 +214,10 @@ class Bug(ndb.Model):
 
   def id(self):
     """Get the bug ID."""
+    if self.db_id:
+      return self.db_id
+
+    # TODO(ochang): Remove once all existing bugs have IDs migrated.
     if re.match(r'^\d+', self.key.id()):
       return self.OSV_ID_PREFIX + self.key.id()
 
@@ -228,7 +235,11 @@ def repo_url(self):
   @classmethod
   def get_by_id(cls, vuln_id, *args, **kwargs):
     """Overridden get_by_id to handle OSV allocated IDs."""
-    # OSV allocated bug IDs are stored without the prefix.
+    result = cls.query(cls.db_id == vuln_id).get()
+    if result:
+      return result
+
+    # TODO(ochang): Remove once all exsting bugs have IDs migrated.
     if vuln_id.startswith(cls.OSV_ID_PREFIX):
       vuln_id = vuln_id[len(cls.OSV_ID_PREFIX):]
 
@@ -273,8 +284,26 @@ def _pre_put_hook(self):
     if self.source_id:
       self.source, _ = sources.parse_source_id(self.source_id)
 
+    if not self.source:
+      raise ValueError('Source not specified for Bug.')
+
+    if not self.db_id:
+      raise ValueError('DB ID not specified for Bug.')
+
+    if not self.key:
+      source_repo = get_source_repository(self.source)
+      if not source_repo:
+        raise ValueError(f'Invalid source {self.source}')
+
+      if source_repo.db_prefix and self.db_id.startswith(source_repo.db_prefix):
+        key_id = self.db_id
+      else:
+        key_id = f'{self.source}:{self.db_id}'
+
+      self.key = ndb.Key(Bug, key_id)
+
   def update_from_vulnerability(self, vulnerability):
-    """Set fields from vulnerability."""
+    """Set fields from vulnerability. Does not set the ID."""
     self.summary = vulnerability.summary
     self.details = vulnerability.details
     self.reference_url_types = {
@@ -427,6 +456,8 @@ class SourceRepository(ndb.Model):
   versions_from_repo = ndb.BooleanProperty(default=True)
   # HTTP link prefix.
   link = ndb.StringProperty()
+  # DB prefix, if the database allocates its own.
+  db_prefix = ndb.StringProperty()
 
   def ignore_file(self, file_path):
     """Return whether or not we should be ignoring a file."""


Test patch:
diff --git a/docker/importer/importer_test.py b/docker/importer/importer_test.py
--- a/docker/importer/importer_test.py
+++ b/docker/importer/importer_test.py
@@ -57,6 +57,7 @@ def setUp(self):
         type=osv.SourceRepositoryType.GIT,
         id='oss-fuzz',
         name='oss-fuzz',
+        db_prefix='OSV-',
         repo_url='file://' + self.remote_source_repo_path,
         repo_username='',
         ignore_patterns=['.*IGNORE.*'])
@@ -69,7 +70,7 @@ def tearDown(self):
   def test_basic(self, mock_publish):
     """Test basic run."""
     osv.Bug(
-        id='2017-134',
+        db_id='OSV-2017-134',
         affected=['FILE5_29', 'FILE5_30'],
         affected_fuzzy=['5-29', '5-30'],
         affected_ranges=[{
@@ -133,7 +134,7 @@ def test_basic(self, mock_publish):
             source='oss-fuzz',
             type='update')
     ])
-    bug = osv.Bug.get_by_id('2017-134')
+    bug = osv.Bug.get_by_id('OSV-2017-134')
     self.assertEqual(osv.SourceOfTruth.SOURCE_REPO, bug.source_of_truth)
 
     source_repo = osv.SourceRepository.get_by_id('oss-fuzz')
@@ -187,8 +188,14 @@ def test_scheduled_updates(self, mock_publish):
     self.mock_repo.add_file('OSV-2021-1338.yaml', '')
     self.mock_repo.commit('OSV', 'infra@osv.dev')
 
+    osv.SourceRepository(
+        type=osv.SourceRepositoryType.GIT,
+        id='source',
+        name='source',
+        repo_url='file://' + self.remote_source_repo_path,
+        repo_username='').put()
     osv.Bug(
-        id='2021-1337',
+        db_id='OSV-2021-1337',
         project='proj',
         ecosystem='OSS-Fuzz',
         status=1,
@@ -196,7 +203,7 @@ def test_scheduled_updates(self, mock_publish):
         source_of_truth=osv.SourceOfTruth.SOURCE_REPO,
         timestamp=datetime.datetime(2020, 1, 1, 0, 0, 0, 0)).put()
     osv.Bug(
-        id='2021-1338',
+        db_id='OSV-2021-1338',
         project='proj',
         source_id='source:OSV-2021-1338.yaml',
         status=1,
@@ -208,7 +215,7 @@ def test_scheduled_updates(self, mock_publish):
             'type': 'GIT',
         }]).put()
     osv.Bug(
-        id='2021-1339',
+        db_id='OSV-2021-1339',
         project='proj',
         ecosystem='OSS-Fuzz',
         status=1,
@@ -232,7 +239,7 @@ def test_scheduled_updates(self, mock_publish):
             type='update'),
         mock.call(
             'projects/oss-vdb/topics/tasks',
-            allocated_id='2021-1339',
+            allocated_id='OSV-2021-1339',
             data=b'',
             source_id='oss-fuzz:124',
             type='impact'),
@@ -251,7 +258,7 @@ def test_scheduled_updates_already_done(self, mock_publish):
     self.mock_repo.add_file('proj/OSV-2021-1337.yaml', '')
     self.mock_repo.commit('OSV', 'infra@osv.dev')
     osv.Bug(
-        id='2021-1337',
+        db_id='OSV-2021-1337',
         project='proj',
         fixed='',
         status=1,
diff --git a/docker/worker/worker_test.py b/docker/worker/worker_test.py
--- a/docker/worker/worker_test.py
+++ b/docker/worker/worker_test.py
@@ -117,8 +117,10 @@ def setUp(self):
     tests.mock_clone(self, return_value=pygit2.Repository('osv-test'))
     tests.mock_datetime(self)
 
+    osv.SourceRepository(id='oss-fuzz', name='oss-fuzz', db_prefix='OSV-').put()
+
     allocated_bug = osv.Bug(
-        id='2020-1337',
+        db_id='OSV-2020-1337',
         timestamp=datetime.datetime(2020, 1, 1),
         source_id='oss-fuzz:123',
         status=osv.BugStatus.UNPROCESSED,
@@ -126,8 +128,8 @@ def setUp(self):
     allocated_bug.put()
 
     should_be_deleted = osv.AffectedCommit(
-        id='2020-1337-abcd',
-        bug_id='2020-1337',
+        id='OSV-2020-1337-abcd',
+        bug_id='OSV-2020-1337',
         commit='abcd',
         project='project',
         ecosystem='ecosystem',
@@ -139,7 +141,7 @@ def test_basic(self):
     message = mock.Mock()
     message.attributes = {
         'source_id': 'oss-fuzz:123',
-        'allocated_id': '2020-1337',
+        'allocated_id': 'OSV-2020-1337',
     }
 
     regress_result = osv.RegressResult(
@@ -169,6 +171,7 @@ def test_basic(self):
     oss_fuzz.process_impact_task('oss-fuzz:123', message)
     self.assertDictEqual(
         {
+            'db_id': 'OSV-2020-1337',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -221,7 +224,7 @@ def test_basic(self):
             'semver_fixed_indexes': [],
             'source': 'oss-fuzz',
         },
-        ndb.Key(osv.Bug, '2020-1337').get()._to_dict())
+        ndb.Key(osv.Bug, 'OSV-2020-1337').get()._to_dict())
 
     affected_commits = list(osv.AffectedCommit.query())
     for commit in affected_commits:
@@ -240,7 +243,7 @@ def test_range(self):
     message = mock.Mock()
     message.attributes = {
         'source_id': 'oss-fuzz:123',
-        'allocated_id': '2020-1337',
+        'allocated_id': 'OSV-2020-1337',
     }
 
     regress_result = osv.RegressResult(
@@ -271,6 +274,7 @@ def test_range(self):
     oss_fuzz.process_impact_task('oss-fuzz:123', message)
     self.assertDictEqual(
         {
+            'db_id': 'OSV-2020-1337',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -319,7 +323,7 @@ def test_range(self):
             'semver_fixed_indexes': [],
             'source': 'oss-fuzz',
         },
-        ndb.Key(osv.Bug, '2020-1337').get()._to_dict())
+        ndb.Key(osv.Bug, 'OSV-2020-1337').get()._to_dict())
 
     affected_commits = list(osv.AffectedCommit.query())
     for commit in affected_commits:
@@ -340,7 +344,7 @@ def test_fixed_range_too_long(self):
     message = mock.Mock()
     message.attributes = {
         'source_id': 'oss-fuzz:123',
-        'allocated_id': '2020-1337',
+        'allocated_id': 'OSV-2020-1337',
     }
 
     regress_result = osv.RegressResult(
@@ -371,6 +375,7 @@ def test_fixed_range_too_long(self):
     oss_fuzz.process_impact_task('oss-fuzz:123', message)
     self.assertDictEqual(
         {
+            'db_id': 'OSV-2020-1337',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -419,7 +424,7 @@ def test_fixed_range_too_long(self):
             'semver_fixed_indexes': [],
             'source': 'oss-fuzz',
         },
-        ndb.Key(osv.Bug, '2020-1337').get()._to_dict())
+        ndb.Key(osv.Bug, 'OSV-2020-1337').get()._to_dict())
 
     affected_commits = list(osv.AffectedCommit.query())
     for commit in affected_commits:
@@ -442,7 +447,7 @@ def test_zero_regression_range(self):
     message = mock.Mock()
     message.attributes = {
         'source_id': 'oss-fuzz:123',
-        'allocated_id': '2020-1337',
+        'allocated_id': 'OSV-2020-1337',
     }
 
     regress_result = osv.RegressResult(
@@ -472,6 +477,7 @@ def test_zero_regression_range(self):
     oss_fuzz.process_impact_task('oss-fuzz:123', message)
     self.assertDictEqual(
         {
+            'db_id': 'OSV-2020-1337',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -517,7 +523,7 @@ def test_zero_regression_range(self):
             'semver_fixed_indexes': [],
             'source': 'oss-fuzz',
         },
-        ndb.Key(osv.Bug, '2020-1337').get()._to_dict())
+        ndb.Key(osv.Bug, 'OSV-2020-1337').get()._to_dict())
 
     affected_commits = list(osv.AffectedCommit.query())
     for commit in affected_commits:
@@ -536,7 +542,7 @@ def test_simplify_range(self):
     message = mock.Mock()
     message.attributes = {
         'source_id': 'oss-fuzz:123',
-        'allocated_id': '2020-1337',
+        'allocated_id': 'OSV-2020-1337',
     }
 
     regress_result = osv.RegressResult(
@@ -568,6 +574,7 @@ def test_simplify_range(self):
     oss_fuzz.process_impact_task('oss-fuzz:123', message)
     self.assertDictEqual(
         {
+            'db_id': 'OSV-2020-1337',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -620,14 +627,14 @@ def test_simplify_range(self):
             'semver_fixed_indexes': [],
             'source': 'oss-fuzz',
         },
-        ndb.Key(osv.Bug, '2020-1337').get()._to_dict())
+        ndb.Key(osv.Bug, 'OSV-2020-1337').get()._to_dict())
 
   def test_not_fixed(self):
     """Test not fixed bug."""
     message = mock.Mock()
     message.attributes = {
         'source_id': 'oss-fuzz:123',
-        'allocated_id': '2020-1337',
+        'allocated_id': 'OSV-2020-1337',
     }
 
     regress_result = osv.RegressResult(
@@ -646,6 +653,7 @@ def test_not_fixed(self):
     oss_fuzz.process_impact_task('oss-fuzz:123', message)
     self.assertDictEqual(
         {
+            'db_id': 'OSV-2020-1337',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -688,7 +696,7 @@ def test_not_fixed(self):
             'semver_fixed_indexes': [],
             'source': 'oss-fuzz',
         },
-        ndb.Key(osv.Bug, '2020-1337').get()._to_dict())
+        ndb.Key(osv.Bug, 'OSV-2020-1337').get()._to_dict())
 
     affected_commits = list(osv.AffectedCommit.query())
     for commit in affected_commits:
@@ -736,9 +744,10 @@ def setUp(self):
 
   def test_mark_bug_invalid(self):
     """Test mark_bug_invalid."""
-    osv.Bug(id='2021-1', source_id='oss-fuzz:1337').put()
-    osv.AffectedCommit(bug_id='2021-1').put()
-    osv.AffectedCommit(bug_id='2021-1').put()
+    osv.SourceRepository(id='oss-fuzz', name='oss-fuzz', db_prefix='OSV-').put()
+    osv.Bug(db_id='OSV-2021-1', source_id='oss-fuzz:1337').put()
+    osv.AffectedCommit(bug_id='OSV-2021-1').put()
+    osv.AffectedCommit(bug_id='OSV-2021-1').put()
 
     message = mock.Mock()
     message.attributes = {
@@ -748,7 +757,7 @@ def test_mark_bug_invalid(self):
     }
 
     worker.mark_bug_invalid(message)
-    bug = ndb.Key(osv.Bug, '2021-1').get()
+    bug = ndb.Key(osv.Bug, 'OSV-2021-1').get()
     self.assertEqual(osv.BugStatus.INVALID, bug.status)
 
     commits = list(osv.AffectedCommit.query())
@@ -846,26 +855,27 @@ def setUp(self):
         type=osv.SourceRepositoryType.GIT,
         id='source',
         name='source',
+        db_prefix='BLAH-',
         repo_url='file://' + self.remote_source_repo_path,
         editable=True,
         repo_username='')
     self.source_repo.put()
 
     osv.Bug(
-        id='BLAH-123',
+        db_id='BLAH-123',
         project='blah.com/package',
         ecosystem='golang',
         source_id='source:BLAH-123.yaml',
         source_of_truth=osv.SourceOfTruth.SOURCE_REPO).put()
     osv.Bug(
-        id='BLAH-124',
+        db_id='BLAH-124',
         regressed='eefe8ec3f1f90d0e684890e810f3f21e8500a4cd',
         project='blah.com/package',
         ecosystem='golang',
         source_id='source:BLAH-124.yaml',
         source_of_truth=osv.SourceOfTruth.SOURCE_REPO).put()
     osv.Bug(
-        id='BLAH-125',
+        db_id='BLAH-125',
         regressed='eefe8ec3f1f90d0e684890e810f3f21e8500a4cd',
         fixed='8d8242f545e9cec3e6d0d2e3f5bde8be1c659735',
         project='blah.com/package',
@@ -873,7 +883,7 @@ def setUp(self):
         source_id='source:BLAH-125.yaml',
         source_of_truth=osv.SourceOfTruth.SOURCE_REPO).put()
     osv.Bug(
-        id='BLAH-127',
+        db_id='BLAH-127',
         project='blah.com/package',
         ecosystem='golang',
         source_id='source:BLAH-127.yaml',
@@ -907,6 +917,7 @@ def test_update(self):
 
     self.assertDictEqual(
         {
+            'db_id': 'BLAH-123',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -998,6 +1009,7 @@ def test_update_add_fix(self):
 
     self.assertDictEqual(
         {
+            'db_id': 'BLAH-124',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -1086,6 +1098,7 @@ def test_update_no_introduced(self):
 
     self.assertDictEqual(
         {
+            'db_id': 'BLAH-127',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -1164,6 +1177,7 @@ def test_update_new(self):
 
     self.assertDictEqual(
         {
+            'db_id': 'BLAH-126',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -1343,6 +1357,7 @@ def test_update_pypi(self):
 
     self.assertDictEqual(
         {
+            'db_id': 'PYSEC-123',
             'aliases': [],
             'purl': None,
             'related': [],
@@ -1412,7 +1427,7 @@ def test_update_pypi(self):
             'semver_fixed_indexes': [],
             'source': 'source',
         },
-        osv.Bug.get_by_id('PYSEC-123')._to_dict())
+        ndb.Key(osv.Bug, 'source:PYSEC-123').get()._to_dict())
 
     affected_commits = list(osv.AffectedCommit.query())
     self.assertCountEqual([
@@ -1442,6 +1457,8 @@ def test_update_bucket(self):
 
     self.assertDictEqual(
         {
+            'db_id':
+                'GO-2021-0085',
             'aliases': ['CVE-2019-16884'],
             'purl':
                 None,
@@ -1521,6 +1538,7 @@ def test_update_bucket(self):
         osv.Bug.get_by_id('GO-2021-0085')._to_dict())
     self.assertDictEqual(
         {
+            'db_id': 'GO-2021-0087',
             'aliases': ['CVE-2019-19921'],
             'purl': None,
             'related': [],
diff --git a/gcp/api/integration_tests.py b/gcp/api/integration_tests.py
--- a/gcp/api/integration_tests.py
+++ b/gcp/api/integration_tests.py
@@ -161,9 +161,6 @@ def remove_modified(self, vuln):
 
   def test_get(self):
     """Test getting a vulnerability."""
-    response = requests.get(_api() + '/v1/vulns/2020-744')
-    self.assert_vuln_equal(self._VULN_744, response.json())
-
     response = requests.get(_api() + '/v1/vulns/OSV-2020-744')
     self.assert_vuln_equal(self._VULN_744, response.json())