Repository: google/osv.dev

Issue:
Exclude files under common vendored directory names in determineversion API
**Is your feature request related to a problem? Please describe.**
OSV determineversion API now is starting to exclude some files based on it's file path, which require more complexity on users building tools to send data to this API, and requires them to keep up to date with our exclusion list.

**Describe the solution you'd like**
By doing the exclusion server side we can keep the API filtering and the indexed data filtering in sync, and reduces the complexity on the users of the API.

**Describe alternatives you've considered**
N/A

**Additional context**
N/A


Discussion:

Resolving patch:
diff --git a/gcp/api/server.py b/gcp/api/server.py
--- a/gcp/api/server.py
+++ b/gcp/api/server.py
@@ -67,6 +67,24 @@
 # This should match the number in the indexer
 _BUCKET_SIZE = 512
 
+# This needs to be kept in sync with
+# https://github.com/google/osv.dev/blob/
+# 666a43e6ae7690fbfa283e9a6f0b08a986be4d32/
+# docker/indexer/stages/processing/processing.go#L77
+_VENDORED_LIB_NAMES = frozenset((
+    '3rdparty',
+    'dep',
+    'deps',
+    'thirdparty',
+    'third-party',
+    'third_party',
+    'libs',
+    'external',
+    'externals',
+    'vendor',
+    'vendored',
+))
+
 # Prefix for the
 _TAG_PREFIX = "refs/tags/"
 
@@ -252,6 +270,20 @@ class QueryContext:
   total_responses: ResponsesCount
 
 
+def should_skip_bucket(path: str) -> bool:
+  """Returns whether or not the given file path should be skipped for the
+  determineversions bucket computation."""
+  if not path:
+    return False
+
+  # Check for a nested vendored directory, as this could mess with results. The
+  # API expects the file path passed to be relative to the potential library
+  # path, so any vendored library names found here would imply it's a nested
+  # vendored library.
+  components = path.split('/')
+  return any(c in _VENDORED_LIB_NAMES for c in components)
+
+
 def process_buckets(
     file_results: List[osv.FileResult]) -> List[osv.RepoIndexBucket]:
   """
@@ -261,6 +293,9 @@ def process_buckets(
   buckets: list[list[bytes]] = [[] for _ in range(_BUCKET_SIZE)]
 
   for fr in file_results:
+    if should_skip_bucket(fr.path):
+      continue
+
     buckets[int.from_bytes(fr.hash[:2], byteorder='big') % _BUCKET_SIZE].append(
         fr.hash)
 


Test patch:
diff --git a/gcp/api/server_test.py b/gcp/api/server_test.py
--- a/gcp/api/server_test.py
+++ b/gcp/api/server_test.py
@@ -16,6 +16,7 @@
 import unittest
 
 from server import _match_purl
+from server import should_skip_bucket
 from packageurl import PackageURL
 
 
@@ -45,6 +46,26 @@ def test_match_purl(self):
           _match_purl(PackageURL.from_string(a), PackageURL.from_string(b)),
           a + ' == ' + b)
 
+  def test_should_skip_bucket(self):
+    """Test should_skip_bucket."""
+    test_cases = [
+        (None, False),
+        ('', False),
+        ('/a', False),
+        ('a', False),
+        ('a/b', False),
+        ('a/b/c', False),
+        ('/a/b/c', False),
+        ('/a/3rdparty/c', True),
+        ('/a/third_party/c/d', True),
+        ('/third_party/c/d', True),
+        ('vendor/c/d', True),
+        ('/a/3rdpartyy/c', False),
+    ]
+
+    for path, expected in test_cases:
+      self.assertEqual(expected, should_skip_bucket(path))
+
 
 if __name__ == '__main__':
   unittest.main()