Repository: google/osv-scanner
Pull request #261 (2023-03-02 23:20:20)

fix: deduplicate packages that appear multiple times in `Pipenv.lock` files
TIL this is apparently a thing


Patch:
diff --git a/pkg/lockfile/fixtures/pipenv/multiple-packages.json b/pkg/lockfile/fixtures/pipenv/multiple-packages.json
new file mode 100644
index 0000000..c78b3d9
--- /dev/null
+++ b/pkg/lockfile/fixtures/pipenv/multiple-packages.json
@@ -0,0 +1,97 @@
+{
+  "_meta": {
+      "hash": {
+          "sha256": "3809f002b3b58ea51dc4258d4e00b7ebae6b28c46d7cb25ab8d4540a7c5f0d7b"
+      },
+      "pipfile-spec": 6,
+      "requires": {
+          "python_version": "3.8"
+      },
+      "sources": [
+          {
+              "name": "pypi",
+              "url": "https://pypi.org/simple",
+              "verify_ssl": true
+          }
+      ]
+  },
+  "default": {
+      "itsdangerous": {
+          "hashes": [
+              "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
+              "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"
+          ],
+          "index": "pypi",
+          "version": "==2.1.2"
+      },
+      "pluggy": {
+          "hashes": [],
+          "markers": "python_version >= '3.6'",
+          "version": "==1.0.1"
+      },
+      "markupsafe": {
+          "hashes": [
+              "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003",
+              "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88",
+              "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5",
+              "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7",
+              "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a",
+              "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603",
+              "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1",
+              "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135",
+              "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247",
+              "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6",
+              "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601",
+              "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77",
+              "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02",
+              "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e",
+              "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63",
+              "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f",
+              "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980",
+              "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b",
+              "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812",
+              "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff",
+              "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96",
+              "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1",
+              "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925",
+              "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a",
+              "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6",
+              "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e",
+              "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f",
+              "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4",
+              "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f",
+              "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3",
+              "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c",
+              "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a",
+              "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417",
+              "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a",
+              "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a",
+              "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37",
+              "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452",
+              "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933",
+              "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a",
+              "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"
+          ],
+          "markers": "python_version >= '3.7'",
+          "version": "==2.1.1"
+      }
+  },
+  "develop": {
+      "itsdangerous": {
+          "hashes": [
+              "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
+              "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"
+          ],
+          "index": "pypi",
+          "version": "==2.1.2"
+      },
+      "pluggy": {
+          "hashes": [
+              "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
+              "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+          ],
+          "markers": "python_version >= '3.6'",
+          "version": "==1.0.0"
+      }
+  }
+}
diff --git a/pkg/lockfile/parse-pipenv-lock.go b/pkg/lockfile/parse-pipenv-lock.go
index ddc03d6..f39ae1f 100644
--- a/pkg/lockfile/parse-pipenv-lock.go
+++ b/pkg/lockfile/parse-pipenv-lock.go
@@ -32,34 +32,33 @@ func ParsePipenvLock(pathToLockfile string) ([]PackageDetails, error) {
 		return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err)
 	}
 
-	packages := make(
-		[]PackageDetails,
-		0,
-		// len cannot return negative numbers, but the types can't reflect that
-		uint64(len(parsedLockfile.Packages))+uint64(len(parsedLockfile.PackagesDev)),
-	)
+	packages := make(map[string]PackageDetails)
 
 	for name, pipenvPackage := range parsedLockfile.Packages {
 		if pipenvPackage.Version == "" {
 			continue
 		}
 
-		packages = append(packages, PackageDetails{
+		version := pipenvPackage.Version[2:]
+
+		packages[name+"@"+version] = PackageDetails{
 			Name:      name,
-			Version:   pipenvPackage.Version[2:],
+			Version:   version,
 			Ecosystem: PipenvEcosystem,
 			CompareAs: PipenvEcosystem,
-		})
+		}
 	}
 
 	for name, pipenvPackage := range parsedLockfile.PackagesDev {
-		packages = append(packages, PackageDetails{
+		version := pipenvPackage.Version[2:]
+
+		packages[name+"@"+version] = PackageDetails{
 			Name:      name,
-			Version:   pipenvPackage.Version[2:],
+			Version:   version,
 			Ecosystem: PipenvEcosystem,
 			CompareAs: PipenvEcosystem,
-		})
+		}
 	}
 
-	return packages, nil
+	return pkgDetailsMapToSlice(packages), nil
 }


Test patch:
diff --git a/pkg/lockfile/parse-pipenv-lock_test.go b/pkg/lockfile/parse-pipenv-lock_test.go
index acbeae3..8a70660 100644
--- a/pkg/lockfile/parse-pipenv-lock_test.go
+++ b/pkg/lockfile/parse-pipenv-lock_test.go
@@ -124,6 +124,43 @@ func TestParsePipenvLock_TwoPackagesAlt(t *testing.T) {
 	})
 }
 
+func TestParsePipenvLock_MultiplePackages(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParsePipenvLock("fixtures/pipenv/multiple-packages.json")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "itsdangerous",
+			Version:   "2.1.2",
+			Ecosystem: lockfile.PipenvEcosystem,
+			CompareAs: lockfile.PipenvEcosystem,
+		},
+		{
+			Name:      "pluggy",
+			Version:   "1.0.1",
+			Ecosystem: lockfile.PipenvEcosystem,
+			CompareAs: lockfile.PipenvEcosystem,
+		},
+		{
+			Name:      "pluggy",
+			Version:   "1.0.0",
+			Ecosystem: lockfile.PipenvEcosystem,
+			CompareAs: lockfile.PipenvEcosystem,
+		},
+		{
+			Name:      "markupsafe",
+			Version:   "2.1.1",
+			Ecosystem: lockfile.PipenvEcosystem,
+			CompareAs: lockfile.PipenvEcosystem,
+		},
+	})
+}
+
 func TestParsePipenvLock_PackageWithoutVersion(t *testing.T) {
 	t.Parallel()