Repository: google/osv-scanner
Pull request #379 (2023-05-12 01:07:14)

Remove --hash from version in requirements.txt
This removes everything after the whitespace of the version to also
catch the other per-requirement options --global-option and
--config-settings and any future options that may be added

https://pip.pypa.io/en/stable/reference/requirements-file-format/#per-requirement-options

fixes: https://github.com/google/osv-scanner/issues/369

While addressing this i noticed the pip documentation example for --hash
used line continuations which weren't currently supported by this parser
so i've added support for this






Patch:
diff --git a/pkg/lockfile/fixtures/pip/line-continuation.txt b/pkg/lockfile/fixtures/pip/line-continuation.txt
new file mode 100644
index 0000000..af78779
--- /dev/null
+++ b/pkg/lockfile/fixtures/pip/line-continuation.txt
@@ -0,0 +1,17 @@
+# unescaped
+foo==\
+\
+ \
+  \
+1.2.3
+
+# escaped, a literal backslash for some reason
+bar == 4.5\\
+.6
+
+# comments are stripped only after line continuations are processed
+baz == 7.8.9 # \
+baz == 1.2.3
+
+# continue to end
+qux == 10.11.12\
diff --git a/pkg/lockfile/fixtures/pip/with-per-requirement-options.txt b/pkg/lockfile/fixtures/pip/with-per-requirement-options.txt
new file mode 100644
index 0000000..cf672f4
--- /dev/null
+++ b/pkg/lockfile/fixtures/pip/with-per-requirement-options.txt
@@ -0,0 +1,12 @@
+boto3==1.26.121 --hash=sha256:f87d694c351eba1dfd19b5bef5892a1047e7adb09c57c2c00049de209a8ab55d
+foo == 1.0.0
+
+# from https://pip.pypa.io/en/stable/topics/secure-installs/#hash-checking-mode
+
+FooProject == 1.2 \
+  --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 \
+  --hash=sha256:486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8e5a6c65260e9cb8a7
+
+# from https://pip.pypa.io/en/stable/reference/requirements-file-format/#influencing-the-build-system
+
+BarProject >= 1.2 --global-option="--no-user-cfg"
diff --git a/pkg/lockfile/parse-requirements-txt.go b/pkg/lockfile/parse-requirements-txt.go
index d7564a4..18e7b2f 100644
--- a/pkg/lockfile/parse-requirements-txt.go
+++ b/pkg/lockfile/parse-requirements-txt.go
@@ -38,12 +38,11 @@ func parseLine(line string) PackageDetails {
 	}
 
 	if constraint != "" {
-		splitted := strings.Split(line, constraint)
-
-		name = strings.TrimSpace(splitted[0])
+		unprocessedName, unprocessedVersion, _ := strings.Cut(line, constraint)
+		name = strings.TrimSpace(unprocessedName)
 
 		if constraint != "!=" {
-			version = strings.TrimSpace(splitted[1])
+			version, _, _ = strings.Cut(strings.TrimSpace(unprocessedVersion), " ")
 		}
 	}
 
@@ -70,7 +69,7 @@ func normalizedRequirementName(name string) string {
 	// per https://www.python.org/dev/peps/pep-0503/#normalized-names
 	name = cachedregexp.MustCompile(`[-_.]+`).ReplaceAllString(name, "-")
 	name = strings.ToLower(name)
-	name = strings.Split(name, "[")[0]
+	name, _, _ = strings.Cut(name, "[")
 
 	return name
 }
@@ -93,6 +92,14 @@ func isNotRequirementLine(line string) bool {
 		strings.HasPrefix(line, "/")
 }
 
+func isLineContinuation(line string) bool {
+	// checks that the line ends with an odd number of back slashes,
+	// meaning the last one isn't escaped
+	var re = cachedregexp.MustCompile(`([^\\]|^)(\\{2})*\\$`)
+
+	return re.MatchString(line)
+}
+
 func ParseRequirementsTxt(pathToLockfile string) ([]PackageDetails, error) {
 	return parseRequirementsTxt(pathToLockfile, map[string]struct{}{})
 }
@@ -106,9 +113,18 @@ func parseRequirementsTxt(pathToLockfile string, requiredAlready map[string]stru
 	defer file.Close()
 
 	scanner := bufio.NewScanner(file)
-
 	for scanner.Scan() {
-		line := removeComments(scanner.Text())
+		line := scanner.Text()
+
+		for isLineContinuation(line) {
+			line = strings.TrimSuffix(line, "\\")
+
+			if scanner.Scan() {
+				line += scanner.Text()
+			}
+		}
+
+		line = removeComments(line)
 
 		if ar := strings.TrimPrefix(line, "-r "); ar != line {
 			ar = filepath.Join(filepath.Dir(pathToLockfile), ar)


Test patch:
diff --git a/pkg/lockfile/parse-requirements-txt_test.go b/pkg/lockfile/parse-requirements-txt_test.go
index 845bb15..8206732 100644
--- a/pkg/lockfile/parse-requirements-txt_test.go
+++ b/pkg/lockfile/parse-requirements-txt_test.go
@@ -469,6 +469,7 @@ func TestParseRequirementsTxt_DuplicateROptions(t *testing.T) {
 		},
 	})
 }
+
 func TestParseRequirementsTxt_CyclicRSelf(t *testing.T) {
 	t.Parallel()
 
@@ -493,6 +494,7 @@ func TestParseRequirementsTxt_CyclicRSelf(t *testing.T) {
 		},
 	})
 }
+
 func TestParseRequirementsTxt_CyclicRComplex(t *testing.T) {
 	t.Parallel()
 
@@ -523,3 +525,77 @@ func TestParseRequirementsTxt_CyclicRComplex(t *testing.T) {
 		},
 	})
 }
+
+func TestParseRequirementsTxt_WithPerRequirementOptions(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-per-requirement-options.txt")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "boto3",
+			Version:   "1.26.121",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "foo",
+			Version:   "1.0.0",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "fooproject",
+			Version:   "1.2",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "barproject",
+			Version:   "1.2",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+	})
+}
+
+func TestParseRequirementsTxt_LineContinuation(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/line-continuation.txt")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "foo",
+			Version:   "1.2.3",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "bar",
+			Version:   "4.5\\\\",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "baz",
+			Version:   "7.8.9",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "qux",
+			Version:   "10.11.12",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+	})
+}