Repository: google/osv-scanner
Pull request #656 (2023-11-15 03:13:46)

feat: add support for comparing CRAN versions
Part of #642

See https://github.com/G-Rath/osv-detector/pull/235 for the journey I went on with R for this

codecov-commenter: ## [Codecov](https://app.codecov.io/gh/google/osv-scanner/pull/656?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Comparison is base [(`007b8e4`)](https://app.codecov.io/gh/google/osv-scanner/commit/007b8e4be4d1366dce16602712b6969bceee6dd5?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google) 78.14% compared to head [(`91c91eb`)](https://app.codecov.io/gh/google/osv-scanner/pull/656?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google) 78.19%.


<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #656      +/-   ##
==========================================
+ Coverage   78.14%   78.19%   +0.05%     
==========================================
  Files          81       82       +1     
  Lines        5866     5894      +28     
==========================================
+ Hits         4584     4609      +25     
- Misses       1082     1084       +2     
- Partials      200      201       +1     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/google/osv-scanner/pull/656?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google).

G-Rath: also just a note: from what I can tell this is actually more "R" rather than "CRAN" aka this is exactly the same for the Bioconductor ecosystem, but I've not worried about that at all because 1. `semantic` is internal and 2. there are advisories for any Bioconductor package.

I think it should be pretty easy if just slightly "special-case-y" to handle if/when there's Bioconductor advisories (this will also apply to #668)

Patch:
diff --git a/.github/workflows/lint-action/action.yml b/.github/workflows/lint-action/action.yml
index a0322ae..245842e 100644
--- a/.github/workflows/lint-action/action.yml
+++ b/.github/workflows/lint-action/action.yml
@@ -22,6 +22,6 @@ runs:
       uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
       with:
         # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-        version: v1.51.1
+        version: v1.54.2
         # https://github.com/golangci/golangci-lint-action/issues/135
-        skip-pkg-cache: true
\ No newline at end of file
+        skip-pkg-cache: true
diff --git a/internal/semantic/fixtures/cran-versions-generated.txt b/internal/semantic/fixtures/cran-versions-generated.txt
new file mode 100644
index 0000000..5ec1f92
--- /dev/null
+++ b/internal/semantic/fixtures/cran-versions-generated.txt
@@ -0,0 +1,57 @@
+0.1.0 < 0.1.1
+0.1.1 < 1.0.0
+1.0.0 < 1.1.0
+1.1.0 < 1.2.0
+1.2.0 < 1.4.1
+0.9.12 < 0.9.13
+0.9.13 < 0.9.14
+0.9.14 < 0.9.15
+0.9.15 < 0.9.16
+0.9.16 < 0.9.17
+0.9.17 < 0.9.18
+0.9.18 < 0.9.19
+0.9.19 < 0.9.20
+0.9.20 < 0.9.21
+0.9.21 < 0.9.22
+0.9.22 < 1.0
+1.0 < 1.1
+1.1 < 1.2
+1.2 < 1.3
+1.3 < 1.4
+1.4 < 1.5
+1.5 < 1.6
+1.6 < 1.6.1
+1.6.1 < 1.7.0
+1.7.0 < 1.7.1
+1.7.1 < 1.7.2
+1.7.2 < 1.7.3
+1.7.3 < 1.8.0
+1.8.0 < 1.8.1
+1.8.1 < 1.8.2
+1.8.2 < 1.8.3
+1.8.3 < 1.8.4
+1.8.4 < 1.8.5
+1.8.5 < 1.8.6
+1.8.6 < 1.8.7
+0.7.1 < 1.0.0
+1.0.0 < 1.0.1
+1.0.1 < 1.1.1
+1.1.1 < 1.1.2
+1.1.2 < 1.2.1
+1.2.1 < 1.2.2
+0.1.1 < 0.2.0
+0.2.0 < 0.2.1
+0.2.1 < 1.0.0
+0.2 < 0.4
+0.4 < 0.5
+0.5 < 0.6
+0.6 < 0.7
+0.7 < 0.8
+0.8 < 0.9
+0.9 < 1.0
+1.2 < 1.4
+1.6 < 1.7
+1.7 < 1.8
+1.8 < 1.8.0
+1.8.1 < 1.9
+1.9 < 1.9.0
diff --git a/internal/semantic/fixtures/cran-versions.txt b/internal/semantic/fixtures/cran-versions.txt
new file mode 100644
index 0000000..1a8d08b
--- /dev/null
+++ b/internal/semantic/fixtures/cran-versions.txt
@@ -0,0 +1,34 @@
+0.01 < 0.1-0
+0.01.0 = 0.1-0
+0.9 < 0.75
+
+1.0-0 < 1.1-0
+
+# ---
+0.1.0.0 < 0.2.0
+0.1.0 < 0.2.0.0
+0.1.0 < 0.1.1.0
+0.1.0 < 0.1.0.0
+0.1.0 > 0.0.1.0
+
+# https://cran.r-project.org/src/contrib/Archive/abctools/
+0.1-2 < 0.2-2
+0.2 < 0.3-2
+1.0 < 1.0.1
+1.0.2 < 1.0.3
+
+# https://cran.r-project.org/src/contrib/Archive/AntibodyTiters/
+0.1.4 < 0.1.18
+
+# https://cran.r-project.org/src/contrib/Archive/AdaptGauss/
+1.0 < 1.1.0
+1.5 < 1.5.4
+
+# https://cran.r-project.org/src/contrib/Archive/AcceptanceSampling/
+0.1-1 < 0.1-4
+1.0-0 < 1.0-1
+
+# https://cran.r-project.org/src/contrib/Archive/DHARMa/
+0.1.0 < 0.1.0.0
+0.3.2.0 > 0.3.2
+0.3.2.0 < 0.3.3.0
diff --git a/internal/semantic/parse.go b/internal/semantic/parse.go
index 78f2f83..e19ec60 100644
--- a/internal/semantic/parse.go
+++ b/internal/semantic/parse.go
@@ -44,6 +44,8 @@ func Parse(str string, ecosystem Ecosystem) (Version, error) {
 		return parseSemverVersion(str), nil
 	case "ConanCenter":
 		return parseSemverVersion(str), nil
+	case "CRAN":
+		return parseCRANVersion(str), nil
 	}
 
 	return nil, fmt.Errorf("%w %s", ErrUnsupportedEcosystem, ecosystem)
diff --git a/internal/semantic/version-cran.go b/internal/semantic/version-cran.go
new file mode 100644
index 0000000..19fa37c
--- /dev/null
+++ b/internal/semantic/version-cran.go
@@ -0,0 +1,54 @@
+package semantic
+
+import (
+	"math/big"
+	"strings"
+)
+
+// CRANVersion is the representation of a version of a package that is held
+// in the CRAN ecosystem (https://cran.r-project.org/).
+//
+// A version is a sequence of at least two non-negative integers separated by
+// either a period or a dash.
+//
+// See https://astrostatistics.psu.edu/su07/R/html/base/html/package_version.html
+type CRANVersion struct {
+	components Components
+}
+
+func (v CRANVersion) Compare(w CRANVersion) int {
+	if diff := v.components.Cmp(w.components); diff != 0 {
+		return diff
+	}
+
+	// versions are only equal if they also have the same number of components,
+	// otherwise the longer one is considered greater
+	if len(v.components) == len(w.components) {
+		return 0
+	}
+
+	if len(v.components) > len(w.components) {
+		return 1
+	}
+
+	return -1
+}
+
+func (v CRANVersion) CompareStr(str string) int {
+	return v.Compare(parseCRANVersion(str))
+}
+
+func parseCRANVersion(str string) CRANVersion {
+	// dashes and periods have the same weight, so we can just normalize to periods
+	parts := strings.Split(strings.ReplaceAll(str, "-", "."), ".")
+
+	components := make(Components, 0, len(parts))
+
+	for _, s := range parts {
+		v, _ := new(big.Int).SetString(s, 10)
+
+		components = append(components, v)
+	}
+
+	return CRANVersion{components}
+}
diff --git a/pkg/models/vulnerability.go b/pkg/models/vulnerability.go
index db7a0e1..1f87b5f 100644
--- a/pkg/models/vulnerability.go
+++ b/pkg/models/vulnerability.go
@@ -10,8 +10,8 @@ import (
 //
 // See: https://ossf.github.io/osv-schema/#affectedpackage-field
 type Package struct {
-	Ecosystem Ecosystem `json:"ecosystem" yaml:"ecosystem"`
-	Name      string    `json:"name" yaml:"name"`
+	Ecosystem Ecosystem `json:"ecosystem"      yaml:"ecosystem"`
+	Name      string    `json:"name"           yaml:"name"`
 	Purl      string    `json:"purl,omitempty" yaml:"purl,omitempty"`
 }
 
@@ -27,19 +27,19 @@ type Package struct {
 //
 // See: https://ossf.github.io/osv-schema/#affectedrangesevents-fields
 type Event struct {
-	Introduced   string `json:"introduced,omitempty" yaml:"introduced,omitempty"`
-	Fixed        string `json:"fixed,omitempty" yaml:"fixed,omitempty"`
+	Introduced   string `json:"introduced,omitempty"    yaml:"introduced,omitempty"`
+	Fixed        string `json:"fixed,omitempty"         yaml:"fixed,omitempty"`
 	LastAffected string `json:"last_affected,omitempty" yaml:"last_affected,omitempty"`
-	Limit        string `json:"limit,omitempty" yaml:"limit,omitempty"`
+	Limit        string `json:"limit,omitempty"         yaml:"limit,omitempty"`
 }
 
 // Range describes the affected range of given version for a specific package.
 //
 // See: https://ossf.github.io/osv-schema/#affectedranges-field
 type Range struct {
-	Type             RangeType              `json:"type" yaml:"type"`
-	Events           []Event                `json:"events" yaml:"events"`
-	Repo             string                 `json:"repo,omitempty" yaml:"repo,omitempty"`
+	Type             RangeType              `json:"type"                        yaml:"type"`
+	Events           []Event                `json:"events"                      yaml:"events"`
+	Repo             string                 `json:"repo,omitempty"              yaml:"repo,omitempty"`
 	DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty" yaml:"database_specific,omitempty"`
 }
 
@@ -48,7 +48,7 @@ type Range struct {
 //
 // See: https://ossf.github.io/osv-schema/#severity-field
 type Severity struct {
-	Type  SeverityType `json:"type" yaml:"type"`
+	Type  SeverityType `json:"type"  yaml:"type"`
 	Score string       `json:"score" yaml:"score"`
 }
 
@@ -57,11 +57,11 @@ type Severity struct {
 //
 // See: https://ossf.github.io/osv-schema/#affected-fields
 type Affected struct {
-	Package           Package                `json:"package,omitempty" yaml:"package,omitempty"`
-	Severity          []Severity             `json:"severity,omitempty" yaml:"severity,omitempty"`
-	Ranges            []Range                `json:"ranges,omitempty" yaml:"ranges,omitempty"`
-	Versions          []string               `json:"versions,omitempty" yaml:"versions,omitempty"`
-	DatabaseSpecific  map[string]interface{} `json:"database_specific,omitempty" yaml:"database_specific,omitempty"`
+	Package           Package                `json:"package,omitempty"            yaml:"package,omitempty"`
+	Severity          []Severity             `json:"severity,omitempty"           yaml:"severity,omitempty"`
+	Ranges            []Range                `json:"ranges,omitempty"             yaml:"ranges,omitempty"`
+	Versions          []string               `json:"versions,omitempty"           yaml:"versions,omitempty"`
+	DatabaseSpecific  map[string]interface{} `json:"database_specific,omitempty"  yaml:"database_specific,omitempty"`
 	EcosystemSpecific map[string]interface{} `json:"ecosystem_specific,omitempty" yaml:"ecosystem_specific,omitempty"`
 }
 
@@ -92,7 +92,7 @@ func (a Affected) MarshalJSON() ([]byte, error) {
 // See: https://ossf.github.io/osv-schema/#references-field
 type Reference struct {
 	Type ReferenceType `json:"type" yaml:"type"`
-	URL  string        `json:"url" yaml:"url"`
+	URL  string        `json:"url"  yaml:"url"`
 }
 
 // Credit gives credit for the discovery, confirmation, patch, or other events
@@ -100,8 +100,8 @@ type Reference struct {
 //
 // See: https://ossf.github.io/osv-schema/#credits-fields
 type Credit struct {
-	Name    string     `json:"name" yaml:"name"`
-	Type    CreditType `json:"type,omitempty" yaml:"type,omitempty"`
+	Name    string     `json:"name"              yaml:"name"`
+	Type    CreditType `json:"type,omitempty"    yaml:"type,omitempty"`
 	Contact []string   `json:"contact,omitempty" yaml:"contact,omitempty"`
 }
 
@@ -110,19 +110,19 @@ type Credit struct {
 // The full documentation for the schema is available at
 // https://ossf.github.io/osv-schema.
 type Vulnerability struct {
-	SchemaVersion    string                 `json:"schema_version,omitempty" yaml:"schema_version,omitempty"`
-	ID               string                 `json:"id" yaml:"id"`
-	Modified         time.Time              `json:"modified" yaml:"modified"`
-	Published        time.Time              `json:"published,omitempty" yaml:"published,omitempty"`
-	Withdrawn        time.Time              `json:"withdrawn,omitempty" yaml:"withdrawn,omitempty"`
-	Aliases          []string               `json:"aliases,omitempty" yaml:"aliases,omitempty"`
-	Related          []string               `json:"related,omitempty" yaml:"related,omitempty"`
-	Summary          string                 `json:"summary,omitempty" yaml:"summary,omitempty"`
-	Details          string                 `json:"details,omitempty" yaml:"details,omitempty"`
-	Affected         []Affected             `json:"affected,omitempty" yaml:"affected,omitempty"`
-	Severity         []Severity             `json:"severity,omitempty" yaml:"severity,omitempty"`
-	References       []Reference            `json:"references,omitempty" yaml:"references,omitempty"`
-	Credits          []Credit               `json:"credits,omitempty" yaml:"credits,omitempty"`
+	SchemaVersion    string                 `json:"schema_version,omitempty"    yaml:"schema_version,omitempty"`
+	ID               string                 `json:"id"                          yaml:"id"`
+	Modified         time.Time              `json:"modified"                    yaml:"modified"`
+	Published        time.Time              `json:"published,omitempty"         yaml:"published,omitempty"`
+	Withdrawn        time.Time              `json:"withdrawn,omitempty"         yaml:"withdrawn,omitempty"`
+	Aliases          []string               `json:"aliases,omitempty"           yaml:"aliases,omitempty"`
+	Related          []string               `json:"related,omitempty"           yaml:"related,omitempty"`
+	Summary          string                 `json:"summary,omitempty"           yaml:"summary,omitempty"`
+	Details          string                 `json:"details,omitempty"           yaml:"details,omitempty"`
+	Affected         []Affected             `json:"affected,omitempty"          yaml:"affected,omitempty"`
+	Severity         []Severity             `json:"severity,omitempty"          yaml:"severity,omitempty"`
+	References       []Reference            `json:"references,omitempty"        yaml:"references,omitempty"`
+	Credits          []Credit               `json:"credits,omitempty"           yaml:"credits,omitempty"`
 	DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty" yaml:"database_specific,omitempty"`
 }
 
diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go
index bc334e7..38ed2f7 100644
--- a/pkg/osvscanner/osvscanner.go
+++ b/pkg/osvscanner/osvscanner.go
@@ -247,6 +247,10 @@ func queryDetermineVersions(repoDir string) (*osv.DetermineVersionResponse, erro
 				// results with our regular git commit scanning.
 				return filepath.SkipDir
 			}
+			if _, ok := vendoredLibNames[strings.ToLower(info.Name())]; ok {
+				// Ignore nested vendored libraries, as they can cause bad matches.
+				return filepath.SkipDir
+			}
 
 			return nil
 		}
diff --git a/scripts/generators/generate-cran-versions.R b/scripts/generators/generate-cran-versions.R
new file mode 100755
index 0000000..10013dd
--- /dev/null
+++ b/scripts/generators/generate-cran-versions.R
@@ -0,0 +1,183 @@
+#!/usr/bin/env Rscript
+
+install.packages("jsonlite", repos = 'https://cran.r-project.org')
+
+library(utils)
+library(jsonlite)
+
+# An array of version comparisons that are known to be unsupported and so
+# should be commented out in the generated fixture.
+#
+# Generally this is because the native implementation has a suspected bug
+# that causes the comparison to return incorrect results, and so supporting
+# such comparisons in the detector would in fact be wrong.
+UNSUPPORTED_COMPARISONS <- c()
+
+download_cran_db <- function() {
+  url <- "https://osv-vulnerabilities.storage.googleapis.com/CRAN/all.zip"
+  dest <- "cran-db.zip"
+  download.file(url, dest, method = "auto")
+}
+
+extract_packages_with_versions <- function(osvs) {
+  result <- list()
+
+  for (osv in osvs) {
+    for (affected in osv$affected) {
+      package <- affected$package$name
+
+      if (!(package %in% names(result))) {
+        result[[package]] <- list()
+      }
+
+      for (version in affected$versions) {
+        tryCatch(
+          {
+            as.package_version(version)
+            result[[package]] <- c(result[[package]], version)
+          },
+          error = function(e) {
+            cat(sprintf("skipping invalid version %s for %s\n", version, package))
+          }
+        )
+      }
+    }
+  }
+
+  # deduplicate and sort the versions for each package
+  for (package in names(result)) {
+    result[[package]] <- sort(numeric_version(unique(result[[package]])))
+  }
+
+  return(result)
+}
+
+is_unsupported_comparison <- function(line) {
+  line %in% UNSUPPORTED_COMPARISONS
+}
+
+uncomment <- function(line) {
+  if (startsWith(line, "#")) {
+    return(substr(line, 2, nchar(line)))
+  }
+  if (startsWith(line, "//")) {
+    return(substr(line, 3, nchar(line)))
+  }
+  return(line)
+}
+
+compare <- function(v1, relate, v2) {
+  ops <- list('<' = function(result) result < 0,
+              '=' = function(result) result == 0,
+              '>' = function(result) result > 0)
+
+  return(ops[[relate]](compareVersion(v1, v2)))
+}
+
+compare_versions <- function(lines, select="all") {
+  has_any_failed <- FALSE
+
+  for (line in lines) {
+    line <- trimws(line)
+
+    if (line == "" || grepl("^#", line) || grepl("^//", line)) {
+      maybe_unsupported <- trimws(uncomment(line))
+
+      if (is_unsupported_comparison(maybe_unsupported)) {
+        cat(sprintf("\033[96mS\033[0m: \033[93m%s\033[0m\n", maybe_unsupported))
+      }
+      next
+    }
+
+    parts <- strsplit(trimws(line), " ")[[1]]
+    v1 <- parts[1]
+    op <- parts[2]
+    v2 <- parts[3]
+
+    r <- compare(v1, op, v2)
+
+    if (!r) {
+      has_any_failed <- TRUE
+    }
+
+    if (select == "failures" && r) {
+      next
+    }
+
+    if (select == "successes" && !r) {
+      next
+    }
+
+    color <- ifelse(r, '\033[92m', '\033[91m')
+    rs <- ifelse(r, "T", "F")
+    cat(sprintf("%s%s\033[0m: \033[93m%s\033[0m\n", color, rs, line))
+  }
+  return(has_any_failed)
+}
+
+compare_versions_in_file <- function(filepath, select="all") {
+  lines <- readLines(filepath)
+  return(compare_versions(lines, select))
+}
+
+generate_version_compares <- function(versions) {
+  comparisons <- character()
+
+  for (i in seq_along(versions)) {
+    if (i == 1) {
+      next
+    }
+
+    comparison <- sprintf("%s < %s", versions[i - 1], versions[i])
+
+    if (is_unsupported_comparison(trimws(comparison))) {
+      comparison <- paste("#", comparison)
+    }
+
+    comparisons <- c(comparisons, comparison)
+  }
+
+  return(comparisons)
+}
+
+generate_package_compares <- function(packages) {
+  comparisons <- character()
+
+  for (package in names(packages)) {
+    versions <- packages[[package]]
+    comparisons <- c(comparisons, generate_version_compares(versions))
+  }
+
+  # return unique comparisons
+  return(unique(comparisons))
+}
+
+fetch_packages_versions <- function() {
+  download_cran_db()
+  osvs <- list()
+
+  with_zip <- unzip("cran-db.zip", list = TRUE)
+
+  for (fname in with_zip$Name) {
+    osv <- jsonlite::fromJSON(unzip("cran-db.zip", files = fname, exdir = tempdir()), simplifyDataFrame = FALSE)
+    osvs <- c(osvs, list(osv))
+  }
+
+  return(extract_packages_with_versions(osvs))
+}
+
+outfile <- "internal/semantic/fixtures/cran-versions-generated.txt"
+
+packs <- fetch_packages_versions()
+writeLines(generate_package_compares(packs), outfile, sep = "\n")
+cat("\n")
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show <- Sys.getenv("VERSION_GENERATOR_PRINT", "failures")
+
+did_any_fail <- compare_versions_in_file(outfile, show)
+
+if (did_any_fail) {
+  q(status = 1)
+}
diff --git a/scripts/run_lints.sh b/scripts/run_lints.sh
index 42ee567..0bc3e76 100755
--- a/scripts/run_lints.sh
+++ b/scripts/run_lints.sh
@@ -2,4 +2,4 @@
 
 set -ex
 
-go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.1 run ./... --max-same-issues 0
+go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2 run ./... --max-same-issues 0


Test patch:
diff --git a/internal/semantic/compare_test.go b/internal/semantic/compare_test.go
index de20024..99ed53b 100644
--- a/internal/semantic/compare_test.go
+++ b/internal/semantic/compare_test.go
@@ -217,6 +217,14 @@ func TestVersion_Compare_Ecosystems(t *testing.T) {
 			name: "Debian",
 			file: "debian-versions-generated.txt",
 		},
+		{
+			name: "CRAN",
+			file: "cran-versions.txt",
+		},
+		{
+			name: "CRAN",
+			file: "cran-versions-generated.txt",
+		},
 	}
 	for _, tt := range tests {
 		tt := tt
diff --git a/internal/semantic/parse_test.go b/internal/semantic/parse_test.go
index 6072aca..8fcea3f 100644
--- a/internal/semantic/parse_test.go
+++ b/internal/semantic/parse_test.go
@@ -13,6 +13,9 @@ func TestParse(t *testing.T) {
 
 	ecosystems := lockfile.KnownEcosystems()
 
+	// todo: remove once CRAN is supported by lockfile
+	ecosystems = append(ecosystems, "CRAN")
+
 	for _, ecosystem := range ecosystems {
 		_, err := semantic.Parse("", ecosystem)
 
@@ -33,6 +36,9 @@ func TestMustParse(t *testing.T) {
 
 	ecosystems := lockfile.KnownEcosystems()
 
+	// todo: remove once CRAN is supported by lockfile
+	ecosystems = append(ecosystems, "CRAN")
+
 	for _, ecosystem := range ecosystems {
 		semantic.MustParse("", ecosystem)
 	}