Repository: G-Rath/osv-detector
Pull request #68 (2022-03-12 23:52:51)

feat: add `--cache-all-databases` flag
This is useful on it's own, and also ends up giving `--list-ecosystems` a use since it'll be able to just call `KnownEcosystems`.

Will do some deduplication with `numberOfLockfileParsers` in a followup.



Patch:
diff --git a/README.md b/README.md
index 74075c8..a046151 100644
--- a/README.md
+++ b/README.md
@@ -57,6 +57,16 @@ This requires the detector to have successfully downloaded a copy of ecosystem
 databases required to check the packages discovered during parsing at least
 once.
 
+You can have the detector cache the databases for all known ecosystems supported
+by the detector for later offline use with the `--cache-all-databases`:
+
+```shell
+osv-detector --cache-all-databases
+```
+
+This can be useful if you're planning to run the detector over a number of
+lockfiles in bulk.
+
 ### Auxiliary output commands
 
 The detector supports a few auxiliary commands that have it output information
diff --git a/internal/lockfile/ecosystems.go b/internal/lockfile/ecosystems.go
new file mode 100644
index 0000000..94e8f4e
--- /dev/null
+++ b/internal/lockfile/ecosystems.go
@@ -0,0 +1,12 @@
+package lockfile
+
+func KnownEcosystems() []Ecosystem {
+	return []Ecosystem{
+		NpmEcosystem,
+		CargoEcosystem,
+		BundlerEcosystem,
+		ComposerEcosystem,
+		GoEcosystem,
+		PipEcosystem,
+	}
+}
diff --git a/main.go b/main.go
index d426bdf..5d535ef 100644
--- a/main.go
+++ b/main.go
@@ -109,12 +109,19 @@ func loadEcosystemDatabases(ecosystems []internal.Ecosystem, offline bool) OSVDa
 	return dbs
 }
 
+func cacheAllEcosystemDatabases() {
+	ecosystems := lockfile.KnownEcosystems()
+
+	loadEcosystemDatabases(ecosystems, false)
+}
+
 func main() {
 	offline := flag.Bool("offline", false, "Update the OSV database")
 	parseAs := flag.String("parse-as", "", "Name of a supported lockfile to use to determine how to parse the given file")
 	printVersion := flag.Bool("version", false, "Print version information")
 	listEcosystems := flag.Bool("list-ecosystems", false, "List all the ecosystems present in the loaded OSV database")
 	listPackages := flag.Bool("list-packages", false, "List all the packages that were parsed from the given file")
+	cacheAllDatabases := flag.Bool("cache-all-databases", false, "Cache all the known ecosystem databases for offline use")
 
 	flag.Parse()
 
@@ -123,6 +130,11 @@ func main() {
 		os.Exit(0)
 	}
 
+	if *cacheAllDatabases {
+		cacheAllEcosystemDatabases()
+		os.Exit(0)
+	}
+
 	pathToLockOrDirectory := flag.Arg(0)
 
 	packages, err := lockfile.Parse(pathToLockOrDirectory, *parseAs)


Test patch:
diff --git a/internal/lockfile/ecosystems_test.go b/internal/lockfile/ecosystems_test.go
new file mode 100644
index 0000000..6a42b5b
--- /dev/null
+++ b/internal/lockfile/ecosystems_test.go
@@ -0,0 +1,55 @@
+package lockfile_test
+
+import (
+	"io/ioutil"
+	"osv-detector/internal/lockfile"
+	"strings"
+	"testing"
+)
+
+func numberOfLockfileParsers(t *testing.T) int {
+	t.Helper()
+
+	directories, err := ioutil.ReadDir(".")
+
+	if err != nil {
+		t.Fatalf("unable to read current directory: ")
+	}
+
+	count := 0
+
+	for _, directory := range directories {
+		if strings.HasPrefix(directory.Name(), "parse-") &&
+			!strings.HasSuffix(directory.Name(), "_test.go") {
+			count++
+		}
+	}
+
+	return count
+}
+
+func TestKnownEcosystems(t *testing.T) {
+	t.Parallel()
+
+	expectedCount := numberOfLockfileParsers(t)
+
+	// npm, yarn, and pnpm all use the same ecosystem,
+	// so "ignore" those parsers in the count
+	expectedCount -= 2
+
+	ecosystems := lockfile.KnownEcosystems()
+
+	if knownCount := len(ecosystems); knownCount != expectedCount {
+		t.Errorf("Expected to know about %d ecosystems, but knew about %d", expectedCount, knownCount)
+	}
+
+	uniq := make(map[lockfile.Ecosystem]int)
+
+	for _, ecosystem := range ecosystems {
+		uniq[ecosystem]++
+
+		if uniq[ecosystem] > 1 {
+			t.Errorf(`Ecosystem "%s" was listed more than once`, ecosystem)
+		}
+	}
+}