Repository: sonatype-nexus-community/nancy
Pull request #269 (2022-06-14 16:28:39)

Support additional exclude vulnerability files
This change adds the possibility to split `.nancy-ignore` files into multiple ones in a non-breaking way. The "root" file is still kept as `.nancy-ignore` or can be overwritten with the `-x` / `--exclude-vulnerability-file` flag but on top of that a list of additional files can be set via the `-a` / `--additional-exclude-vulnerability-files` flag.

The most common use case for using multiple files is when an organisation wants centrally control some of the common known vulnerabilities across hundreds of repositories by generating a file. It makes sense when there are common 3rd party libraries used in upstream dependencies that are either not updated anymore and the organisation cannot change the usage of them in a short period of time, but it takes a short migration for example. By generating am ignore file via automated means controlled by a central force it can save the headache for the rest of the organisation to handle these one-by-one.

However there can still be known vulnerabilities a team wants to ignore for their own repository that is specific to only that one, so the build pipeline for an organisation could look something like:

```shell
go list -json -deps | nancy sleuth -a .nancy-ignore.local
``` 

I chose to add an additional flag to keep backwards compatibility when someone uses the `-x` / `--exclude-vulnerability-file`. An other approach could be to change that flag to a list instead and rename it to `-x` / `--exclude-vulnerability-files`. Because of this it is still possible to control the root ignore file via that flag and do something like the below example.

```shell
go list -json -deps | nancy sleuth -x .nancy-ignore.generated -a .nancy-ignore.local
```

- [x] If there are changes to user behavior in general, please make sure to update the docs, as well

cc @bhamail / @DarthHater


sonatypecla[bot]: Thanks for the contribution!  Before we can merge this, we need @uvegla to [sign the Sonatype Contributor License Agreement](https://sonatypecla.herokuapp.com/sign-cla).
uvegla: @bhamail @DarthHater Can I please get your feedback / review on this change? Thanks in advance! 🙇🏻 

Patch:
diff --git a/README.md b/README.md
index 881e418..5f978f7 100644
--- a/README.md
+++ b/README.md
@@ -86,11 +86,12 @@ Examples:
   nancy sleuth -p Gopkg.lock --username your_user --token your_token
 
 Flags:
-  -e, --exclude-vulnerability CveListFlag   Comma separated list of CVEs or OSS Index IDs to exclude (default [])
-  -x, --exclude-vulnerability-file string   Path to a file containing newline separated CVEs or OSS Index IDs to be excluded (default "./.nancy-ignore")
-  -h, --help                                help for sleuth
-  -n, --no-color                            indicate output should not be colorized
-  -o, --output string                       Styling for output format. json, json-pretty, text, csv (default "text")
+  -a, --additional-exclude-vulnerability-files strings   Path to additional files containing newline separated CVEs or OSS Index IDs to be excluded
+  -e, --exclude-vulnerability CveListFlag                Comma separated list of CVEs or OSS Index IDs to exclude (default [])
+  -x, --exclude-vulnerability-file string                Path to a file containing newline separated CVEs or OSS Index IDs to be excluded (default "./.nancy-ignore")
+  -h, --help                                             help for sleuth
+  -n, --no-color                                         indicate output should not be colorized
+  -o, --output string                                    Styling for output format. json, json-pretty, text, csv (default "text")
 
 Global Flags:
   -v, -- count                 Set log level, multiple v's is more verbose
@@ -262,13 +263,22 @@ We support exclusion of vulnerability either by CVE-ID (ex: `CVE-2018-20303`) or
 
 ##### Via file
 
-By default if a file named `.nancy-ignore` exists in the same directory that nancy is run it will use it, will no other options need to be passed.
+By default, if a file named `.nancy-ignore` exists in the same directory that nancy is run it will use it - no other options need to be passed.
 
 If you would like to define the path to the file you can use the following
 
 - `nancy sleuth --exclude-vulnerability-file=/path/to/your/exclude-file -p /path/to/your/Gopkg.lock`
 - `go list -json -deps | nancy sleuth --exclude-vulnerability-file=/path/to/your/exclude-file`
 
+If you would like to split up your excludes into multiple files besides your root `.nancy-ignore` you can pass them via the `-a` or `--additional-exclude-vulnerability-files` flags.
+
+- `nancy sleuth --additional-exclude-vulnerability-files=/path/to/first,/path/to/second`
+- `nancy sleuth -a /path/to/first -a /path/to/second`
+
+You can also combine it with the `-x` / `--exclude-vulnerability-file` flag. Nancy merges the additional files on top of the root `.nancy-ignore`.
+
+- `nancy sleuth -x .nancy-ignore.global -a .nancy-ignore.local`
+
 The file format requires each vulnerability that you want to exclude to be on a separate line. Comments are allowed in the file as well to help provide context when needed. See an example file below.
 
 ```
diff --git a/internal/cmd/root.go b/internal/cmd/root.go
index 28bc7d4..62eecbe 100644
--- a/internal/cmd/root.go
+++ b/internal/cmd/root.go
@@ -97,15 +97,16 @@ func cleanUserName(origUsername string) string {
 
 //goland:noinspection GoErrorStringFormat
 var (
-	cfgFile                      string
-	configOssi                   types.Configuration
-	excludeVulnerabilityFilePath string
-	outputFormat                 string
-	logLady                      *logrus.Logger
-	ossiCreator                  ossiServerFactory = ossiFactory{}
-	unixComments                                   = regexp.MustCompile(`#.*$`)
-	untilComment                                   = regexp.MustCompile(`(until=)(.*)`)
-	errStdInInvalid                                = fmt.Errorf("StdIn is invalid or empty. Did you forget to pipe 'go list' to nancy?")
+	cfgFile                                 string
+	configOssi                              types.Configuration
+	excludeVulnerabilityFilePath            string
+	additionalExcludeVulnerabilityFilePaths []string
+	outputFormat                            string
+	logLady                                 *logrus.Logger
+	ossiCreator                             ossiServerFactory = ossiFactory{}
+	unixComments                                              = regexp.MustCompile(`#.*$`)
+	untilComment                                              = regexp.MustCompile(`(until=)(.*)`)
+	errStdInInvalid                                           = fmt.Errorf("StdIn is invalid or empty. Did you forget to pipe 'go list' to nancy?")
 )
 
 //Substitute the _ to .
@@ -286,12 +287,12 @@ func processConfig() (err error) {
 
 	printHeader(!getIsQuiet() && reflect.TypeOf(configOssi.Formatter).String() == "audit.AuditLogTextFormatter")
 
-	// todo: should errors from this call be ignored
+	// todo: should errors from getCVEExcludesFromFile() calls be ignored?
 	_ = getCVEExcludesFromFile(excludeVulnerabilityFilePath)
-	/*	if err = getCVEExcludesFromFile(excludeVulnerabilityFilePath); err != nil {
-			return
-		}
-	*/
+
+	for _, additionalExcludeVulnerabilityFilePath := range additionalExcludeVulnerabilityFilePaths {
+		_ = getCVEExcludesFromFile(additionalExcludeVulnerabilityFilePath)
+	}
 
 	if configOssi.Path != "" {
 		if err = doDepAndParse(ossIndex, configOssi.Path); err != nil {
@@ -305,6 +306,8 @@ func processConfig() (err error) {
 		}
 	}
 
+	deduplicateCveList()
+
 	return
 }
 
@@ -421,6 +424,19 @@ func determineIfLineIsExclusion(ogLine string) error {
 	return nil
 }
 
+func deduplicateCveList() {
+	allKeys := make(map[string]bool)
+	var list []string
+	for _, item := range configOssi.CveList.Cves {
+		if _, value := allKeys[item]; !value {
+			allKeys[item] = true
+			list = append(list, item)
+		}
+	}
+
+	configOssi.CveList.Cves = list
+}
+
 func printHeader(print bool) {
 	if print {
 		figure.NewFigure("Nancy", "larry3d", true).Print()
diff --git a/internal/cmd/sleuth.go b/internal/cmd/sleuth.go
index 7b1b593..79017fe 100644
--- a/internal/cmd/sleuth.go
+++ b/internal/cmd/sleuth.go
@@ -31,6 +31,7 @@ func init() {
 	sleuthCmd.Flags().BoolVarP(&configOssi.NoColor, "no-color", "n", false, "indicate output should not be colorized")
 	sleuthCmd.Flags().VarP(&configOssi.CveList, "exclude-vulnerability", "e", "Comma separated list of CVEs or OSS Index IDs to exclude")
 	sleuthCmd.Flags().StringVarP(&excludeVulnerabilityFilePath, "exclude-vulnerability-file", "x", defaultExcludeFilePath, "Path to a file containing newline separated CVEs or OSS Index IDs to be excluded")
+	sleuthCmd.Flags().StringSliceVarP(&additionalExcludeVulnerabilityFilePaths, "additional-exclude-vulnerability-files", "a", []string{}, "Path to additional files containing newline separated CVEs or OSS Index IDs to be excluded")
 	sleuthCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Styling for output format. json, json-pretty, text, csv")
 }
 


Test patch:
diff --git a/internal/cmd/root_test.go b/internal/cmd/root_test.go
index c044dc0..59618f2 100644
--- a/internal/cmd/root_test.go
+++ b/internal/cmd/root_test.go
@@ -273,6 +273,10 @@ func validateConfigOssi(t *testing.T, expectedConfig types.Configuration, args .
 	}()
 	configOssi = types.Configuration{}
 
+	defer func() {
+		additionalExcludeVulnerabilityFilePaths = []string{}
+	}()
+
 	_, err := executeCommand(rootCmd, args...)
 	assert.Nil(t, err)
 	assert.Equal(t, expectedConfig, configOssi)
diff --git a/internal/cmd/sleuth_test.go b/internal/cmd/sleuth_test.go
index 65622a0..9a059a6 100644
--- a/internal/cmd/sleuth_test.go
+++ b/internal/cmd/sleuth_test.go
@@ -107,6 +107,28 @@ func TestConfigOssi_exclude_vulnerabilities_passed_as_directory_does_not_matter(
 		[]string{sleuthCmd.Use, "--exclude-vulnerability-file=" + dir}...)
 }
 
+func TestConfigOssi_additional_exclude_vulnerabilities(t *testing.T) {
+	rootFile, _ := os.Open(testdataDir + "/normalIgnore")
+	normalAdditionalIgnore, _ := os.Open(testdataDir + "/normalAdditionalIgnore")
+	messyAdditionalIgnore, _ := os.Open(testdataDir + "/messyAdditionalIgnore")
+	validateConfigOssi(t, types.Configuration{CveList: types.CveListFlag{Cves: []string{"CVF-000", "CVF-123", "CVF-9999", "CVX-1016", "CVX-1032", "CVX-1512", "CVX-2064"}}, Formatter: defaultAuditLogFormatter},
+		[]string{sleuthCmd.Use, "--exclude-vulnerability-file=" + rootFile.Name(), "--additional-exclude-vulnerability-files=" + normalAdditionalIgnore.Name() + "," + messyAdditionalIgnore.Name()}...)
+}
+
+func TestConfigOssi_additional_exclude_vulnerabilities_with_empty_base(t *testing.T) {
+	rootFile, _ := os.Open(testdataDir + "/emptyFile")
+	normalAdditionalIgnore, _ := os.Open(testdataDir + "/normalAdditionalIgnore")
+	validateConfigOssi(t, types.Configuration{CveList: types.CveListFlag{Cves: []string{"CVF-000", "CVX-1016", "CVX-1032"}}, Formatter: defaultAuditLogFormatter},
+		[]string{sleuthCmd.Use, "--exclude-vulnerability-file=" + rootFile.Name(), "--additional-exclude-vulnerability-files=" + normalAdditionalIgnore.Name()}...)
+}
+
+func TestConfigOssi_additional_exclude_vulnerabilities_with_empty_additional(t *testing.T) {
+	rootFile, _ := os.Open(testdataDir + "/commented")
+	normalAdditionalIgnore, _ := os.Open(testdataDir + "/emptyFile")
+	validateConfigOssi(t, types.Configuration{CveList: types.CveListFlag{Cves: []string{"CVN-111", "CVN-123", "CVN-543"}}, Formatter: defaultAuditLogFormatter},
+		[]string{sleuthCmd.Use, "--exclude-vulnerability-file=" + rootFile.Name(), "--additional-exclude-vulnerability-files=" + normalAdditionalIgnore.Name()}...)
+}
+
 func TestConfigOssi_exclude_vulnerabilities_does_not_need_to_be_passed_if_default_value_is_used(t *testing.T) {
 	defaultFileName := ".nancy-ignore"
 	err := ioutil.WriteFile(defaultFileName, []byte("DEF-111\nDEF-222"), 0644)
diff --git a/internal/cmd/testdata/sleuth/messyAdditionalIgnore b/internal/cmd/testdata/sleuth/messyAdditionalIgnore
new file mode 100644
index 0000000..36114e8
--- /dev/null
+++ b/internal/cmd/testdata/sleuth/messyAdditionalIgnore
@@ -0,0 +1,5 @@
+
+                CVX-1032
+    CVX-1512
+
+        CVX-2064
diff --git a/internal/cmd/testdata/sleuth/normalAdditionalIgnore b/internal/cmd/testdata/sleuth/normalAdditionalIgnore
new file mode 100644
index 0000000..336f8f5
--- /dev/null
+++ b/internal/cmd/testdata/sleuth/normalAdditionalIgnore
@@ -0,0 +1,4 @@
+CVF-000
+
+CVX-1016
+CVX-1032