Repository: sonatype-nexus-community/nancy
Pull request #96 (2020-03-16 17:52:34)

Remove dupes from list, add tests for that
A few users have been scanning go projects with LOTS of sub modules, and they've been running into issues because some of the projects have duplicate dependencies, and this fails validation with IQ Server. So, let's solve that!

This pull request makes the following changes:
* Add a removeDuplicates function to `parse.go`
* Add two test cases for checking that de-duplication is happening

This is the first time I've gone "Hey Golang, why don't you have a Set!". Been writing too much TypeScript :)

cc @bhamail / @DarthHater / @zendern / @fitzoh 


bhamail: Maybe for a different PR, but I did a little testing, and it appears these tests no longer use `testdata/Gopkg.lock`.

Patch:
diff --git a/parse/parse.go b/parse/parse.go
index 67fd8d4..ffbabbd 100644
--- a/parse/parse.go
+++ b/parse/parse.go
@@ -15,9 +15,12 @@ package parse
 
 import (
 	"bufio"
-	"github.com/sonatype-nexus-community/nancy/types"
+	"fmt"
 	"os"
 	"strings"
+
+	. "github.com/sonatype-nexus-community/nancy/logger"
+	"github.com/sonatype-nexus-community/nancy/types"
 )
 
 var goModDependencyCriteria = func(s []string) bool {
@@ -31,6 +34,10 @@ func GoList(stdIn *bufio.Scanner) (deps types.ProjectList, err error) {
 	for stdIn.Scan() {
 		parseSpaceSeparatedDependency(stdIn, &deps, goListDependencyCriteria)
 	}
+
+	LogLady.Info("Beginning to dedupe list")
+	deps = removeDuplicates(&deps)
+
 	return deps, nil
 }
 
@@ -46,6 +53,10 @@ func GoSum(path string) (deps types.ProjectList, err error) {
 	for scanner.Scan() {
 		parseSpaceSeparatedDependency(scanner, &deps, goModDependencyCriteria)
 	}
+
+	LogLady.Info("Beginning to dedupe list")
+	deps = removeDuplicates(&deps)
+
 	return deps, nil
 }
 
@@ -56,3 +67,19 @@ func parseSpaceSeparatedDependency(scanner *bufio.Scanner, deps *types.ProjectLi
 		deps.Projects = append(deps.Projects, types.Projects{Name: s[0], Version: s[1]})
 	}
 }
+
+func removeDuplicates(deps *types.ProjectList) (newDeps types.ProjectList) {
+	encountered := map[string]bool{}
+
+	for _, v := range deps.Projects {
+		if encountered[fmt.Sprintf("%s-%s", v.Name, v.Version)] == true {
+			LogLady.WithField("dep", v).Debug("Found duplicate dependency, eliminating it")
+		} else {
+			LogLady.WithField("dep", v).Debug("Unique dependency, adding it")
+			encountered[fmt.Sprintf("%s-%s", v.Name, v.Version)] = true
+			newDeps.Projects = append(newDeps.Projects, v)
+		}
+	}
+
+	return
+}


Test patch:
diff --git a/parse/parse_test.go b/parse/parse_test.go
index 3a036be..fda96ff 100644
--- a/parse/parse_test.go
+++ b/parse/parse_test.go
@@ -20,7 +20,18 @@ import (
 )
 
 func TestGoSum(t *testing.T) {
-	deps, err := GoSum("go.sum")
+	deps, err := GoSum("../testdata/parse/go.sum")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(deps.Projects) != 10 {
+		t.Error(deps)
+	}
+}
+
+func TestGoSumDuplicates(t *testing.T) {
+	deps, err := GoSum("../testdata/parse/gosumdupes.sum")
 	if err != nil {
 		t.Error(err)
 	}
@@ -60,8 +71,43 @@ golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb`
 	}
 }
 
+func TestGoListAllDuplicates(t *testing.T) {
+	goListMAllOutput := `github.com/sonatype-nexus-community/nancy
+github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7
+github.com/BurntSushi/toml v0.3.1
+github.com/davecgh/go-spew v1.1.0
+github.com/dgraph-io/badger v1.5.5-0.20181004181505-439fd464b155
+github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102
+github.com/dustin/go-humanize v1.0.0
+github.com/dustin/go-humanize v1.0.0
+github.com/dustin/go-humanize v1.0.0
+github.com/golang/protobuf v1.2.0
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e
+github.com/pkg/errors v0.8.0
+github.com/pkg/errors v0.8.0
+github.com/pmezard/go-difflib v1.0.0
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
+github.com/stretchr/objx v0.1.0
+github.com/stretchr/objx v0.1.0
+github.com/stretchr/objx v0.1.0
+github.com/stretchr/testify v1.3.0
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4
+golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb`
+	scanner := bufio.NewScanner(strings.NewReader(goListMAllOutput))
+
+	deps, err := GoList(scanner)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(deps.Projects) != 16 {
+		t.Error(deps)
+	}
+}
+
 func TestGoSumError(t *testing.T) {
-	_, err := GoSum("go.notsum")
+	_, err := GoSum("../testdata/parse/go.notsum")
 	if err == nil {
 		t.Error(err)
 	}
diff --git a/parse/Gopkg.lock b/testdata/parse/Gopkg.lock
similarity index 100%
rename from parse/Gopkg.lock
rename to testdata/parse/Gopkg.lock
diff --git a/parse/go.sum b/testdata/parse/go.sum
similarity index 100%
rename from parse/go.sum
rename to testdata/parse/go.sum
diff --git a/testdata/parse/gosumdupes.sum b/testdata/parse/gosumdupes.sum
new file mode 100644
index 0000000..44dc225
--- /dev/null
+++ b/testdata/parse/gosumdupes.sum
@@ -0,0 +1,24 @@
+github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo=
+github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg=
+github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
+github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg=
+github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb h1:pf3XwC90UUdNPYWZdFjhGBE7DUFuK3Ct1zWmZ65QN30=
+golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=