Repository: sonatype-nexus-community/nancy
Pull request #98 (2020-03-17 18:53:45)

Dedupe, the saga
Turns out my deduplication worked perfectly in #96 , but the problem is Golang creates it's own Duplicates! Since dependencies can be referred to by their github.com location or gopkg.in redirect location, one could have the same dependency, and it wouldn't bubble up until much later! This works fine on OSS Index but fails on IQ on validation.

This pull request makes the following changes:
* Moves the dedupe logic into the `mod` `ExtractPurlsFromManifest` and `ExtractPurlsFromManifestForIQ` methods
* Adds test
* Deletes old code!

cc @bhamail / @DarthHater / @zendern / @fitzoh 



Patch:
diff --git a/packages/mod.go b/packages/mod.go
index 5c6d05e..64e9251 100644
--- a/packages/mod.go
+++ b/packages/mod.go
@@ -13,11 +13,15 @@
 // limitations under the License.
 package packages
 
-import "strings"
-import "github.com/sonatype-nexus-community/nancy/types"
-import "fmt"
-import "github.com/sonatype-nexus-community/nancy/customerrors"
-import "os"
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/sonatype-nexus-community/nancy/customerrors"
+	. "github.com/sonatype-nexus-community/nancy/logger"
+	"github.com/sonatype-nexus-community/nancy/types"
+)
 
 type Mod struct {
 	ProjectList types.ProjectList
@@ -34,7 +38,10 @@ func (m Mod) ExtractPurlsFromManifest() (purls []string) {
 			purls = append(purls, purl)
 		}
 	}
-	return purls
+
+	purls = removeDuplicates(purls)
+
+	return
 }
 
 func (m Mod) ExtractPurlsFromManifestForIQ() (purls []string) {
@@ -46,7 +53,9 @@ func (m Mod) ExtractPurlsFromManifestForIQ() (purls []string) {
 			purls = append(purls, purl)
 		}
 	}
-	return purls
+	purls = removeDuplicates(purls)
+
+	return
 }
 
 func (m Mod) CheckExistenceOfManifest() bool {
@@ -55,3 +64,19 @@ func (m Mod) CheckExistenceOfManifest() bool {
 	}
 	return true
 }
+
+func removeDuplicates(purls []string) (dedupedPurls []string) {
+	encountered := map[string]bool{}
+
+	for _, v := range purls {
+		if encountered[v] == true {
+			LogLady.WithField("dep", v).Debug("Found duplicate dependency, eliminating it")
+		} else {
+			LogLady.WithField("dep", v).Debug("Unique dependency, adding it")
+			encountered[v] = true
+			dedupedPurls = append(dedupedPurls, v)
+		}
+	}
+
+	return
+}
diff --git a/parse/parse.go b/parse/parse.go
index ffbabbd..64bdb83 100644
--- a/parse/parse.go
+++ b/parse/parse.go
@@ -15,11 +15,9 @@ package parse
 
 import (
 	"bufio"
-	"fmt"
 	"os"
 	"strings"
 
-	. "github.com/sonatype-nexus-community/nancy/logger"
 	"github.com/sonatype-nexus-community/nancy/types"
 )
 
@@ -35,9 +33,6 @@ func GoList(stdIn *bufio.Scanner) (deps types.ProjectList, err error) {
 		parseSpaceSeparatedDependency(stdIn, &deps, goListDependencyCriteria)
 	}
 
-	LogLady.Info("Beginning to dedupe list")
-	deps = removeDuplicates(&deps)
-
 	return deps, nil
 }
 
@@ -54,9 +49,6 @@ func GoSum(path string) (deps types.ProjectList, err error) {
 		parseSpaceSeparatedDependency(scanner, &deps, goModDependencyCriteria)
 	}
 
-	LogLady.Info("Beginning to dedupe list")
-	deps = removeDuplicates(&deps)
-
 	return deps, nil
 }
 
@@ -67,19 +59,3 @@ func parseSpaceSeparatedDependency(scanner *bufio.Scanner, deps *types.ProjectLi
 		deps.Projects = append(deps.Projects, types.Projects{Name: s[0], Version: s[1]})
 	}
 }
-
-func removeDuplicates(deps *types.ProjectList) (newDeps types.ProjectList) {
-	encountered := map[string]bool{}
-
-	for _, v := range deps.Projects {
-		if encountered[fmt.Sprintf("%s-%s", v.Name, v.Version)] == true {
-			LogLady.WithField("dep", v).Debug("Found duplicate dependency, eliminating it")
-		} else {
-			LogLady.WithField("dep", v).Debug("Unique dependency, adding it")
-			encountered[fmt.Sprintf("%s-%s", v.Name, v.Version)] = true
-			newDeps.Projects = append(newDeps.Projects, v)
-		}
-	}
-
-	return
-}


Test patch:
diff --git a/packages/mod_test.go b/packages/mod_test.go
index 13f681f..9f6e5a7 100644
--- a/packages/mod_test.go
+++ b/packages/mod_test.go
@@ -14,8 +14,9 @@
 package packages
 
 import (
-	"github.com/sonatype-nexus-community/nancy/types"
 	"testing"
+
+	"github.com/sonatype-nexus-community/nancy/types"
 )
 
 var testGoSumName = "go.sum"
@@ -36,6 +37,25 @@ func getProjectList() (projectList types.ProjectList) {
 	return projectList
 }
 
+func getProjectListDuplicates() (projectList types.ProjectList) {
+	appendProject("github.com/AndreasBriese/bbloom", "", &projectList)
+	appendProject("gopkg.in/BurntSushi/toml", "v0.3.1", &projectList)
+	appendProject("github.com/dgraph-io/badger", "v1.5.4", &projectList)
+	appendProject("github.com/dgraph-io/badger", "v1.5.4", &projectList)
+	appendProject("gopkg.in/dgraph-io/badger", "v1.5.4", &projectList)
+	appendProject("github.com/dgryski/go-farm", "", &projectList)
+	appendProject("github.com/golang/protobuf", "v1.2.0", &projectList)
+	appendProject("github.com/golang/protobuf", "v1.2.0", &projectList)
+	appendProject("github.com/golang/protobuf", "v1.2.0", &projectList)
+	appendProject("github.com/logrusorgru/aurora", "", &projectList)
+	appendProject("github.com/pkg/errors", "v0.8.0", &projectList)
+	appendProject("github.com/shopspring/decimal", "1.1.0", &projectList)
+	appendProject("golang.org/x/net", "", &projectList)
+	appendProject("golang.org/x/sys", "", &projectList)
+
+	return projectList
+}
+
 func appendProject(name string, version string, projectList *types.ProjectList) {
 	projectList.Projects = append(projectList.Projects, types.Projects{Name: name, Version: version})
 }
@@ -64,3 +84,18 @@ func TestModExtractPurlsFromManifest(t *testing.T) {
 		t.Error(result)
 	}
 }
+
+func TestModExtractPurlsFromManifestDuplicates(t *testing.T) {
+	var err error
+	mod := Mod{}
+	mod.GoSumPath = testGoSumName
+	mod.ProjectList = getProjectListDuplicates()
+	if err != nil {
+		t.Error(err)
+	}
+
+	result := mod.ExtractPurlsFromManifest()
+	if len(result) != 5 {
+		t.Error(result)
+	}
+}
diff --git a/parse/parse_test.go b/parse/parse_test.go
index fda96ff..080ffbe 100644
--- a/parse/parse_test.go
+++ b/parse/parse_test.go
@@ -30,17 +30,6 @@ func TestGoSum(t *testing.T) {
 	}
 }
 
-func TestGoSumDuplicates(t *testing.T) {
-	deps, err := GoSum("../testdata/parse/gosumdupes.sum")
-	if err != nil {
-		t.Error(err)
-	}
-
-	if len(deps.Projects) != 10 {
-		t.Error(deps)
-	}
-}
-
 func TestGoListAll(t *testing.T) {
 	goListMAllOutput := `github.com/sonatype-nexus-community/nancy
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7
@@ -71,41 +60,6 @@ golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb`
 	}
 }
 
-func TestGoListAllDuplicates(t *testing.T) {
-	goListMAllOutput := `github.com/sonatype-nexus-community/nancy
-github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7
-github.com/BurntSushi/toml v0.3.1
-github.com/davecgh/go-spew v1.1.0
-github.com/dgraph-io/badger v1.5.5-0.20181004181505-439fd464b155
-github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102
-github.com/dustin/go-humanize v1.0.0
-github.com/dustin/go-humanize v1.0.0
-github.com/dustin/go-humanize v1.0.0
-github.com/golang/protobuf v1.2.0
-github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e
-github.com/pkg/errors v0.8.0
-github.com/pkg/errors v0.8.0
-github.com/pmezard/go-difflib v1.0.0
-github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
-github.com/stretchr/objx v0.1.0
-github.com/stretchr/objx v0.1.0
-github.com/stretchr/objx v0.1.0
-github.com/stretchr/testify v1.3.0
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4
-golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb`
-	scanner := bufio.NewScanner(strings.NewReader(goListMAllOutput))
-
-	deps, err := GoList(scanner)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if len(deps.Projects) != 16 {
-		t.Error(deps)
-	}
-}
-
 func TestGoSumError(t *testing.T) {
 	_, err := GoSum("../testdata/parse/go.notsum")
 	if err == nil {
diff --git a/testdata/parse/gosumdupes.sum b/testdata/parse/gosumdupes.sum
deleted file mode 100644
index 44dc225..0000000
--- a/testdata/parse/gosumdupes.sum
+++ /dev/null
@@ -1,24 +0,0 @@
-github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo=
-github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg=
-github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
-github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg=
-github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
-github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA=
-github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA=
-github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k=
-github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
-github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb h1:pf3XwC90UUdNPYWZdFjhGBE7DUFuK3Ct1zWmZ65QN30=
-golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=