Repository: gitleaks/gitleaks
Pull request #1597 (2024-10-23 18:10:37)

Tweak python paths in global allowlist
### Description:
This tweaks the paths in the global allowlist to catch more false positives from Python virtual environments.

### Checklist:

* [x] Does your PR pass tests?
* [x] Have you written new tests for your changes?
* [ ] Have you lint your code locally prior to submission?



Patch:
diff --git a/cmd/generate/config/base/config.go b/cmd/generate/config/base/config.go
index 767e92d..b0aa7c8 100644
--- a/cmd/generate/config/base/config.go
+++ b/cmd/generate/config/base/config.go
@@ -84,18 +84,10 @@ func CreateGlobalConfig() config.Config {
 
 				// ----------- Python files -----------
 				// Dependencies and lock files.
-				regexp.MustCompile(`(^|/)Pipfile\.lock$`),
-				regexp.MustCompile(`(^|/)poetry\.lock$`),
+				regexp.MustCompile(`(^|/)(Pipfile|poetry)\.lock$`),
 				// Virtual environments
-				// env/lib/python3.7/site-packages/urllib3/util/url.py
-				regexp.MustCompile(`(?i)/?(v?env|virtualenv)/lib/.+$`),
-				// /python/3.7.4/Lib/site-packages/dask/bytes/tests/test_bytes_utils.py
-				// python/3.7.4/Lib/site-packages/fsspec/utils.py
-				// python/2.7.16.32/Lib/bsddb/test/test_dbenv.py
-				regexp.MustCompile(`(?i)/?python/[23](\.\d{1,2})+/lib/.+$`),
-				// python/lib/python3.8/site-packages/boto3/data/ec2/2016-04-01/resources-1.json
-				// python/lib/python3.8/site-packages/botocore/data/alexaforbusiness/2017-11-09/service-2.json
-				regexp.MustCompile(`(?i)/?python/lib/python[23](\.\d{1,2})+/.+$`),
+				regexp.MustCompile(`(?i)/?(v?env|virtualenv)/lib(64)?/.+$`),
+				regexp.MustCompile(`(?i)(^|/)(lib(64)?/python[23](\.\d{1,2})+/|python/[23](\.\d{1,2})+/lib(64)?/).+$`),
 				// dist-info directory (https://py-pkgs.org/04-package-structure.html#building-sdists-and-wheels)
 				regexp.MustCompile(`(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info/.+$`),
 
diff --git a/config/allowlist.go b/config/allowlist.go
index deba4c3..39ae863 100644
--- a/config/allowlist.go
+++ b/config/allowlist.go
@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+
+	"golang.org/x/exp/maps"
 )
 
 type AllowlistMatchCondition int
@@ -95,5 +97,22 @@ func (a *Allowlist) Validate() error {
 		return fmt.Errorf("[[rules.allowlists]] must contain at least one check for: commits, paths, regexes, or stopwords")
 	}
 
+	// Deduplicate commits and stopwords.
+	if len(a.Commits) > 0 {
+		uniqueCommits := make(map[string]struct{})
+		for _, commit := range a.Commits {
+			uniqueCommits[commit] = struct{}{}
+		}
+		a.Commits = maps.Keys(uniqueCommits)
+	}
+
+	if len(a.StopWords) > 0 {
+		uniqueStopwords := make(map[string]struct{})
+		for _, stopWord := range a.StopWords {
+			uniqueStopwords[stopWord] = struct{}{}
+		}
+		a.StopWords = maps.Keys(uniqueStopwords)
+	}
+
 	return nil
 }
diff --git a/config/gitleaks.toml b/config/gitleaks.toml
index 13f3060..6e5af51 100644
--- a/config/gitleaks.toml
+++ b/config/gitleaks.toml
@@ -44,11 +44,9 @@ paths = [
     '''(^|/)npm-shrinkwrap\.json$''',
     '''(^|/)bower_components/.*?$''',
     '''(^|/)jquery(-ui)?-[a-zA-Z0-9.-]+\.js$''',
-    '''(^|/)Pipfile\.lock$''',
-    '''(^|/)poetry\.lock$''',
-    '''(?i)/?(v?env|virtualenv)/lib/.+$''',
-    '''(?i)/?python/[23](\.\d{1,2})+/lib/.+$''',
-    '''(?i)/?python/lib/python[23](\.\d{1,2})+/.+$''',
+    '''(^|/)(Pipfile|poetry)\.lock$''',
+    '''(?i)/?(v?env|virtualenv)/lib(64)?/.+$''',
+    '''(?i)(^|/)(lib(64)?/python[23](\.\d{1,2})+/|python/[23](\.\d{1,2})+/lib(64)?/).+$''',
     '''(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info/.+$''',
     '''(^|/)vendor/(bundle|ruby)/.*?$''',
     '''\.gem$''',


Test patch:
diff --git a/cmd/generate/config/base/config_test.go b/cmd/generate/config/base/config_test.go
index e871525..e35612e 100644
--- a/cmd/generate/config/base/config_test.go
+++ b/cmd/generate/config/base/config_test.go
@@ -95,6 +95,28 @@ func TestConfigAllowlistPaths(t *testing.T) {
 				`src/main/resources/static/js/jquery-ui-1.10.4.min.js`,
 			},
 		},
+		"python": {
+			invalid: []string{
+				// lock files
+				`Pipfile.lock`, `poetry.lock`,
+				// virtual environments
+				"env/lib/python3.7/site-packages/urllib3/util/url.py",
+				"venv/Lib/site-packages/regex-2018.08.29.dist-info/DESCRIPTION.rst",
+				"venv/lib64/python3.5/site-packages/pynvml.py",
+				"python/python3/virtualenv/Lib/site-packages/pyphonetics/utils.py",
+				"virtualenv/lib64/python3.7/base64.py",
+				// packages
+				"cde-root/usr/lib64/python2.4/site-packages/Numeric.pth",
+				"lib/python3.9/site-packages/setuptools/_distutils/msvccompiler.py",
+				"lib/python3.8/site-packages/botocore/data/alexaforbusiness/2017-11-09/service-2.json",
+				"code/python/3.7.4/Lib/site-packages/dask/bytes/tests/test_bytes_utils.py",
+				"python/3.7.4/Lib/site-packages/fsspec/utils.py",
+				"python/2.7.16.32/Lib/bsddb/test/test_dbenv.py",
+				"python/lib/python3.8/site-packages/boto3/data/ec2/2016-04-01/resources-1.json",
+				// distinfo
+				"libs/PyX-0.15.dist-info/AUTHORS",
+			},
+		},
 	}
 
 	cfg := CreateGlobalConfig()
diff --git a/config/allowlist_test.go b/config/allowlist_test.go
index be416ee..c66304b 100644
--- a/config/allowlist_test.go
+++ b/config/allowlist_test.go
@@ -1,9 +1,12 @@
 package config
 
 import (
+	"errors"
 	"regexp"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -91,3 +94,62 @@ func TestPathAllowed(t *testing.T) {
 		assert.Equal(t, tt.pathAllowed, tt.allowlist.PathAllowed(tt.path))
 	}
 }
+
+func TestValidate(t *testing.T) {
+	tests := map[string]struct {
+		input    Allowlist
+		expected Allowlist
+		wantErr  error
+	}{
+		"empty conditions": {
+			input:   Allowlist{},
+			wantErr: errors.New("[[rules.allowlists]] must contain at least one check for: commits, paths, regexes, or stopwords"),
+		},
+		"deduplicated commits and stopwords": {
+			input: Allowlist{
+				Commits:   []string{"commitA", "commitB", "commitA"},
+				StopWords: []string{"stopwordA", "stopwordB", "stopwordA"},
+			},
+			expected: Allowlist{
+				Commits:   []string{"commitA", "commitB"},
+				StopWords: []string{"stopwordA", "stopwordB"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		// Expected an error.
+		err := tt.input.Validate()
+		if err != nil {
+			if tt.wantErr == nil {
+				t.Fatalf("Received unexpected error: %v", err)
+			} else if !assert.EqualError(t, err, tt.wantErr.Error()) {
+				t.Fatalf("Received unexpected error, expected '%v', got '%v'", tt.wantErr, err)
+			}
+		} else {
+			if tt.wantErr != nil {
+				t.Fatalf("Did not receive expected error: %v", tt.wantErr)
+			}
+		}
+
+		var (
+			regexComparer = func(x, y *regexp.Regexp) bool {
+				// Compare the string representation of the regex patterns.
+				if x == nil || y == nil {
+					return x == y
+				}
+				return x.String() == y.String()
+			}
+			arrayComparer = func(a, b string) bool {
+				return a < b
+			}
+			opts = cmp.Options{
+				cmp.Comparer(regexComparer),
+				cmpopts.SortSlices(arrayComparer),
+			}
+		)
+		if diff := cmp.Diff(tt.input, tt.expected, opts); diff != "" {
+			t.Errorf("diff: (-want +got)\n%s", diff)
+		}
+	}
+}
diff --git a/config/config_test.go b/config/config_test.go
index 56ef111..8a336a7 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -381,7 +381,6 @@ func TestTranslate(t *testing.T) {
 			}
 
 			var regexComparer = func(x, y *regexp.Regexp) bool {
-				// Compare the string representation of the regex patterns.
 				if x == nil || y == nil {
 					return x == y
 				}