Repository: gitleaks/gitleaks
Pull request #1556 (2024-10-09 20:11:57)

Fix rule extension
### Description:
PR #1524 broke the ability to specify a `[[rules]]` with the same ID as a built-in rule in order to extend the default configuration.

This PR restores this ability, but merges the provided configuration with the default rule instead of overwriting the default rule wholesale.

### Checklist:

* [x] Does your PR pass tests?
* [x] Have you written new tests for your changes?
* [x] Have you lint your code locally prior to submission?


rgmz: (Continuing the discussion from #1524.)

> @rgmz Just a minimal example. Our actual configuration looks like this:

Ah, I see you want to completely replace the rule. I haven't seen anyone do that before. The impetus for this change was because quite a few people were doing stuff similar to your minimal example.

The `[extend]` functionality seems to be poorly understood / lack definite clarity on what gets appended versus overwritten. This change makes sense — although it is inconsistent that tags/keywords are overwritten but allowlist is appended.
zricethezav: > The [extend] functionality seems to be poorly understood / lack definite clarity on what gets appended versus overwritten. This change makes sense — although it is inconsistent that tags/keywords are overwritten but allowlist is appended.

I think we need to be consistent across the board for the default case. _All the things_ should be appended rather than overwritten. A feature could be included to configure the behavior of `extend` down the line -- `append` vs `overwrite`, or something.

Patch:
diff --git a/cmd/generate/config/base/config.go b/cmd/generate/config/base/config.go
index 2900132..540a91b 100644
--- a/cmd/generate/config/base/config.go
+++ b/cmd/generate/config/base/config.go
@@ -1,8 +1,10 @@
 package base
 
 import (
+	"fmt"
 	"github.com/zricethezav/gitleaks/v8/config"
 	"regexp"
+	"strings"
 )
 
 func CreateGlobalConfig() config.Config {
@@ -13,6 +15,27 @@ func CreateGlobalConfig() config.Config {
 			Regexes: []*regexp.Regexp{
 				// ----------- General placeholders -----------
 				regexp.MustCompile(`(?i)^true|false|null$`),
+				// Awkward workaround to detect repeated characters.
+				func() *regexp.Regexp {
+					var (
+						letters  = "abcdefghijklmnopqrstuvwxyz*."
+						patterns []string
+					)
+					for _, char := range letters {
+						if char == '*' || char == '.' {
+							patterns = append(patterns, fmt.Sprintf("\\%c+", char))
+						} else {
+							patterns = append(patterns, fmt.Sprintf("%c+", char))
+						}
+					}
+					return regexp.MustCompile("^(?i:" + strings.Join(patterns, "|") + ")$")
+				}(),
+
+				// ----------- Environment Variables -----------
+				regexp.MustCompile(`^\$(\d+|{\d+})$`),
+				regexp.MustCompile(`^\$([A-Z_]+|[a-z_]+)$`),
+				regexp.MustCompile(`^\${([A-Z_]+|[a-z_]+)}$`),
+
 				// ----------- Interpolated Variables -----------
 				// Ansible (https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html)
 				regexp.MustCompile(`^\{\{[ \t]*[\w ().|]+[ \t]*}}$`),
@@ -22,19 +45,18 @@ func CreateGlobalConfig() config.Config {
 				regexp.MustCompile(`^\$\{\{[ \t]*((env|github|secrets|vars)(\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$`),
 				// NuGet (https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file#using-environment-variables)
 				regexp.MustCompile(`^%([A-Z_]+|[a-z_]+)%$`),
+				// String formatting.
+				regexp.MustCompile(`^%[+\-# 0]?[bcdeEfFgGoOpqstTUvxX]$`), // Golang (https://pkg.go.dev/fmt)
+				regexp.MustCompile(`^\{\d{0,2}}$`),                       // Python (https://docs.python.org/3/tutorial/inputoutput.html)
 				// Urban Code Deploy (https://www.ibm.com/support/pages/replace-token-step-replaces-replacement-values-windows-variables)
 				regexp.MustCompile(`^@([A-Z_]+|[a-z_]+)@$`),
-
-				// ----------- Environment Variables -----------
-				regexp.MustCompile(`^\$(\d+|{\d+})$`),
-				regexp.MustCompile(`^\$([A-Z_]+|[a-z_]+)$`),
-				regexp.MustCompile(`^\${([A-Z_]+|[a-z_]+)}$`),
 			},
 			Paths: []*regexp.Regexp{
 				regexp.MustCompile(`gitleaks\.toml`),
 
 				// ----------- Documents and media -----------
-				regexp.MustCompile(`(?i)\.(bmp|gif|jpe?g|svg|tiff?)$`),
+				regexp.MustCompile(`(?i)\.(bmp|gif|jpe?g|svg|tiff?)$`), // Images
+				regexp.MustCompile(`\.(eot|[ot]tf|woff2?)$`),           // Fonts
 				regexp.MustCompile(`(.*?)(doc|docx|zip|xls|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$`),
 
 				// ----------- Golang files -----------
diff --git a/config/config.go b/config/config.go
index 6b4e286..50934a9 100644
--- a/config/config.go
+++ b/config/config.go
@@ -252,6 +252,23 @@ func (c *Config) extend(extensionConfig Config) {
 			c.OrderedRules = append(c.OrderedRules, ruleID)
 		} else {
 			// Rule exists, merge our changes into the base.
+			if currentRule.Description != "" {
+				baseRule.Description = currentRule.Description
+			}
+			if currentRule.Entropy != 0 {
+				baseRule.Entropy = currentRule.Entropy
+			}
+			if currentRule.SecretGroup != 0 {
+				baseRule.SecretGroup = currentRule.SecretGroup
+			}
+			if currentRule.Regex != nil {
+				baseRule.Regex = currentRule.Regex
+			}
+			if currentRule.Path != nil {
+				baseRule.Path = currentRule.Path
+			}
+			baseRule.Tags = append(baseRule.Tags, currentRule.Tags...)
+			baseRule.Keywords = append(baseRule.Keywords, currentRule.Keywords...)
 			baseRule.Allowlist.Commits = append(baseRule.Allowlist.Commits, currentRule.Allowlist.Commits...)
 			baseRule.Allowlist.Paths = append(baseRule.Allowlist.Paths, currentRule.Allowlist.Paths...)
 			baseRule.Allowlist.Regexes = append(baseRule.Allowlist.Regexes, currentRule.Allowlist.Regexes...)
diff --git a/config/gitleaks.toml b/config/gitleaks.toml
index c75cfe9..7e82c32 100644
--- a/config/gitleaks.toml
+++ b/config/gitleaks.toml
@@ -14,17 +14,21 @@ title = "gitleaks config"
 description = "global allow lists"
 regexes = [
     '''(?i)^true|false|null$''',
+    '''^(?i:a+|b+|c+|d+|e+|f+|g+|h+|i+|j+|k+|l+|m+|n+|o+|p+|q+|r+|s+|t+|u+|v+|w+|x+|y+|z+|\*+|\.+)$''',
+    '''^\$(\d+|{\d+})$''',
+    '''^\$([A-Z_]+|[a-z_]+)$''',
+    '''^\${([A-Z_]+|[a-z_]+)}$''',
     '''^\{\{[ \t]*[\w ().|]+[ \t]*}}$''',
     '''^\$\{\{[ \t]*((env|github|secrets|vars)(\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$''',
     '''^%([A-Z_]+|[a-z_]+)%$''',
+    '''^%[+\-# 0]?[bcdeEfFgGoOpqstTUvxX]$''',
+    '''^\{\d{0,2}}$''',
     '''^@([A-Z_]+|[a-z_]+)@$''',
-    '''^\$(\d+|{\d+})$''',
-    '''^\$([A-Z_]+|[a-z_]+)$''',
-    '''^\${([A-Z_]+|[a-z_]+)}$''',
 ]
 paths = [
     '''gitleaks\.toml''',
     '''(?i)\.(bmp|gif|jpe?g|svg|tiff?)$''',
+    '''\.(eot|[ot]tf|woff2?)$''',
     '''(.*?)(doc|docx|zip|xls|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$''',
     '''go\.(mod|sum|work(\.sum)?)$''',
     '''(^|/)vendor/modules\.txt$''',


Test patch:
diff --git a/cmd/generate/config/base/config_test.go b/cmd/generate/config/base/config_test.go
index 0c2316b..2f64171 100644
--- a/cmd/generate/config/base/config_test.go
+++ b/cmd/generate/config/base/config_test.go
@@ -14,6 +14,16 @@ func TestConfigAllowlistRegexes(t *testing.T) {
 				`true`, `True`, `false`, `False`, `null`, `NULL`,
 			},
 		},
+		"general placeholders - repeated characters": {
+			invalid: []string{
+				`aaaaaaaaaaaaaaaaa`, `BBBBBBBBBBbBBBBBBBbBB`, `********************`,
+			},
+			valid: []string{`aaaaaaaaaaaaaaaaaaabaa`, `pas*************d`},
+		},
+		"environment variables": {
+			invalid: []string{`$2`, `$GIT_PASSWORD`, `${GIT_PASSWORD}`, `$password`},
+			valid:   []string{`$yP@R.@=ibxI`, `$2a6WCust9aE`, `${not_complete1`},
+		},
 		"interpolated variables - ansible": {
 			invalid: []string{
 				`{{ x }}`, `{{ password }}`, `{{password}}`, `{{ data.proxy_password }}`,
@@ -39,14 +49,20 @@ func TestConfigAllowlistRegexes(t *testing.T) {
 				`%MY_PASSWORD%`, `%password%`,
 			},
 		},
+		"interpolated variables - string fmt - golang": {
+			invalid: []string{
+				`%b`, `%c`, `%d`, `% d`, `%e`, `%E`, `%f`, `%F`, `%g`, `%G`, `%o`, `%O`, `%p`, `%q`, `%-s`, `%s`, `%t`, `%T`, `%U`, `%#U`, `%+v`, `%#v`, `%v`, `%x`, `%X`,
+			},
+		},
+		"interpolated variables - string fmt - python": {
+			invalid: []string{
+				`{}`, `{0}`, `{10}`,
+			},
+		},
 		"interpolated variables - ucd": {
 			invalid: []string{`@password@`, `@LDAP_PASS@`},
 			valid:   []string{`@username@mastodon.example`},
 		},
-		"environment variables": {
-			invalid: []string{`$2`, `$GIT_PASSWORD`, `${GIT_PASSWORD}`, `$password`},
-			valid:   []string{`$yP@R.@=ibxI`, `$2a6WCust9aE`, `${not_complete1`},
-		},
 	}
 
 	cfg := CreateGlobalConfig()
diff --git a/config/config_test.go b/config/config_test.go
index cbec979..53d4b12 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -15,8 +15,13 @@ const configPath = "../testdata/config/"
 
 func TestTranslate(t *testing.T) {
 	tests := []struct {
-		cfgName   string
-		cfg       Config
+		// Configuration file basename to load, from `../testdata/config/`.
+		cfgName string
+		// Expected result.
+		cfg Config
+		// Rules to compare.
+		rules []string
+		// Error to expect.
 		wantError error
 	}{
 		{
@@ -174,6 +179,107 @@ func TestTranslate(t *testing.T) {
 				},
 			},
 		},
+		{
+			cfgName: "override_description",
+			rules:   []string{"aws-access-key"},
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					RuleID:      "aws-access-key",
+					Description: "Puppy Doggy",
+					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
+					Keywords:    []string{},
+					Tags:        []string{"key", "AWS"},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "override_entropy",
+			rules:   []string{"aws-access-key"},
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					RuleID:      "aws-access-key",
+					Description: "AWS Access Key",
+					Entropy:     999.0,
+					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
+					Keywords:    []string{},
+					Tags:        []string{"key", "AWS"},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "override_secret_group",
+			rules:   []string{"aws-access-key"},
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					RuleID:      "aws-access-key",
+					Description: "AWS Access Key",
+					Regex:       regexp.MustCompile("(?:a)(?:a)"),
+					SecretGroup: 2,
+					Keywords:    []string{},
+					Tags:        []string{"key", "AWS"},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "override_regex",
+			rules:   []string{"aws-access-key"},
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					RuleID:      "aws-access-key",
+					Description: "AWS Access Key",
+					Regex:       regexp.MustCompile("(?:a)"),
+					Keywords:    []string{},
+					Tags:        []string{"key", "AWS"},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "override_path",
+			rules:   []string{"aws-access-key"},
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					RuleID:      "aws-access-key",
+					Description: "AWS Access Key",
+					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
+					Path:        regexp.MustCompile("(?:puppy)"),
+					Keywords:    []string{},
+					Tags:        []string{"key", "AWS"},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "override_tags",
+			rules:   []string{"aws-access-key"},
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					RuleID:      "aws-access-key",
+					Description: "AWS Access Key",
+					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
+					Keywords:    []string{},
+					Tags:        []string{"key", "AWS", "puppy"},
+				},
+				},
+			},
+		},
+		{
+			cfgName: "override_keywords",
+			rules:   []string{"aws-access-key"},
+			cfg: Config{
+				Rules: map[string]Rule{"aws-access-key": {
+					RuleID:      "aws-access-key",
+					Description: "AWS Access Key",
+					Regex:       regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
+					Keywords:    []string{"puppy"},
+					Tags:        []string{"key", "AWS"},
+				},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -197,6 +303,14 @@ func TestTranslate(t *testing.T) {
 				return
 			}
 
+			if len(tt.rules) > 0 {
+				rules := make(map[string]Rule)
+				for _, name := range tt.rules {
+					rules[name] = cfg.Rules[name]
+				}
+				cfg.Rules = rules
+			}
+
 			var regexComparer = func(x, y *regexp.Regexp) bool {
 				// Compare the string representation of the regex patterns.
 				if x == nil || y == nil {
diff --git a/testdata/config/override_description.toml b/testdata/config/override_description.toml
new file mode 100644
index 0000000..59af87f
--- /dev/null
+++ b/testdata/config/override_description.toml
@@ -0,0 +1,8 @@
+title = "override a built-in rule's description"
+
+[extend]
+path = "../testdata/config/simple.toml"
+
+[[rules]]
+id = "aws-access-key"
+description = "Puppy Doggy"
diff --git a/testdata/config/override_entropy.toml b/testdata/config/override_entropy.toml
new file mode 100644
index 0000000..4703222
--- /dev/null
+++ b/testdata/config/override_entropy.toml
@@ -0,0 +1,8 @@
+title = "override a built-in rule's entropy"
+
+[extend]
+path = "../testdata/config/simple.toml"
+
+[[rules]]
+id = "aws-access-key"
+entropy = 999
diff --git a/testdata/config/override_keywords.toml b/testdata/config/override_keywords.toml
new file mode 100644
index 0000000..51082a1
--- /dev/null
+++ b/testdata/config/override_keywords.toml
@@ -0,0 +1,8 @@
+title = "override a built-in rule's keywords"
+
+[extend]
+path = "../testdata/config/simple.toml"
+
+[[rules]]
+id = "aws-access-key"
+keywords = ["puppy"]
diff --git a/testdata/config/override_path.toml b/testdata/config/override_path.toml
new file mode 100644
index 0000000..dfc4535
--- /dev/null
+++ b/testdata/config/override_path.toml
@@ -0,0 +1,8 @@
+title = "override a built-in rule's path"
+
+[extend]
+path = "../testdata/config/simple.toml"
+
+[[rules]]
+id = "aws-access-key"
+path = '''(?:puppy)'''
diff --git a/testdata/config/override_regex.toml b/testdata/config/override_regex.toml
new file mode 100644
index 0000000..8cf0111
--- /dev/null
+++ b/testdata/config/override_regex.toml
@@ -0,0 +1,8 @@
+title = "override a built-in rule's regex"
+
+[extend]
+path = "../testdata/config/simple.toml"
+
+[[rules]]
+id = "aws-access-key"
+regex = '''(?:a)'''
diff --git a/testdata/config/override_secret_group.toml b/testdata/config/override_secret_group.toml
new file mode 100644
index 0000000..a557240
--- /dev/null
+++ b/testdata/config/override_secret_group.toml
@@ -0,0 +1,9 @@
+title = "override a built-in rule's secretGroup"
+
+[extend]
+path = "../testdata/config/simple.toml"
+
+[[rules]]
+id = "aws-access-key"
+regex = '''(?:a)(?:a)'''
+secretGroup = 2
diff --git a/testdata/config/override_tags.toml b/testdata/config/override_tags.toml
new file mode 100644
index 0000000..73dde82
--- /dev/null
+++ b/testdata/config/override_tags.toml
@@ -0,0 +1,8 @@
+title = "override a built-in rule's tags"
+
+[extend]
+path = "../testdata/config/simple.toml"
+
+[[rules]]
+id = "aws-access-key"
+tags = ["puppy"]