Repository: gitleaks/gitleaks
Pull request #1547 (2024-10-04 21:04:22)

Add placeholder regexes to global allowlist
### Description:
This leverages #1546 to add `[allowlist.regexes]` entries for common placeholder/variable formats, with tests. 

### Checklist:

* [x] Does your PR pass tests?
* [x] Have you written new tests for your changes?
* [x] Have you lint your code locally prior to submission?



Patch:
diff --git a/Dockerfile b/Dockerfile
index f258607..40962cc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21 AS build
+FROM golang:1.22 AS build
 WORKDIR /go/src/github.com/zricethezav/gitleaks
 COPY . .
 RUN VERSION=$(git describe --tags --abbrev=0) && \
diff --git a/cmd/generate/config/base/config.go b/cmd/generate/config/base/config.go
index 97d0197..d310d28 100644
--- a/cmd/generate/config/base/config.go
+++ b/cmd/generate/config/base/config.go
@@ -10,10 +10,30 @@ func CreateGlobalConfig() config.Config {
 		Title: "gitleaks config",
 		Allowlist: config.Allowlist{
 			Description: "global allow lists",
+			Regexes: []*regexp.Regexp{
+				// ----------- General placeholders -----------
+				regexp.MustCompile(`(?i)^true|false|null$`),
+				// ----------- Interpolated Variables -----------
+				// Ansible (https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html)
+				regexp.MustCompile(`^\{\{[ \t]*[\w ().|]+[ \t]*}}$`),
+				// GitHub Actions
+				// https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables
+				// https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions
+				regexp.MustCompile(`^\$\{\{[ \t]*((env|github|secrets|vars)(\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$`),
+				// NuGet (https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file#using-environment-variables)
+				regexp.MustCompile(`^%([A-Z_]+|[a-z_]+)%$`),
+				// Urban Code Deploy (https://www.ibm.com/support/pages/replace-token-step-replaces-replacement-values-windows-variables)
+				regexp.MustCompile(`^@([A-Z_]+|[a-z_]+)@$`),
+
+				// ----------- Environment Variables -----------
+				regexp.MustCompile(`^\$(\d+|{\d+})$`),
+				regexp.MustCompile(`^\$([A-Z_]+|[a-z_]+)$`),
+				regexp.MustCompile(`^\${([A-Z_]+|[a-z_]+)}$`),
+			},
 			Paths: []*regexp.Regexp{
 				regexp.MustCompile(`^gitleaks\.toml$`),
 
-				// Images
+				// ----------- Documents and media -----------
 				regexp.MustCompile(`(?i)\.(bmp|gif|jpe?g|svg|tiff?)$`),
 				regexp.MustCompile(`(.*?)(doc|docx|zip|xls|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$`),
 
diff --git a/cmd/generate/config/main.go b/cmd/generate/config/main.go
index 86d7011..508a6c0 100644
--- a/cmd/generate/config/main.go
+++ b/cmd/generate/config/main.go
@@ -49,6 +49,7 @@ func main() {
 		rules.CloudflareAPIKey(),
 		rules.CloudflareGlobalAPIKey(),
 		rules.CloudflareOriginCAKey(),
+		rules.CohereAPIToken(),
 		rules.ConfluentAccessToken(),
 		rules.ConfluentSecretKey(),
 		rules.Contentful(),
@@ -152,6 +153,7 @@ func main() {
 		rules.PlanetScaleOAuthToken(),
 		rules.PostManAPI(),
 		rules.Prefect(),
+		rules.PrivateAIToken(),
 		rules.PrivateKey(),
 		rules.PulumiAPIToken(),
 		rules.PyPiUploadToken(),
diff --git a/cmd/generate/config/rules/cohere.go b/cmd/generate/config/rules/cohere.go
new file mode 100644
index 0000000..d803eb0
--- /dev/null
+++ b/cmd/generate/config/rules/cohere.go
@@ -0,0 +1,32 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func CohereAPIToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		RuleID:      "cohere-api-token",
+		Description: "Identified a Cohere Token, posing a risk of unauthorized access to AI services and data manipulation.",
+		Regex:       utils.GenerateSemiGenericRegex([]string{"cohere", "CO_API_KEY"}, `[a-zA-Z0-9]{40}`, false),
+		Entropy:     4,
+		Keywords: []string{
+			"cohere",
+			"CO_API_KEY",
+		},
+	}
+
+	// validate
+	tps := []string{
+		utils.GenerateSampleSecret("cohere", secrets.NewSecret(`[a-zA-Z0-9]{40}`)),
+		// https://github.com/cohere-ai/cohere-go/blob/abe8044073ed498ffbb206a602d03c2414b64512/client/client.go#L38C30-L38C40
+		`export CO_API_KEY=` + secrets.NewSecret(`[a-zA-Z0-9]{40}`),
+	}
+	fps := []string{
+		`CO_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`,
+	}
+	return utils.Validate(r, tps, fps)
+}
diff --git a/cmd/generate/config/rules/config.tmpl b/cmd/generate/config/rules/config.tmpl
index 5c2da97..2806fd8 100644
--- a/cmd/generate/config/rules/config.tmpl
+++ b/cmd/generate/config/rules/config.tmpl
@@ -15,8 +15,8 @@ description = "{{.Allowlist.Description}}"
 {{- with .Allowlist.RegexTarget }}
 regexTarget = "{{ . }}"{{ end -}}
 {{- with .Allowlist.Regexes }}
-regexes = [
-    {{ range $i, $regex := . }}'''{{ $regex }}''',{{ end }}
+regexes = [{{ range $i, $regex := . }}
+    '''{{ $regex }}''',{{ end }}
 ]{{ end }}
 {{- with .Allowlist.Paths }}
 paths = [{{ range $i, $path := . }}
@@ -45,20 +45,21 @@ tags = [
 ]{{ end }}
 {{ if or $rule.Allowlist.Regexes $rule.Allowlist.Paths $rule.Allowlist.Commits $rule.Allowlist.StopWords }}
 [rules.allowlist]
-{{ with $rule.Allowlist.RegexTarget }}
+{{- with $rule.Allowlist.RegexTarget }}
 regexTarget = "{{ . }}"{{ end -}}
 {{- with $rule.Allowlist.Regexes }}
-regexes = [
-    {{ range $j, $regex := . }}'''{{ $regex }}''',{{ end }}
+regexes = [{{ range $i, $regex := . }}
+    '''{{ $regex }}''',{{ end }}
 ]{{ end }}
 {{- with $rule.Allowlist.Paths }}paths = [
-    {{ range $j, $path := . }}"{{ $path }}",{{ end }}
+    {{ range $j, $path := . }}'''{{ $path }}''',{{ end }}
 ]{{ end }}
 {{- with $rule.Allowlist.Commits }}commits = [
     {{ range $j, $commit := . }}"{{ $commit }}",{{ end }}
 ]{{ end }}
-{{- with $rule.Allowlist.StopWords }}stopwords = [{{ range $j, $stopword := . }}
+{{- with $rule.Allowlist.StopWords }}
+stopwords = [{{ range $j, $stopword := . }}
     "{{ $stopword }}",{{ end }}
 ]{{ end }}
 {{ end }}
-{{ end }}
+{{ end -}}
diff --git a/cmd/generate/config/rules/privateai.go b/cmd/generate/config/rules/privateai.go
new file mode 100644
index 0000000..6c5fea6
--- /dev/null
+++ b/cmd/generate/config/rules/privateai.go
@@ -0,0 +1,31 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils"
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func PrivateAIToken() *config.Rule {
+	// https://docs.private-ai.com/reference/latest/operation/metrics_metrics_get/
+	r := config.Rule{
+		RuleID:      "privateai-api-token",
+		Description: "Identified a PrivateAI Token, posing a risk of unauthorized access to AI services and data manipulation.",
+		Regex:       utils.GenerateSemiGenericRegex([]string{"private[_-]?ai"}, `[a-z0-9]{32}`, false),
+		Entropy:     3,
+		Keywords: []string{
+			"privateai",
+			"private_ai",
+			"private-ai",
+		},
+	}
+
+	// validate
+	tps := []string{
+		utils.GenerateSampleSecret("privateai", secrets.NewSecret(utils.AlphaNumeric("32"))),
+	}
+	fps := []string{
+		`const privateaiToken = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";`,
+	}
+	return utils.Validate(r, tps, fps)
+}
diff --git a/config/gitleaks.toml b/config/gitleaks.toml
index c1484d3..8e04746 100644
--- a/config/gitleaks.toml
+++ b/config/gitleaks.toml
@@ -12,6 +12,16 @@ title = "gitleaks config"
 
 [allowlist]
 description = "global allow lists"
+regexes = [
+    '''(?i)^true|false|null$''',
+    '''^\{\{[ \t]*[\w ().|]+[ \t]*}}$''',
+    '''^\$\{\{[ \t]*((env|github|secrets|vars)(\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$''',
+    '''^%([A-Z_]+|[a-z_]+)%$''',
+    '''^@([A-Z_]+|[a-z_]+)@$''',
+    '''^\$(\d+|{\d+})$''',
+    '''^\$([A-Z_]+|[a-z_]+)$''',
+    '''^\${([A-Z_]+|[a-z_]+)}$''',
+]
 paths = [
     '''^gitleaks\.toml$''',
     '''(?i)\.(bmp|gif|jpe?g|svg|tiff?)$''',
@@ -196,6 +206,16 @@ description = "Found a pattern resembling a Codecov Access Token, posing a risk
 regex = '''(?i)(?:codecov)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
 keywords = ["codecov"]
 
+[[rules]]
+id = "cohere-api-token"
+description = "Identified a Cohere Token, posing a risk of unauthorized access to AI services and data manipulation."
+regex = '''(?i:(?:cohere|CO_API_KEY)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3})(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([a-zA-Z0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+entropy = 4
+keywords = [
+    "cohere",
+    "co_api_key",
+]
+
 [[rules]]
 id = "coinbase-access-token"
 description = "Detected a Coinbase Access Token, posing a risk of unauthorized access to cryptocurrency accounts and financial transactions."
@@ -2139,7 +2159,6 @@ path = '''(?i)\.ya?ml$'''
 keywords = ["secret"]
 
 [rules.allowlist]
-
 regexes = [
     '''[\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:\{\{[ \t\w"|$:=,.-]+}}|""|'')''',
 ]
@@ -2326,9 +2345,10 @@ entropy = 1
 keywords = ["<add key="]
 
 [rules.allowlist]
-
 regexes = [
-    '''33f!!lloppa''','''hal\+9ooo_da!sY''','''^\%\S.*\%$''',
+    '''33f!!lloppa''',
+    '''hal\+9ooo_da!sY''',
+    '''^\%\S.*\%$''',
 ]
 
 [[rules]]
@@ -2416,6 +2436,17 @@ description = "Identified a Private Key, which may compromise cryptographic secu
 regex = '''(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]*?KEY(?: BLOCK)?-----'''
 keywords = ["-----begin"]
 
+[[rules]]
+id = "privateai-api-token"
+description = "Identified a PrivateAI Token, posing a risk of unauthorized access to AI services and data manipulation."
+regex = '''(?i:(?:private[_-]?ai)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3})(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{32})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+entropy = 3
+keywords = [
+    "privateai",
+    "private_ai",
+    "private-ai",
+]
+
 [[rules]]
 id = "pulumi-api-token"
 description = "Found a Pulumi API token, posing a risk to infrastructure as code services and cloud resource management."
@@ -2644,7 +2675,6 @@ entropy = 3
 keywords = ["sumo"]
 
 [rules.allowlist]
-
 regexTarget = "line"
 regexes = [
     '''sumOf''',
@@ -2734,7 +2764,6 @@ keywords = [
 ]
 
 [rules.allowlist]
-
 regexes = [
     '''s\.[A-Za-z]{24}''',
 ]
@@ -2763,4 +2792,3 @@ description = "Detected a Zendesk Secret Key, risking unauthorized access to cus
 regex = '''(?i)(?:zendesk)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([a-z0-9]{40})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
 keywords = ["zendesk"]
 
-


Test patch:
diff --git a/cmd/generate/config/base/config_test.go b/cmd/generate/config/base/config_test.go
new file mode 100644
index 0000000..0c2316b
--- /dev/null
+++ b/cmd/generate/config/base/config_test.go
@@ -0,0 +1,69 @@
+package base
+
+import (
+	"testing"
+)
+
+func TestConfigAllowlistRegexes(t *testing.T) {
+	tests := map[string]struct {
+		invalid []string
+		valid   []string
+	}{
+		"general placeholders": {
+			invalid: []string{
+				`true`, `True`, `false`, `False`, `null`, `NULL`,
+			},
+		},
+		"interpolated variables - ansible": {
+			invalid: []string{
+				`{{ x }}`, `{{ password }}`, `{{password}}`, `{{ data.proxy_password }}`,
+				`{{ dict1 | ansible.builtin.combine(dict2) }}`,
+			},
+		},
+		"interpolated variables - github actions": {
+			invalid: []string{
+				`${{ env.First_Name }}`,
+				`${{ env.DAY_OF_WEEK == 'Monday' }}`,
+				`${{env.JAVA_VERSION}}`,
+				`${{ github.event.issue.title }}`,
+				`${{ github.repository == "Gattocrucco/lsqfitgp" }}`,
+				`${{ github.event.pull_request.number || github.ref }}`,
+				`${{ github.event_name == 'pull_request' && github.event.action == 'unassigned' }}`,
+				`${{ secrets.SuperSecret }}`,
+				`${{ vars.JOB_NAME }}`,
+				`${{ vars.USE_VARIABLES == 'true' }}`,
+			},
+		},
+		"interpolated variables - nuget": {
+			invalid: []string{
+				`%MY_PASSWORD%`, `%password%`,
+			},
+		},
+		"interpolated variables - ucd": {
+			invalid: []string{`@password@`, `@LDAP_PASS@`},
+			valid:   []string{`@username@mastodon.example`},
+		},
+		"environment variables": {
+			invalid: []string{`$2`, `$GIT_PASSWORD`, `${GIT_PASSWORD}`, `$password`},
+			valid:   []string{`$yP@R.@=ibxI`, `$2a6WCust9aE`, `${not_complete1`},
+		},
+	}
+
+	cfg := CreateGlobalConfig()
+	allowlist := cfg.Allowlist
+	for name, cases := range tests {
+		t.Run(name, func(t *testing.T) {
+			for _, c := range cases.invalid {
+				if !allowlist.RegexAllowed(c) {
+					t.Errorf("invalid value not marked as allowed: %s", c)
+				}
+			}
+
+			for _, c := range cases.valid {
+				if allowlist.RegexAllowed(c) {
+					t.Errorf("valid value marked as allowed: %s", c)
+				}
+			}
+		})
+	}
+}