Repository: gitleaks/gitleaks (go)

Issue:
gitleaks still scanning node_modules and dist folder even ignoring them in gitleaks.toml in docker mode
**Describe the bug**
Hello dear friends!
My name is Jan Palach. I am facing an weird issue with gitleaks in docker mode. While using it in native executable mode nothing wrong seems to happen. The problem is, while running in docker mapping the target project to scan, gitleaks tell us that it has found the config file,but it still scanning node_modules and dist folders tooking to long to finish because my node_modules is huge. 

**To Reproduce**
1 - Follow the docker mode from the documentation
2 - put the following configfile  in a path mapped path to expose to the docker image:
`title = "Gitleaks Config File"

[extend]
useDefault = true

[allowlist]
description = "global allow lists"
paths = [
    '''.gitleaks.toml''',
    '''(.*?)(jpg|gif|doc|docx|zip|xls|pdf|bin|svg|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe)$''',
    '''(go.mod|go.sum)$''',
    '''gradle.lockfile''',
    '''yarn.lock''',
    '''package-lock.json''',
    '''package.json''',
    '''pnpm-lock.yaml''',
    '''Database.refactorlog''',
    '''vendor''',
    '''.*.test.*''',
    '''.*.swagger.*''',
    '''.*.error.*''',
    '''docs''',
    '''errors''',
    '''messages''',
    '''gitleaks_findings.json''',
    '''pre-commit''',
    '''README.md''',
    '''dist''',
    '''node_modules'''
]
`
3 - Then run the docker image pointing to the config file listed above:
docker run -v project root path:/app zricethezav/gitleaks:latest detect -l debug --no-git --source /app --verbose --no-banner --redact --config=/app/devsecops-secrets-detection/.gitleaks.toml

Result:
Use 'docker scan' to run Snyk tests against images to find vulnerabilities and learn how to fix them
2:48PM DBG using gitleaks config /app/devsecops-secrets-detection/.gitleaks.toml from `--config`
2:48PM DBG extending config with default config
**2:50PM INF scan completed in 1m51s**
2:50PM WRN leaks found: 1
Finding:     const fooAwsSecret = "REDACTED"
Secret:      REDACTED
RuleID:      aws-access-token
Entropy:     3.646439
File:       foofile
Line:        68
Fingerprint:foofile:aws-access-token:68

**Expected behavior**
It sholud run ignoring node_modules and dist paths, but it does not. Removing these paths the docker image runs the application in an expected time window.

**Basic Info (please complete the following information):**
 - OS: MacOS
 - Gitleaks Version: Latest docker image 8.18.0



I do really appreciate your attention my friends!


cc @zricethezav

<!--

Check out the Secret Scanning Discord Server for live discussions:

https://discord.gg/sydS6AHTUP

--->


Pull request:
Directory source should eagerly check allowlist
### Description:
This fixes #1286.

Comparison against a directory with a decent size `node_modules` folder.

**Before**
```
6:34PM INF scan completed in 2.57s
6:34PM WRN leaks found: 35
```

**After**
```
6:35PM INF scan completed in 113ms
6:35PM WRN leaks found: 35
```

### Checklist:

* [x] Does your PR pass tests?
* [ ] Have you written new tests for your changes?
* [x] Have you lint your code locally prior to submission?


Patch:
diff --git a/cmd/detect.go b/cmd/detect.go
index 3287981..db9c18b 100644
--- a/cmd/detect.go
+++ b/cmd/detect.go
@@ -62,6 +62,10 @@ func runDetect(cmd *cobra.Command, args []string) {
 
 	detector := Detector(cmd, cfg, source)
 
+	// set follow symlinks flag
+	if detector.FollowSymlinks, err = cmd.Flags().GetBool("follow-symlinks"); err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
 	// set exit code
 	exitCode, err := cmd.Flags().GetInt("exit-code")
 	if err != nil {
@@ -83,7 +87,12 @@ func runDetect(cmd *cobra.Command, args []string) {
 	// start the detector scan
 	if noGit {
 		var paths <-chan sources.ScanTarget
-		paths, err = sources.DirectoryTargets(source, detector.Sema, detector.FollowSymlinks)
+		paths, err = sources.DirectoryTargets(
+			source,
+			detector.Sema,
+			detector.FollowSymlinks,
+			detector.Config.Allowlist.PathAllowed,
+		)
 		if err != nil {
 			log.Fatal().Err(err)
 		}
@@ -120,10 +129,5 @@ func runDetect(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	// set follow symlinks flag
-	if detector.FollowSymlinks, err = cmd.Flags().GetBool("follow-symlinks"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-
 	findingSummaryAndExit(findings, cmd, cfg, exitCode, start, err)
 }
diff --git a/cmd/directory.go b/cmd/directory.go
index 656f867..a4289bd 100644
--- a/cmd/directory.go
+++ b/cmd/directory.go
@@ -45,6 +45,10 @@ func runDirectory(cmd *cobra.Command, args []string) {
 
 	detector := Detector(cmd, cfg, source)
 
+	// set follow symlinks flag
+	if detector.FollowSymlinks, err = cmd.Flags().GetBool("follow-symlinks"); err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
 	// set exit code
 	exitCode, err := cmd.Flags().GetInt("exit-code")
 	if err != nil {
@@ -52,7 +56,12 @@ func runDirectory(cmd *cobra.Command, args []string) {
 	}
 
 	var paths <-chan sources.ScanTarget
-	paths, err = sources.DirectoryTargets(source, detector.Sema, detector.FollowSymlinks)
+	paths, err = sources.DirectoryTargets(
+		source,
+		detector.Sema,
+		detector.FollowSymlinks,
+		detector.Config.Allowlist.PathAllowed,
+	)
 	if err != nil {
 		log.Fatal().Err(err)
 	}
diff --git a/cmd/generate/config/base/config.go b/cmd/generate/config/base/config.go
index 6ca5dae..60d3e6c 100644
--- a/cmd/generate/config/base/config.go
+++ b/cmd/generate/config/base/config.go
@@ -64,7 +64,7 @@ func CreateGlobalConfig() config.Config {
 				// ----------- Golang files -----------
 				regexp.MustCompile(`go\.(mod|sum|work(\.sum)?)$`),
 				regexp.MustCompile(`(^|/)vendor/modules\.txt$`),
-				regexp.MustCompile(`(^|/)vendor/(github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)/.*$`),
+				regexp.MustCompile(`(^|/)vendor/(github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(/.*)?$`),
 
 				// ----------- Java files -----------
 				// Gradle
@@ -75,12 +75,9 @@ func CreateGlobalConfig() config.Config {
 
 				// ----------- JavaScript files -----------
 				// Dependencies and lock files.
-				regexp.MustCompile(`(^|/)node_modules/.*?$`),
-				regexp.MustCompile(`(^|/)package-lock\.json$`),
-				regexp.MustCompile(`(^|/)yarn\.lock$`),
-				regexp.MustCompile(`(^|/)pnpm-lock\.yaml$`),
-				regexp.MustCompile(`(^|/)npm-shrinkwrap\.json$`),
-				regexp.MustCompile(`(^|/)bower_components/.*?$`),
+				regexp.MustCompile(`(^|/)node_modules(/.*)?$`),
+				regexp.MustCompile(`(^|/)(npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$`),
+				regexp.MustCompile(`(^|/)bower_components(/.*)?$`),
 				// TODO: Add more common static assets, such as swagger-ui.
 				regexp.MustCompile(`(^|/)(angular|jquery(-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(\.min)?\.js(\.map)?$`),
 
@@ -88,17 +85,17 @@ func CreateGlobalConfig() config.Config {
 				// Dependencies and lock files.
 				regexp.MustCompile(`(^|/)(Pipfile|poetry)\.lock$`),
 				// Virtual environments
-				regexp.MustCompile(`(?i)/?(v?env|virtualenv)/lib(64)?/.+$`),
-				regexp.MustCompile(`(?i)(^|/)(lib(64)?/python[23](\.\d{1,2})+/|python/[23](\.\d{1,2})+/lib(64)?/).+$`),
+				regexp.MustCompile(`(?i)/?(v?env|virtualenv)/lib(64)?(/.*)?$`),
+				regexp.MustCompile(`(?i)(^|/)(lib(64)?/python[23](\.\d{1,2})+|python/[23](\.\d{1,2})+/lib(64)?)(/.*)?$`),
 				// dist-info directory (https://py-pkgs.org/04-package-structure.html#building-sdists-and-wheels)
-				regexp.MustCompile(`(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info/.+$`),
+				regexp.MustCompile(`(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(/.+)?$`),
 
 				// ----------- Ruby files -----------
-				regexp.MustCompile(`(^|/)vendor/(bundle|ruby)/.*?$`),
+				regexp.MustCompile(`(^|/)vendor/(bundle|ruby)(/.*?)?$`),
 				regexp.MustCompile(`\.gem$`), // tar archive
 
 				// Misc
-				regexp.MustCompile(`verification-metadata.xml`),
+				regexp.MustCompile(`verification-metadata\.xml`),
 				regexp.MustCompile(`Database.refactorlog`),
 				//regexp.MustCompile(`vendor`),
 			},
diff --git a/config/gitleaks.toml b/config/gitleaks.toml
index 09d6c8f..d6d481d 100644
--- a/config/gitleaks.toml
+++ b/config/gitleaks.toml
@@ -32,25 +32,22 @@ paths = [
     '''(.*?)(doc|docx|zip|xls|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$''',
     '''go\.(mod|sum|work(\.sum)?)$''',
     '''(^|/)vendor/modules\.txt$''',
-    '''(^|/)vendor/(github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)/.*$''',
+    '''(^|/)vendor/(github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(/.*)?$''',
     '''(^|/)gradlew(\.bat)?$''',
     '''(^|/)gradle\.lockfile$''',
     '''(^|/)mvnw(\.cmd)?$''',
     '''(^|/)\.mvn/wrapper/MavenWrapperDownloader\.java$''',
-    '''(^|/)node_modules/.*?$''',
-    '''(^|/)package-lock\.json$''',
-    '''(^|/)yarn\.lock$''',
-    '''(^|/)pnpm-lock\.yaml$''',
-    '''(^|/)npm-shrinkwrap\.json$''',
-    '''(^|/)bower_components/.*?$''',
+    '''(^|/)node_modules(/.*)?$''',
+    '''(^|/)(npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$''',
+    '''(^|/)bower_components(/.*)?$''',
     '''(^|/)(angular|jquery(-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(\.min)?\.js(\.map)?$''',
     '''(^|/)(Pipfile|poetry)\.lock$''',
-    '''(?i)/?(v?env|virtualenv)/lib(64)?/.+$''',
-    '''(?i)(^|/)(lib(64)?/python[23](\.\d{1,2})+/|python/[23](\.\d{1,2})+/lib(64)?/).+$''',
-    '''(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info/.+$''',
-    '''(^|/)vendor/(bundle|ruby)/.*?$''',
+    '''(?i)/?(v?env|virtualenv)/lib(64)?(/.*)?$''',
+    '''(?i)(^|/)(lib(64)?/python[23](\.\d{1,2})+|python/[23](\.\d{1,2})+/lib(64)?)(/.*)?$''',
+    '''(?i)(^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(/.+)?$''',
+    '''(^|/)vendor/(bundle|ruby)(/.*?)?$''',
     '''\.gem$''',
-    '''verification-metadata.xml''',
+    '''verification-metadata\.xml''',
     '''Database.refactorlog''',
 ]
 stopwords = [
diff --git a/detect/detect.go b/detect/detect.go
index 0128f84..8a410d2 100644
--- a/detect/detect.go
+++ b/detect/detect.go
@@ -22,7 +22,7 @@ import (
 
 const (
 	gitleaksAllowSignature = "gitleaks:allow"
-	chunkSize              = 10 * 1_000 // 10kb
+	chunkSize              = 100 * 1_000 // 100kb
 )
 
 var newLineRegexp = regexp.MustCompile("\n")
diff --git a/detect/directory.go b/detect/directory.go
index be9410d..7a832fc 100644
--- a/detect/directory.go
+++ b/detect/directory.go
@@ -16,6 +16,7 @@ func (d *Detector) DetectFiles(paths <-chan sources.ScanTarget) ([]report.Findin
 		d.Sema.Go(func() error {
 			logger := log.With().Str("path", pa.Path).Logger()
 			logger.Trace().Msg("Scanning path")
+
 			f, err := os.Open(pa.Path)
 			if err != nil {
 				if os.IsPermission(err) {
@@ -24,7 +25,9 @@ func (d *Detector) DetectFiles(paths <-chan sources.ScanTarget) ([]report.Findin
 				}
 				return err
 			}
-			defer f.Close()
+			defer func() {
+				_ = f.Close()
+			}()
 
 			// Get file size
 			fileInfo, err := f.Stat()
@@ -37,7 +40,7 @@ func (d *Detector) DetectFiles(paths <-chan sources.ScanTarget) ([]report.Findin
 				if rawLength > int64(d.MaxTargetMegaBytes) {
 					logger.Debug().
 						Int64("size", rawLength).
-						Msgf("Skipping file: exceeds --max-target-megabytes")
+						Msg("Skipping file: exceeds --max-target-megabytes")
 					return nil
 				}
 			}
diff --git a/sources/directory.go b/sources/directory.go
index 1d1a52d..a90e3d3 100644
--- a/sources/directory.go
+++ b/sources/directory.go
@@ -1,12 +1,12 @@
 package sources
 
 import (
-	"github.com/rs/zerolog/log"
 	"io/fs"
 	"os"
 	"path/filepath"
 
 	"github.com/fatih/semgroup"
+	"github.com/rs/zerolog/log"
 )
 
 type ScanTarget struct {
@@ -14,13 +14,14 @@ type ScanTarget struct {
 	Symlink string
 }
 
-func DirectoryTargets(source string, s *semgroup.Group, followSymlinks bool) (<-chan ScanTarget, error) {
+func DirectoryTargets(source string, s *semgroup.Group, followSymlinks bool, shouldSkip func(string) bool) (<-chan ScanTarget, error) {
 	paths := make(chan ScanTarget)
 	s.Go(func() error {
 		defer close(paths)
 		return filepath.Walk(source,
 			func(path string, fInfo os.FileInfo, err error) error {
 				logger := log.With().Str("path", path).Logger()
+
 				if err != nil {
 					if os.IsPermission(err) {
 						// This seems to only fail on directories at this stage.
@@ -30,21 +31,18 @@ func DirectoryTargets(source string, s *semgroup.Group, followSymlinks bool) (<-
 					return err
 				}
 
-				if fInfo.Name() == ".git" && fInfo.IsDir() {
-					return filepath.SkipDir
-				}
+				// Empty; nothing to do here.
 				if fInfo.Size() == 0 {
 					return nil
 				}
-				if fInfo.Mode().IsRegular() {
-					paths <- ScanTarget{
-						Path:    path,
-						Symlink: "",
-					}
+
+				// Unwrap symlinks, if |followSymlinks| is set.
+				scanTarget := ScanTarget{
+					Path: path,
 				}
 				if fInfo.Mode().Type() == fs.ModeSymlink {
 					if !followSymlinks {
-						log.Debug().Str("path", path).Msg("Skipping symlink")
+						logger.Debug().Msg("Skipping symlink")
 						return nil
 					}
 
@@ -52,15 +50,39 @@ func DirectoryTargets(source string, s *semgroup.Group, followSymlinks bool) (<-
 					if err != nil {
 						return err
 					}
+
 					realPathFileInfo, _ := os.Stat(realPath)
 					if realPathFileInfo.IsDir() {
-						log.Debug().Msgf("found symlinked directory: %s -> %s [skipping]", path, realPath)
+						logger.Warn().Str("target", realPath).Msg("Skipping symlinked directory")
 						return nil
 					}
-					paths <- ScanTarget{
-						Path:    realPath,
-						Symlink: path,
+
+					scanTarget.Path = realPath
+					scanTarget.Symlink = path
+				}
+
+				// TODO: Also run this check against the resolved symlink?
+				skip := shouldSkip(path)
+				if fInfo.IsDir() {
+					// Directory
+					if skip {
+						logger.Debug().Msg("Skipping directory due to global allowlist")
+						return filepath.SkipDir
+					}
+
+					if fInfo.Name() == ".git" {
+						// Don't scan .git directories.
+						// TODO: Add this to the config allowlist, instead of hard-coding it.
+						return filepath.SkipDir
+					}
+				} else {
+					// File
+					if skip {
+						logger.Debug().Msg("Skipping file due to global allowlist")
+						return nil
 					}
+
+					paths <- scanTarget
 				}
 				return nil
 			})


Test patch:
diff --git a/detect/detect_test.go b/detect/detect_test.go
index 33b2ef9..e2d7611 100644
--- a/detect/detect_test.go
+++ b/detect/detect_test.go
@@ -789,7 +789,7 @@ func TestFromFiles(t *testing.T) {
 		err = detector.AddGitleaksIgnore(ignorePath)
 		require.NoError(t, err)
 		detector.FollowSymlinks = true
-		paths, err := sources.DirectoryTargets(tt.source, detector.Sema, true)
+		paths, err := sources.DirectoryTargets(tt.source, detector.Sema, true, cfg.Allowlist.PathAllowed)
 		require.NoError(t, err)
 		findings, err := detector.DetectFiles(paths)
 		require.NoError(t, err)
@@ -840,7 +840,7 @@ func TestDetectWithSymlinks(t *testing.T) {
 		cfg, _ := vc.Translate()
 		detector := NewDetector(cfg)
 		detector.FollowSymlinks = true
-		paths, err := sources.DirectoryTargets(tt.source, detector.Sema, true)
+		paths, err := sources.DirectoryTargets(tt.source, detector.Sema, true, cfg.Allowlist.PathAllowed)
 		require.NoError(t, err)
 		findings, err := detector.DetectFiles(paths)
 		require.NoError(t, err)