Repository: mgechev/revive (go)

Issue:
too many open files
When running revive over a medium size codebase (~1500 files) it occasionally hits the system file descriptor limit.

Originally we suspected Chrome eating up FDs, but this still happens even with no other software running on the system, and with high system wide limits set.

After some profiling it seems that revive will spawn a new goroutine for every file upon calling `revive.Lint`, all at the same time.

**To Reproduce**

```
/go/.bin/revive -formatter=stylish -config=.revive.toml ./cmd/... ./pkg/...
open pkg/service/flags/flags.go: too many open files
```
<details>
<summary>.revive.toml</summary>

```toml
ignoreGeneratedHeader = false
severity = "error"
confidence = 0.8
errorCode = 1

# https://github.com/mgechev/revive#available-rules
[rule.atomic]
[rule.blank-imports]
[rule.bool-literal-in-expr]
[rule.confusing-results]
[rule.constant-logical-expr]
[rule.context-keys-type]
[rule.dot-imports]
[rule.empty-block]
[rule.error-naming]
[rule.error-return]
[rule.errorf]
[rule.get-return]
[rule.if-return]
[rule.increment-decrement]
[rule.indent-error-flow]
[rule.modifies-parameter]
[rule.modifies-value-receiver]
[rule.range-val-address]
[rule.range-val-in-closure]
[rule.range]
[rule.receiver-naming]
[rule.redefines-builtin-id]
[rule.string-of-int]
[rule.struct-tag]
[rule.superfluous-else]
[rule.time-naming]
[rule.unexported-return]
[rule.unreachable-code]
[rule.var-declaration]
[rule.waitgroup-by-value]
```

</details>

**Expected behavior**

Should lint the full repo without incident.

**Desktop:**
 - OS: MacOS
 - go 1.17.5

I have a proposed solution - PR will follow.

Pull request description:
Max open files
Closes #624 by adding a command line flag `max_open_files` to limit the number of open files.
By default, the flag is set to 0 (unlimited)



Patch:
diff --git a/README.md b/README.md
index 1adb89c..9a0cc32 100644
--- a/README.md
+++ b/README.md
@@ -191,7 +191,7 @@ Please notice that if no particular configuration is provided, `revive` will beh
 
 ### Command Line Flags
 
-`revive` accepts three command line parameters:
+`revive` accepts the following command line parameters:
 
 - `-config [PATH]` - path to config file in TOML format, defaults to `$HOME/revive.toml` if present.
 - `-exclude [PATTERN]` - pattern for files/directories/packages to be excluded for linting. You can specify the files you want to exclude for linting either as package name (i.e. `github.com/mgechev/revive`), list them as individual files (i.e. `file.go`), directories (i.e. `./foo/...`), or any combination of the three.
@@ -203,7 +203,10 @@ Please notice that if no particular configuration is provided, `revive` will beh
   - `friendly` - outputs the failures when found. Shows summary of all the failures.
   - `stylish` - formats the failures in a table. Keep in mind that it doesn't stream the output so it might be perceived as slower compared to others.
   - `checkstyle` - outputs the failures in XML format compatible with that of Java's [Checkstyle](https://checkstyle.org/).
-- `-set_exit_status` - set exit status to 1 if any issues are found, overwrites errorCode and warningCode in config.
+- `-max_open_files` -  maximum number of open files at the same time. Defaults to unlimited.
+- `-set_exit_status` - set exit status to 1 if any issues are found, overwrites `errorCode` and `warningCode` in config.
+- `-version` - get revive version.
+
 
 ### Sample Invocations
 
diff --git a/lint/linter.go b/lint/linter.go
index 4e7be59..9a194f0 100644
--- a/lint/linter.go
+++ b/lint/linter.go
@@ -18,12 +18,34 @@ type disabledIntervalsMap = map[string][]DisabledInterval
 
 // Linter is used for linting set of files.
 type Linter struct {
-	reader ReadFile
+	reader         ReadFile
+	fileReadTokens chan struct{}
 }
 
 // New creates a new Linter
-func New(reader ReadFile) Linter {
-	return Linter{reader: reader}
+func New(reader ReadFile, maxOpenFiles int) Linter {
+	var fileReadTokens chan struct{}
+	if maxOpenFiles > 0 {
+		fileReadTokens = make(chan struct{}, maxOpenFiles)
+	}
+	return Linter{
+		reader:         reader,
+		fileReadTokens: fileReadTokens,
+	}
+}
+
+func (l Linter) readFile(path string) (result []byte, err error) {
+	if l.fileReadTokens != nil {
+		// "take" a token by writing to the channel.
+		// It will block if no more space in the channel's buffer
+		l.fileReadTokens <- struct{}{}
+		defer func() {
+			// "free" a token by reading from the channel
+			<-l.fileReadTokens
+		}()
+	}
+
+	return l.reader(path)
 }
 
 var (
@@ -62,7 +84,7 @@ func (l *Linter) lintPackage(filenames []string, ruleSet []Rule, config Config,
 		mu:    sync.Mutex{},
 	}
 	for _, filename := range filenames {
-		content, err := l.reader(filename)
+		content, err := l.readFile(filename)
 		if err != nil {
 			return err
 		}
diff --git a/main.go b/main.go
index bb842bd..3e27b8c 100644
--- a/main.go
+++ b/main.go
@@ -59,7 +59,7 @@ func main() {
 
 	revive := lint.New(func(file string) ([]byte, error) {
 		return ioutil.ReadFile(file)
-	})
+	}, maxOpenFiles)
 
 	lintingRules, err := config.GetLintingRules(conf)
 	if err != nil {
@@ -155,6 +155,7 @@ var (
 	help          bool
 	versionFlag   bool
 	setExitStatus bool
+	maxOpenFiles  int
 )
 
 var originalUsage = flag.Usage
@@ -204,11 +205,12 @@ func init() {
 
 	// command line help strings
 	const (
-		configUsage     = "path to the configuration TOML file, defaults to $HOME/revive.toml, if present (i.e. -config myconf.toml)"
-		excludeUsage    = "list of globs which specify files to be excluded (i.e. -exclude foo/...)"
-		formatterUsage  = "formatter to be used for the output (i.e. -formatter stylish)"
-		versionUsage    = "get revive version"
-		exitStatusUsage = "set exit status to 1 if any issues are found, overwrites errorCode and warningCode in config"
+		configUsage       = "path to the configuration TOML file, defaults to $HOME/revive.toml, if present (i.e. -config myconf.toml)"
+		excludeUsage      = "list of globs which specify files to be excluded (i.e. -exclude foo/...)"
+		formatterUsage    = "formatter to be used for the output (i.e. -formatter stylish)"
+		versionUsage      = "get revive version"
+		exitStatusUsage   = "set exit status to 1 if any issues are found, overwrites errorCode and warningCode in config"
+		maxOpenFilesUsage = "maximum number of open files at the same time"
 	)
 
 	defaultConfigPath := buildDefaultConfigPath()
@@ -218,6 +220,7 @@ func init() {
 	flag.StringVar(&formatterName, "formatter", "", formatterUsage)
 	flag.BoolVar(&versionFlag, "version", false, versionUsage)
 	flag.BoolVar(&setExitStatus, "set_exit_status", false, exitStatusUsage)
+	flag.IntVar(&maxOpenFiles, "max_open_files", 0, maxOpenFilesUsage)
 	flag.Parse()
 
 	// Output build info (version, commit, date and builtBy)
diff --git a/rule/cyclomatic.go b/rule/cyclomatic.go
index afb556c..bdda5f5 100644
--- a/rule/cyclomatic.go
+++ b/rule/cyclomatic.go
@@ -62,8 +62,9 @@ func (w lintCyclomatic) Visit(_ ast.Node) ast.Visitor {
 				w.onFailure(lint.Failure{
 					Confidence: 1,
 					Category:   "maintenance",
-					Failure:    fmt.Sprintf("function %s has cyclomatic complexity %d", funcName(fn), c),
-					Node:       fn,
+					Failure: fmt.Sprintf("function %s has cyclomatic complexity %d (> max enabled %d)",
+						funcName(fn), c, w.complexity),
+					Node: fn,
 				})
 			}
 		}


Test patch:
diff --git a/test/utils.go b/test/utils.go
index a1ded6a..e067f1b 100644
--- a/test/utils.go
+++ b/test/utils.go
@@ -44,7 +44,7 @@ func testRule(t *testing.T, filename string, rule lint.Rule, config ...*lint.Rul
 func assertSuccess(t *testing.T, baseDir string, fi os.FileInfo, rules []lint.Rule, config map[string]lint.RuleConfig) error {
 	l := lint.New(func(file string) ([]byte, error) {
 		return ioutil.ReadFile(baseDir + file)
-	})
+	}, 0)
 
 	ps, err := l.Lint([][]string{{fi.Name()}}, rules, lint.Config{
 		Rules: config,
@@ -66,7 +66,7 @@ func assertSuccess(t *testing.T, baseDir string, fi os.FileInfo, rules []lint.Ru
 func assertFailures(t *testing.T, baseDir string, fi os.FileInfo, src []byte, rules []lint.Rule, config map[string]lint.RuleConfig) error {
 	l := lint.New(func(file string) ([]byte, error) {
 		return ioutil.ReadFile(baseDir + file)
-	})
+	}, 0)
 
 	ins := parseInstructions(t, fi.Name(), src)
 	if ins == nil {
diff --git a/testdata/cyclomatic-2.go b/testdata/cyclomatic-2.go
index 35609cb..4741e02 100644
--- a/testdata/cyclomatic-2.go
+++ b/testdata/cyclomatic-2.go
@@ -5,7 +5,7 @@ package pkg
 
 import "log"
 
-func f(x int) bool { // MATCH /function f has cyclomatic complexity 4/
+func f(x int) bool { // MATCH /function f has cyclomatic complexity 4 (> max enabled 3)/
 	if x > 0 && true || false {
 		return true
 	} else {
diff --git a/testdata/cyclomatic.go b/testdata/cyclomatic.go
index dca68d6..396b1c4 100644
--- a/testdata/cyclomatic.go
+++ b/testdata/cyclomatic.go
@@ -5,7 +5,7 @@ package pkg
 
 import "log"
 
-func f(x int) bool { // MATCH /function f has cyclomatic complexity 4/
+func f(x int) bool { // MATCH /function f has cyclomatic complexity 4 (> max enabled 1)/
 	if x > 0 && true || false {
 		return true
 	} else {
@@ -14,7 +14,7 @@ func f(x int) bool { // MATCH /function f has cyclomatic complexity 4/
 	return false
 }
 
-func g(f func() bool) string { // MATCH /function g has cyclomatic complexity 2/
+func g(f func() bool) string { // MATCH /function g has cyclomatic complexity 2 (> max enabled 1)/
 	if ok := f(); ok {
 		return "it's okay"
 	} else {