Repository: VKCOM/noverify
Pull request #664 (2020-08-12 12:42:05)

linter: move dirwalk-related code to workspace pkg
The FS traversal does not depend on the linter package
details, so we can move it somewhere else.

workspace package is intended to be used as a toolbox
for working with the PHP project (its files, settings, etc).

Signed-off-by: Iskander Sharipov <quasilyte@gmail.com>


Patch:
diff --git a/src/cmd/flags.go b/src/cmd/flags.go
index b0e2ead..b31274d 100644
--- a/src/cmd/flags.go
+++ b/src/cmd/flags.go
@@ -152,7 +152,7 @@ func bindFlags(ruleSets []*rules.Set, args *cmdlineArguments) {
 	flag.StringVar(&args.misspellList, "misspell-list", "Eng",
 		"Comma-separated list of misspelling dicts; predefined sets are Eng, Eng/US and Eng/UK")
 
-	flag.StringVar(&args.phpExtensionsArg, "php-extensions", "php,inc,php5,phtml,inc", "List of PHP extensions to be recognized")
+	flag.StringVar(&args.phpExtensionsArg, "php-extensions", "php,inc,php5,phtml", "List of PHP extensions to be recognized")
 
 	flag.StringVar(&args.fullAnalysisFiles, "full-analysis-files", "", "Comma-separated list of files to do full analysis")
 	flag.StringVar(&args.indexOnlyFiles, "index-only-files", "", "Comma-separated list of files to do indexing")
diff --git a/src/cmd/git_main.go b/src/cmd/git_main.go
index 4a59df5..9a78afe 100644
--- a/src/cmd/git_main.go
+++ b/src/cmd/git_main.go
@@ -9,6 +9,7 @@ import (
 	"github.com/VKCOM/noverify/src/git"
 	"github.com/VKCOM/noverify/src/linter"
 	"github.com/VKCOM/noverify/src/meta"
+	"github.com/VKCOM/noverify/src/workspace"
 )
 
 func gitParseUntracked(l *linterRunner) []*linter.Report {
@@ -21,7 +22,7 @@ func gitParseUntracked(l *linterRunner) []*linter.Report {
 		log.Fatalf("get untracked files: %v", err)
 	}
 
-	return linter.ParseFilenames(linter.ReadFilenames(filenames, nil), l.allowDisableRegex)
+	return linter.ParseFilenames(workspace.ReadFilenames(filenames, nil), l.allowDisableRegex)
 }
 
 func parseIndexOnlyFiles(l *linterRunner) {
@@ -29,7 +30,7 @@ func parseIndexOnlyFiles(l *linterRunner) {
 		return
 	}
 	filenames := strings.Split(l.args.indexOnlyFiles, ",")
-	linter.ParseFilenames(linter.ReadFilenames(filenames, nil), l.allowDisableRegex)
+	linter.ParseFilenames(workspace.ReadFilenames(filenames, nil), l.allowDisableRegex)
 }
 
 // Not the best name, and not the best function signature.
@@ -58,14 +59,14 @@ func gitRepoComputeReportsFromCommits(l *linterRunner, logArgs, diffArgs []strin
 		}
 
 		start := time.Now()
-		linter.ParseFilenames(linter.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, nil), l.allowDisableRegex)
+		linter.ParseFilenames(workspace.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, nil), l.allowDisableRegex)
 		parseIndexOnlyFiles(l)
 		log.Printf("Indexed old commit in %s", time.Since(start))
 
 		meta.SetIndexingComplete(true)
 
 		start = time.Now()
-		oldReports = linter.ParseFilenames(linter.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, linter.ExcludeRegex), l.allowDisableRegex)
+		oldReports = linter.ParseFilenames(workspace.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, linter.ExcludeRegex), l.allowDisableRegex)
 		log.Printf("Parsed old commit for %s (%d reports)", time.Since(start), len(oldReports))
 
 		meta.ResetInfo()
@@ -74,34 +75,34 @@ func gitRepoComputeReportsFromCommits(l *linterRunner, logArgs, diffArgs []strin
 		}
 
 		start = time.Now()
-		linter.ParseFilenames(linter.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitTo, nil), l.allowDisableRegex)
+		linter.ParseFilenames(workspace.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitTo, nil), l.allowDisableRegex)
 		log.Printf("Indexed new commit in %s", time.Since(start))
 
 		meta.SetIndexingComplete(true)
 
 		start = time.Now()
-		reports = linter.ParseFilenames(linter.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitTo, linter.ExcludeRegex), l.allowDisableRegex)
+		reports = linter.ParseFilenames(workspace.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitTo, linter.ExcludeRegex), l.allowDisableRegex)
 		log.Printf("Parsed new commit in %s (%d reports)", time.Since(start), len(reports))
 	} else {
 		start := time.Now()
-		linter.ParseFilenames(linter.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitTo, nil), l.allowDisableRegex)
+		linter.ParseFilenames(workspace.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitTo, nil), l.allowDisableRegex)
 		parseIndexOnlyFiles(l)
 		log.Printf("Indexing complete in %s", time.Since(start))
 
 		meta.SetIndexingComplete(true)
 
 		start = time.Now()
-		oldReports = linter.ParseFilenames(linter.ReadOldFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, changes), l.allowDisableRegex)
+		oldReports = linter.ParseFilenames(workspace.ReadOldFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, changes), l.allowDisableRegex)
 		log.Printf("Parsed old files versions for %s", time.Since(start))
 
 		start = time.Now()
 		meta.SetIndexingComplete(false)
-		linter.ParseFilenames(linter.ReadFilesFromGitWithChanges(l.args.gitRepo, l.args.mutable.gitCommitTo, changes), l.allowDisableRegex)
+		linter.ParseFilenames(workspace.ReadFilesFromGitWithChanges(l.args.gitRepo, l.args.mutable.gitCommitTo, changes), l.allowDisableRegex)
 		meta.SetIndexingComplete(true)
 		log.Printf("Indexed files versions for %s", time.Since(start))
 
 		start = time.Now()
-		reports = linter.ParseFilenames(linter.ReadFilesFromGitWithChanges(l.args.gitRepo, l.args.mutable.gitCommitTo, changes), l.allowDisableRegex)
+		reports = linter.ParseFilenames(workspace.ReadFilesFromGitWithChanges(l.args.gitRepo, l.args.mutable.gitCommitTo, changes), l.allowDisableRegex)
 		log.Printf("Parsed new file versions in %s", time.Since(start))
 	}
 
@@ -128,25 +129,25 @@ func gitRepoComputeReportsFromLocalChanges(l *linterRunner) (oldReports, reports
 	log.Printf("You have changes in your work tree, showing diff between %s and work tree", l.args.mutable.gitCommitFrom)
 
 	start := time.Now()
-	linter.ParseFilenames(linter.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, nil), l.allowDisableRegex)
+	linter.ParseFilenames(workspace.ReadFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, nil), l.allowDisableRegex)
 	parseIndexOnlyFiles(l)
 	log.Printf("Indexing complete in %s", time.Since(start))
 
 	meta.SetIndexingComplete(true)
 
 	start = time.Now()
-	oldReports = linter.ParseFilenames(linter.ReadOldFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, changes), l.allowDisableRegex)
+	oldReports = linter.ParseFilenames(workspace.ReadOldFilesFromGit(l.args.gitRepo, l.args.mutable.gitCommitFrom, changes), l.allowDisableRegex)
 	log.Printf("Parsed old files versions for %s", time.Since(start))
 
 	start = time.Now()
 	meta.SetIndexingComplete(false)
-	linter.ParseFilenames(linter.ReadChangesFromWorkTree(l.args.gitWorkTree, changes), l.allowDisableRegex)
+	linter.ParseFilenames(workspace.ReadChangesFromWorkTree(l.args.gitWorkTree, changes), l.allowDisableRegex)
 	gitParseUntracked(l)
 	meta.SetIndexingComplete(true)
 	log.Printf("Indexed new files versions for %s", time.Since(start))
 
 	start = time.Now()
-	reports = linter.ParseFilenames(linter.ReadChangesFromWorkTree(l.args.gitWorkTree, changes), l.allowDisableRegex)
+	reports = linter.ParseFilenames(workspace.ReadChangesFromWorkTree(l.args.gitWorkTree, changes), l.allowDisableRegex)
 	reports = append(reports, gitParseUntracked(l)...)
 	log.Printf("Parsed new file versions in %s", time.Since(start))
 
diff --git a/src/cmd/linter_runner.go b/src/cmd/linter_runner.go
index 540da2e..6592090 100644
--- a/src/cmd/linter_runner.go
+++ b/src/cmd/linter_runner.go
@@ -13,6 +13,7 @@ import (
 	"github.com/VKCOM/noverify/src/baseline"
 	"github.com/VKCOM/noverify/src/linter"
 	"github.com/VKCOM/noverify/src/rules"
+	"github.com/VKCOM/noverify/src/workspace"
 )
 
 type linterRunner struct {
@@ -20,7 +21,7 @@ type linterRunner struct {
 
 	outputFp io.Writer
 
-	filenameFilter *linter.FilenameFilter
+	filenameFilter *workspace.FilenameFilter
 
 	reportsExcludeChecksSet map[string]bool
 	reportsIncludeChecksSet map[string]bool
@@ -42,7 +43,7 @@ func (l *linterRunner) IsEnabledByFlags(checkName string) bool {
 }
 
 func (l *linterRunner) collectGitIgnoreFiles() error {
-	l.filenameFilter = linter.NewFilenameFilter(linter.ExcludeRegex)
+	l.filenameFilter = workspace.NewFilenameFilter(linter.ExcludeRegex)
 
 	if !l.args.gitignore {
 		return nil
@@ -59,7 +60,7 @@ func (l *linterRunner) collectGitIgnoreFiles() error {
 	// We collect all gitignore files along the way.
 	dir := workingDir
 	for {
-		m, err := linter.ParseGitignoreFromDir(dir)
+		m, err := workspace.ParseGitignoreFromDir(dir)
 		if err != nil {
 			return fmt.Errorf("read .gitignore: %v", err)
 		}
diff --git a/src/cmd/main.go b/src/cmd/main.go
index 32aa683..c921fdb 100644
--- a/src/cmd/main.go
+++ b/src/cmd/main.go
@@ -23,6 +23,7 @@ import (
 	"github.com/VKCOM/noverify/src/linter/lintapi"
 	"github.com/VKCOM/noverify/src/meta"
 	"github.com/VKCOM/noverify/src/rules"
+	"github.com/VKCOM/noverify/src/workspace"
 )
 
 // Line below implies that we have `https://github.com/VKCOM/phpstorm-stubs.git` cloned
@@ -138,7 +139,7 @@ func mainNoExit(ruleSets []*rules.Set, args *cmdlineArguments, cfg *MainConfig)
 		}()
 	}
 
-	linter.PHPExtensions = strings.Split(args.phpExtensionsArg, ",")
+	workspace.PHPExtensions = strings.Split(args.phpExtensionsArg, ",")
 
 	var l linterRunner
 	if err := l.Init(ruleSets, args); err != nil {
@@ -167,7 +168,7 @@ func mainNoExit(ruleSets []*rules.Set, args *cmdlineArguments, cfg *MainConfig)
 	linter.AnalysisFiles = flag.Args()
 
 	log.Printf("Indexing %+v", flag.Args())
-	linter.ParseFilenames(linter.ReadFilenames(flag.Args(), nil), l.allowDisableRegex)
+	linter.ParseFilenames(workspace.ReadFilenames(flag.Args(), nil), l.allowDisableRegex)
 	parseIndexOnlyFiles(&l)
 	meta.SetIndexingComplete(true)
 
@@ -177,7 +178,7 @@ func mainNoExit(ruleSets []*rules.Set, args *cmdlineArguments, cfg *MainConfig)
 	}
 
 	log.Printf("Linting")
-	reports := linter.ParseFilenames(linter.ReadFilenames(filenames, l.filenameFilter), l.allowDisableRegex)
+	reports := linter.ParseFilenames(workspace.ReadFilenames(filenames, l.filenameFilter), l.allowDisableRegex)
 	if args.outputBaseline {
 		if err := createBaseline(&l, cfg, reports); err != nil {
 			return 1, fmt.Errorf("write baseline: %v", err)
@@ -321,7 +322,7 @@ func initStubs() error {
 func LoadEmbeddedStubs(filenames []string) error {
 	var errorsCount int64
 
-	readStubs := func(ch chan linter.FileInfo) {
+	readStubs := func(ch chan workspace.FileInfo) {
 		for _, filename := range filenames {
 			data, err := stubs.Asset(filename)
 			if err != nil {
@@ -329,7 +330,7 @@ func LoadEmbeddedStubs(filenames []string) error {
 				atomic.AddInt64(&errorsCount, 1)
 				continue
 			}
-			ch <- linter.FileInfo{
+			ch <- workspace.FileInfo{
 				Filename: filename,
 				Contents: data,
 			}
diff --git a/src/langsrv/langsrv.go b/src/langsrv/langsrv.go
index f5d2c75..e4ca85c 100644
--- a/src/langsrv/langsrv.go
+++ b/src/langsrv/langsrv.go
@@ -23,6 +23,7 @@ import (
 	"github.com/VKCOM/noverify/src/php/parser/php7"
 	"github.com/VKCOM/noverify/src/solver"
 	"github.com/VKCOM/noverify/src/vscode"
+	"github.com/VKCOM/noverify/src/workspace"
 	"go.lsp.dev/uri"
 )
 
@@ -167,8 +168,8 @@ func handleInitialize(req *baseRequest) error {
 	go func() {
 		linter.AnalysisFiles = []string{params.RootURI.Filename()}
 
-		filter := linter.NewFilenameFilter(linter.ExcludeRegex)
-		linter.ParseFilenames(linter.ReadFilenames(linter.AnalysisFiles, filter), nil)
+		filter := workspace.NewFilenameFilter(linter.ExcludeRegex)
+		linter.ParseFilenames(workspace.ReadFilenames(linter.AnalysisFiles, filter), nil)
 
 		meta.SetIndexingComplete(true)
 
diff --git a/src/langsrv/refs.go b/src/langsrv/refs.go
index bc0b90a..7074706 100644
--- a/src/langsrv/refs.go
+++ b/src/langsrv/refs.go
@@ -14,6 +14,7 @@ import (
 	"github.com/VKCOM/noverify/src/solver"
 	"github.com/VKCOM/noverify/src/state"
 	"github.com/VKCOM/noverify/src/vscode"
+	"github.com/VKCOM/noverify/src/workspace"
 	"go.lsp.dev/uri"
 )
 
@@ -192,8 +193,8 @@ func refPosition(filename string, pos *position.Position) vscode.Location {
 type parseFn func(filename string, rootNode ir.Node, contents []byte, parser *php7.Parser) []vscode.Location
 
 func findReferences(substr string, parse parseFn) []vscode.Location {
-	cb := linter.ReadFilenames(linter.AnalysisFiles, nil)
-	ch := make(chan linter.FileInfo)
+	cb := workspace.ReadFilenames(linter.AnalysisFiles, nil)
+	ch := make(chan workspace.FileInfo)
 	go func() {
 		cb(ch)
 		close(ch)
diff --git a/src/linter/conf.go b/src/linter/conf.go
index 52a6e87..c278753 100644
--- a/src/linter/conf.go
+++ b/src/linter/conf.go
@@ -46,7 +46,6 @@ var (
 	Debug          bool
 	MaxConcurrency = runtime.NumCPU()
 	MaxFileSize    int
-	PHPExtensions  []string
 
 	// DebugParseDuration specifies the minimum parse duration for it to be printed to debug output.
 	DebugParseDuration time.Duration
diff --git a/src/linter/parser.go b/src/linter/parser.go
index 95da791..8233dfa 100644
--- a/src/linter/parser.go
+++ b/src/linter/parser.go
@@ -4,10 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"log"
-	"os"
-	"path/filepath"
 	"regexp"
 	dbg "runtime/debug"
 	"strings"
@@ -15,8 +12,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/karrick/godirwalk"
-	"github.com/monochromegane/go-gitignore"
 	"github.com/quasilyte/regex/syntax"
 
 	"github.com/VKCOM/noverify/src/git"
@@ -28,51 +23,9 @@ import (
 	"github.com/VKCOM/noverify/src/php/parser/php7"
 	"github.com/VKCOM/noverify/src/quickfix"
 	"github.com/VKCOM/noverify/src/rules"
+	"github.com/VKCOM/noverify/src/workspace"
 )
 
-type FileInfo struct {
-	Filename   string
-	Contents   []byte
-	LineRanges []git.LineRange
-}
-
-func isPHPExtension(filename string) bool {
-	fileExt := filepath.Ext(filename)
-	if fileExt == "" {
-		return false
-	}
-
-	fileExt = fileExt[1:] // cut "." in the beginning
-
-	for _, ext := range PHPExtensions {
-		if fileExt == ext {
-			return true
-		}
-	}
-
-	return false
-}
-
-func makePHPExtensionSuffixes() [][]byte {
-	res := make([][]byte, 0, len(PHPExtensions))
-	for _, ext := range PHPExtensions {
-		res = append(res, []byte("."+ext))
-	}
-	return res
-}
-
-func isPHPExtensionBytes(filename []byte, suffixes [][]byte) bool {
-	for _, suffix := range suffixes {
-		if bytes.HasSuffix(filename, suffix) {
-			return true
-		}
-	}
-
-	return false
-}
-
-type ReadCallback func(ch chan FileInfo)
-
 // ParseContents parses specified contents (or file) and returns *RootWalker.
 // Function does not update global meta.
 func ParseContents(filename string, contents []byte, lineRanges []git.LineRange, allowDisabled *regexp.Regexp) (rootNode *ir.Root, w *RootWalker, err error) {
@@ -229,306 +182,8 @@ func DebugMessage(msg string, args ...interface{}) {
 	}
 }
 
-// ParseGitignoreFromDir tries to parse a gitignore file at path/.gitignore.
-// If no such file exists, <nil, nil> is returned.
-func ParseGitignoreFromDir(path string) (gitignore.IgnoreMatcher, error) {
-	f, err := os.Open(filepath.Join(path, ".gitignore"))
-	switch {
-	case os.IsNotExist(err):
-		return nil, nil // No gitignore file, not an error
-	case err != nil:
-		return nil, err // Some unexpected error (e.g. access failure)
-	}
-	defer f.Close()
-	matcher := gitignore.NewGitIgnoreFromReader(path, f)
-	return matcher, nil
-}
-
-func readFilenames(ch chan<- FileInfo, filename string, filter *FilenameFilter) {
-	absFilename, err := filepath.Abs(filename)
-	if err == nil {
-		filename = absFilename
-	}
-
-	if filter == nil {
-		// No-op filter that doesn't track gitignore files.
-		filter = &FilenameFilter{}
-	}
-
-	// If we use stat here, it will return file info of an entry
-	// pointed by a symlink (if filename is a link).
-	// lstat is required for a symlink test below to succeed.
-	// If we ever want to permit top-level (CLI args) symlinks,
-	// caller should resolve them to a files that are pointed by them.
-	st, err := os.Lstat(filename)
-	if err != nil {
-		log.Fatalf("Could not stat file %s: %s", filename, err.Error())
-	}
-	if st.Mode()&os.ModeSymlink != 0 {
-		// filepath.Walk does not follow symlinks, but it does
-		// accept it as a root argument without an error.
-		// godirwalk.Walk can traverse symlinks with FollowSymbolicLinks=true,
-		// but we don't use it. It will give an error if root is
-		// a symlink, so we avoid calling Walk() on them.
-		return
-	}
-
-	if !st.IsDir() {
-		if filter.IgnoreFile(filename) {
-			return
-		}
-
-		ch <- FileInfo{Filename: filename}
-		return
-	}
-
-	// Start with a sentinel "" path to make last(gitignorePaths) safe
-	// without a length check.
-	gitignorePaths := []string{""}
-
-	walkOptions := &godirwalk.Options{
-		Unsorted: true,
-
-		Callback: func(path string, de *godirwalk.Dirent) error {
-			if de.IsDir() {
-				if filter.IgnoreDir(path) {
-					return filepath.SkipDir
-				}
-				// During indexing phase and with -gitignore=false
-				// we don't want to do extra FS operations.
-				if !filter.GitignoreIsEnabled() {
-					return nil
-				}
-
-				matcher, err := ParseGitignoreFromDir(path)
-				if err != nil {
-					linterError(path, "read .gitignore: %v", err)
-				}
-				if matcher != nil {
-					gitignorePaths = append(gitignorePaths, path)
-					filter.GitignorePush(path, matcher)
-				}
-				return nil
-			}
-
-			if !isPHPExtension(path) {
-				return nil
-			}
-			if filter.IgnoreFile(path) {
-				return nil
-			}
-
-			ch <- FileInfo{Filename: path}
-			return nil
-		},
-	}
-
-	if filter.GitignoreIsEnabled() {
-		walkOptions.PostChildrenCallback = func(path string, de *godirwalk.Dirent) error {
-			topGitignorePath := gitignorePaths[len(gitignorePaths)-1]
-			if topGitignorePath == path {
-				gitignorePaths = gitignorePaths[:len(gitignorePaths)-1]
-				filter.GitignorePop(path)
-			}
-			return nil
-		}
-	}
-
-	if err := godirwalk.Walk(filename, walkOptions); err != nil {
-		log.Fatalf("Could not walk filepath %s (%v)", filename, err)
-	}
-}
-
-// ReadFilenames returns callback that reads filenames into channel
-func ReadFilenames(filenames []string, filter *FilenameFilter) ReadCallback {
-	return func(ch chan FileInfo) {
-		for _, filename := range filenames {
-			readFilenames(ch, filename, filter)
-		}
-	}
-}
-
-// ReadChangesFromWorkTree returns callback that reads files from workTree dir that are changed
-func ReadChangesFromWorkTree(dir string, changes []git.Change) ReadCallback {
-	return func(ch chan FileInfo) {
-		for _, c := range changes {
-			if c.Type == git.Deleted {
-				continue
-			}
-
-			if !isPHPExtension(c.NewName) {
-				continue
-			}
-
-			filename := filepath.Join(dir, c.NewName)
-
-			contents, err := ioutil.ReadFile(filename)
-			if err != nil {
-				log.Fatalf("Could not read file %s: %s", filename, err.Error())
-			}
-
-			ch <- FileInfo{
-				Filename: filename,
-				Contents: contents,
-			}
-		}
-	}
-}
-
-// ReadFilesFromGit parses file contents in the specified commit
-func ReadFilesFromGit(repo, commitSHA1 string, ignoreRegex *regexp.Regexp) ReadCallback {
-	catter, err := git.NewCatter(repo)
-	if err != nil {
-		log.Fatalf("Could not start catter: %s", err.Error())
-	}
-
-	tree, err := git.GetTreeSHA1(catter, commitSHA1)
-	if err != nil {
-		log.Fatalf("Could not get tree sha1: %s", err.Error())
-	}
-
-	suffixes := makePHPExtensionSuffixes()
-
-	return func(ch chan FileInfo) {
-		start := time.Now()
-		idx := 0
-
-		err = catter.Walk(
-			"",
-			tree,
-			func(filename []byte) bool {
-				return isPHPExtensionBytes(filename, suffixes)
-			},
-			func(filename string, contents []byte) {
-				idx++
-				if time.Since(start) >= 2*time.Second {
-					start = time.Now()
-					action := "Indexed"
-					if meta.IsIndexingComplete() {
-						action = "Analyzed"
-					}
-					log.Printf("%s %d files from git", action, idx)
-				}
-
-				if ignoreRegex != nil && ignoreRegex.MatchString(filename) {
-					return
-				}
-
-				ch <- FileInfo{
-					Filename: filename,
-					Contents: contents,
-				}
-			},
-		)
-
-		if err != nil {
-			log.Fatalf("Could not walk: %s", err.Error())
-		}
-	}
-}
-
-// ReadOldFilesFromGit parses file contents in the specified commit, the old version
-func ReadOldFilesFromGit(repo, commitSHA1 string, changes []git.Change) ReadCallback {
-	changedMap := make(map[string][]git.LineRange, len(changes))
-	for _, ch := range changes {
-		if ch.Type == git.Added {
-			continue
-		}
-		changedMap[ch.OldName] = append(changedMap[ch.OldName], ch.OldLineRanges...)
-	}
-
-	catter, err := git.NewCatter(repo)
-	if err != nil {
-		log.Fatalf("Could not start catter: %s", err.Error())
-	}
-
-	tree, err := git.GetTreeSHA1(catter, commitSHA1)
-	if err != nil {
-		log.Fatalf("Could not get tree sha1: %s", err.Error())
-	}
-
-	suffixes := makePHPExtensionSuffixes()
-
-	return func(ch chan FileInfo) {
-		err = catter.Walk(
-			"",
-			tree,
-			func(filename []byte) bool {
-				if !isPHPExtensionBytes(filename, suffixes) {
-					return false
-				}
-
-				_, ok := changedMap[string(filename)]
-				return ok
-			},
-			func(filename string, contents []byte) {
-				ch <- FileInfo{
-					Filename:   filename,
-					Contents:   contents,
-					LineRanges: changedMap[filename],
-				}
-			},
-		)
-
-		if err != nil {
-			log.Fatalf("Could not walk: %s", err.Error())
-		}
-	}
-}
-
-// ReadFilesFromGitWithChanges parses file contents in the specified commit, but only specified ranges
-func ReadFilesFromGitWithChanges(repo, commitSHA1 string, changes []git.Change) ReadCallback {
-	changedMap := make(map[string][]git.LineRange, len(changes))
-	for _, ch := range changes {
-		if ch.Type == git.Deleted {
-			// TODO: actually support deletes too
-			continue
-		}
-
-		changedMap[ch.NewName] = append(changedMap[ch.NewName], ch.LineRanges...)
-	}
-
-	catter, err := git.NewCatter(repo)
-	if err != nil {
-		log.Fatalf("Could not start catter: %s", err.Error())
-	}
-
-	tree, err := git.GetTreeSHA1(catter, commitSHA1)
-	if err != nil {
-		log.Fatalf("Could not get tree sha1: %s", err.Error())
-	}
-
-	suffixes := makePHPExtensionSuffixes()
-
-	return func(ch chan FileInfo) {
-		err = catter.Walk(
-			"",
-			tree,
-			func(filename []byte) bool {
-				if !isPHPExtensionBytes(filename, suffixes) {
-					return false
-				}
-
-				_, ok := changedMap[string(filename)]
-				return ok
-			},
-			func(filename string, contents []byte) {
-				ch <- FileInfo{
-					Filename:   filename,
-					Contents:   contents,
-					LineRanges: changedMap[filename],
-				}
-			},
-		)
-
-		if err != nil {
-			log.Fatalf("Could not walk: %s", err.Error())
-		}
-	}
-}
-
 // ParseFilenames is used to do initial parsing of files.
-func ParseFilenames(readFileNamesFunc ReadCallback, allowDisabled *regexp.Regexp) []*Report {
+func ParseFilenames(readFileNamesFunc workspace.ReadCallback, allowDisabled *regexp.Regexp) []*Report {
 	start := time.Now()
 	defer func() {
 		lintdebug.Send("Processing time: %s", time.Since(start))
@@ -543,7 +198,7 @@ func ParseFilenames(readFileNamesFunc ReadCallback, allowDisabled *regexp.Regexp
 
 	lintdebug.Send("Parsing using %d cores", MaxConcurrency)
 
-	filenamesCh := make(chan FileInfo, 512)
+	filenamesCh := make(chan workspace.FileInfo, 512)
 
 	go func() {
 		readFileNamesFunc(filenamesCh)
@@ -574,7 +229,7 @@ func ParseFilenames(readFileNamesFunc ReadCallback, allowDisabled *regexp.Regexp
 	return allReports
 }
 
-func doParseFile(f FileInfo, needReports bool, allowDisabled *regexp.Regexp) (reports []*Report) {
+func doParseFile(f workspace.FileInfo, needReports bool, allowDisabled *regexp.Regexp) (reports []*Report) {
 	var err error
 
 	if DebugParseDuration > 0 {
@@ -604,7 +259,7 @@ func doParseFile(f FileInfo, needReports bool, allowDisabled *regexp.Regexp) (re
 	return reports
 }
 
-func InitStubs(readFileNamesFunc ReadCallback) {
+func InitStubs(readFileNamesFunc workspace.ReadCallback) {
 	meta.SetLoadingStubs(true)
 	ParseFilenames(readFileNamesFunc, nil)
 	meta.Info.InitStubs()
@@ -613,5 +268,5 @@ func InitStubs(readFileNamesFunc ReadCallback) {
 
 // InitStubsFromDir parses directory with PHPStorm stubs which has all internal PHP classes and functions declared.
 func InitStubsFromDir(dir string) {
-	InitStubs(ReadFilenames([]string{dir}, nil))
+	InitStubs(workspace.ReadFilenames([]string{dir}, nil))
 }
diff --git a/src/linter/filename_filter.go b/src/workspace/filename_filter.go
similarity index 86%
rename from src/linter/filename_filter.go
rename to src/workspace/filename_filter.go
index 789a631..1e8efcf 100644
--- a/src/linter/filename_filter.go
+++ b/src/workspace/filename_filter.go
@@ -1,4 +1,4 @@
-package linter
+package workspace
 
 import (
 	"regexp"
@@ -29,7 +29,6 @@ func (filter *FilenameFilter) InitialGitignorePush(path string, matcher gitignor
 	if !filter.gitignoreEnabled {
 		panic("add when gitignore is disabled")
 	}
-	DebugMessage("gitignore: add %s/.gitignore", path)
 	filter.initialMatchers[path] = struct{}{}
 	filter.matchers = append(filter.matchers, matcher)
 }
@@ -39,10 +38,8 @@ func (filter *FilenameFilter) GitignorePush(path string, matcher gitignore.Ignor
 		panic("pop when gitignore is disabled")
 	}
 	if _, ok := filter.initialMatchers[path]; ok {
-		DebugMessage("gitignore: don't push %s/.gitignore", path)
 		return
 	}
-	DebugMessage("gitignore: push %s/.gitignore", path)
 	filter.matchers = append(filter.matchers, matcher)
 }
 
@@ -51,10 +48,8 @@ func (filter *FilenameFilter) GitignorePop(path string) {
 		panic("pop when gitignore is disabled")
 	}
 	if _, ok := filter.initialMatchers[path]; ok {
-		DebugMessage("gitignore: don't pop %s/.gitignore", path)
 		return
 	}
-	DebugMessage("gitignore: pop %s/.gitignore", path)
 	filter.matchers = filter.matchers[:len(filter.matchers)-1]
 }
 
diff --git a/src/workspace/files.go b/src/workspace/files.go
new file mode 100644
index 0000000..1644beb
--- /dev/null
+++ b/src/workspace/files.go
@@ -0,0 +1,141 @@
+package workspace
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/VKCOM/noverify/src/git"
+	"github.com/karrick/godirwalk"
+)
+
+type ReadCallback func(ch chan FileInfo)
+
+type FileInfo struct {
+	Filename   string
+	Contents   []byte
+	LineRanges []git.LineRange
+}
+
+// ReadFilenames returns callback that reads filenames into channel
+func ReadFilenames(filenames []string, filter *FilenameFilter) ReadCallback {
+	return func(ch chan FileInfo) {
+		for _, filename := range filenames {
+			readFilenames(ch, filename, filter)
+		}
+	}
+}
+
+func readFilenames(ch chan<- FileInfo, filename string, filter *FilenameFilter) {
+	absFilename, err := filepath.Abs(filename)
+	if err == nil {
+		filename = absFilename
+	}
+
+	if filter == nil {
+		// No-op filter that doesn't track gitignore files.
+		filter = &FilenameFilter{}
+	}
+
+	// If we use stat here, it will return file info of an entry
+	// pointed by a symlink (if filename is a link).
+	// lstat is required for a symlink test below to succeed.
+	// If we ever want to permit top-level (CLI args) symlinks,
+	// caller should resolve them to a files that are pointed by them.
+	st, err := os.Lstat(filename)
+	if err != nil {
+		log.Fatalf("Could not stat file %s: %s", filename, err.Error())
+	}
+	if st.Mode()&os.ModeSymlink != 0 {
+		// filepath.Walk does not follow symlinks, but it does
+		// accept it as a root argument without an error.
+		// godirwalk.Walk can traverse symlinks with FollowSymbolicLinks=true,
+		// but we don't use it. It will give an error if root is
+		// a symlink, so we avoid calling Walk() on them.
+		return
+	}
+
+	if !st.IsDir() {
+		if filter.IgnoreFile(filename) {
+			return
+		}
+
+		ch <- FileInfo{Filename: filename}
+		return
+	}
+
+	// Start with a sentinel "" path to make last(gitignorePaths) safe
+	// without a length check.
+	gitignorePaths := []string{""}
+
+	walkOptions := &godirwalk.Options{
+		Unsorted: true,
+
+		Callback: func(path string, de *godirwalk.Dirent) error {
+			if de.IsDir() {
+				if filter.IgnoreDir(path) {
+					return filepath.SkipDir
+				}
+				// During indexing phase and with -gitignore=false
+				// we don't want to do extra FS operations.
+				if !filter.GitignoreIsEnabled() {
+					return nil
+				}
+
+				matcher, err := ParseGitignoreFromDir(path)
+				if err != nil {
+					log.Printf("read .gitignore: %v", err)
+				}
+				if matcher != nil {
+					gitignorePaths = append(gitignorePaths, path)
+					filter.GitignorePush(path, matcher)
+				}
+				return nil
+			}
+
+			if !isPHPExtension(path) {
+				return nil
+			}
+			if filter.IgnoreFile(path) {
+				return nil
+			}
+
+			ch <- FileInfo{Filename: path}
+			return nil
+		},
+	}
+
+	if filter.GitignoreIsEnabled() {
+		walkOptions.PostChildrenCallback = func(path string, de *godirwalk.Dirent) error {
+			topGitignorePath := gitignorePaths[len(gitignorePaths)-1]
+			if topGitignorePath == path {
+				gitignorePaths = gitignorePaths[:len(gitignorePaths)-1]
+				filter.GitignorePop(path)
+			}
+			return nil
+		}
+	}
+
+	if err := godirwalk.Walk(filename, walkOptions); err != nil {
+		log.Fatalf("Could not walk filepath %s (%v)", filename, err)
+	}
+}
+
+var PHPExtensions = []string{"php", "inc", "php5", "phtml"}
+
+func isPHPExtension(filename string) bool {
+	fileExt := filepath.Ext(filename)
+	if fileExt == "" {
+		return false
+	}
+
+	fileExt = fileExt[len("."):]
+
+	for _, ext := range PHPExtensions {
+		if fileExt == ext {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/src/workspace/git_files.go b/src/workspace/git_files.go
new file mode 100644
index 0000000..2c64640
--- /dev/null
+++ b/src/workspace/git_files.go
@@ -0,0 +1,227 @@
+package workspace
+
+import (
+	"bytes"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"time"
+
+	"github.com/VKCOM/noverify/src/git"
+	"github.com/VKCOM/noverify/src/meta"
+	"github.com/monochromegane/go-gitignore"
+)
+
+// ParseGitignoreFromDir tries to parse a gitignore file at path/.gitignore.
+// If no such file exists, <nil, nil> is returned.
+func ParseGitignoreFromDir(path string) (gitignore.IgnoreMatcher, error) {
+	f, err := os.Open(filepath.Join(path, ".gitignore"))
+	switch {
+	case os.IsNotExist(err):
+		return nil, nil // No gitignore file, not an error
+	case err != nil:
+		return nil, err // Some unexpected error (e.g. access failure)
+	}
+	defer f.Close()
+	matcher := gitignore.NewGitIgnoreFromReader(path, f)
+	return matcher, nil
+}
+
+// ReadChangesFromWorkTree returns callback that reads files from workTree dir that are changed
+func ReadChangesFromWorkTree(dir string, changes []git.Change) ReadCallback {
+	return func(ch chan FileInfo) {
+		for _, c := range changes {
+			if c.Type == git.Deleted {
+				continue
+			}
+
+			if !isPHPExtension(c.NewName) {
+				continue
+			}
+
+			filename := filepath.Join(dir, c.NewName)
+
+			contents, err := ioutil.ReadFile(filename)
+			if err != nil {
+				log.Fatalf("Could not read file %s: %s", filename, err.Error())
+			}
+
+			ch <- FileInfo{
+				Filename: filename,
+				Contents: contents,
+			}
+		}
+	}
+}
+
+// ReadFilesFromGit parses file contents in the specified commit
+func ReadFilesFromGit(repo, commitSHA1 string, ignoreRegex *regexp.Regexp) ReadCallback {
+	catter, err := git.NewCatter(repo)
+	if err != nil {
+		log.Fatalf("Could not start catter: %s", err.Error())
+	}
+
+	tree, err := git.GetTreeSHA1(catter, commitSHA1)
+	if err != nil {
+		log.Fatalf("Could not get tree sha1: %s", err.Error())
+	}
+
+	suffixes := makePHPExtensionSuffixes()
+
+	return func(ch chan FileInfo) {
+		start := time.Now()
+		idx := 0
+
+		err = catter.Walk(
+			"",
+			tree,
+			func(filename []byte) bool {
+				return isPHPExtensionBytes(filename, suffixes)
+			},
+			func(filename string, contents []byte) {
+				idx++
+				if time.Since(start) >= 2*time.Second {
+					start = time.Now()
+					action := "Indexed"
+					if meta.IsIndexingComplete() {
+						action = "Analyzed"
+					}
+					log.Printf("%s %d files from git", action, idx)
+				}
+
+				if ignoreRegex != nil && ignoreRegex.MatchString(filename) {
+					return
+				}
+
+				ch <- FileInfo{
+					Filename: filename,
+					Contents: contents,
+				}
+			},
+		)
+
+		if err != nil {
+			log.Fatalf("Could not walk: %s", err.Error())
+		}
+	}
+}
+
+// ReadOldFilesFromGit parses file contents in the specified commit, the old version
+func ReadOldFilesFromGit(repo, commitSHA1 string, changes []git.Change) ReadCallback {
+	changedMap := make(map[string][]git.LineRange, len(changes))
+	for _, ch := range changes {
+		if ch.Type == git.Added {
+			continue
+		}
+		changedMap[ch.OldName] = append(changedMap[ch.OldName], ch.OldLineRanges...)
+	}
+
+	catter, err := git.NewCatter(repo)
+	if err != nil {
+		log.Fatalf("Could not start catter: %s", err.Error())
+	}
+
+	tree, err := git.GetTreeSHA1(catter, commitSHA1)
+	if err != nil {
+		log.Fatalf("Could not get tree sha1: %s", err.Error())
+	}
+
+	suffixes := makePHPExtensionSuffixes()
+
+	return func(ch chan FileInfo) {
+		err = catter.Walk(
+			"",
+			tree,
+			func(filename []byte) bool {
+				if !isPHPExtensionBytes(filename, suffixes) {
+					return false
+				}
+
+				_, ok := changedMap[string(filename)]
+				return ok
+			},
+			func(filename string, contents []byte) {
+				ch <- FileInfo{
+					Filename:   filename,
+					Contents:   contents,
+					LineRanges: changedMap[filename],
+				}
+			},
+		)
+
+		if err != nil {
+			log.Fatalf("Could not walk: %s", err.Error())
+		}
+	}
+}
+
+// ReadFilesFromGitWithChanges parses file contents in the specified commit, but only specified ranges
+func ReadFilesFromGitWithChanges(repo, commitSHA1 string, changes []git.Change) ReadCallback {
+	changedMap := make(map[string][]git.LineRange, len(changes))
+	for _, ch := range changes {
+		if ch.Type == git.Deleted {
+			// TODO: actually support deletes too
+			continue
+		}
+
+		changedMap[ch.NewName] = append(changedMap[ch.NewName], ch.LineRanges...)
+	}
+
+	catter, err := git.NewCatter(repo)
+	if err != nil {
+		log.Fatalf("Could not start catter: %s", err.Error())
+	}
+
+	tree, err := git.GetTreeSHA1(catter, commitSHA1)
+	if err != nil {
+		log.Fatalf("Could not get tree sha1: %s", err.Error())
+	}
+
+	suffixes := makePHPExtensionSuffixes()
+
+	return func(ch chan FileInfo) {
+		err = catter.Walk(
+			"",
+			tree,
+			func(filename []byte) bool {
+				if !isPHPExtensionBytes(filename, suffixes) {
+					return false
+				}
+
+				_, ok := changedMap[string(filename)]
+				return ok
+			},
+			func(filename string, contents []byte) {
+				ch <- FileInfo{
+					Filename:   filename,
+					Contents:   contents,
+					LineRanges: changedMap[filename],
+				}
+			},
+		)
+
+		if err != nil {
+			log.Fatalf("Could not walk: %s", err.Error())
+		}
+	}
+}
+
+func makePHPExtensionSuffixes() [][]byte {
+	res := make([][]byte, 0, len(PHPExtensions))
+	for _, ext := range PHPExtensions {
+		res = append(res, []byte("."+ext))
+	}
+	return res
+}
+
+func isPHPExtensionBytes(filename []byte, suffixes [][]byte) bool {
+	for _, suffix := range suffixes {
+		if bytes.HasSuffix(filename, suffix) {
+			return true
+		}
+	}
+
+	return false
+}


Test patch:
diff --git a/src/tests/exprtype/exprtype_test.go b/src/tests/exprtype/exprtype_test.go
index e669aff..b88dda7 100644
--- a/src/tests/exprtype/exprtype_test.go
+++ b/src/tests/exprtype/exprtype_test.go
@@ -12,6 +12,7 @@ import (
 	"github.com/VKCOM/noverify/src/linter"
 	"github.com/VKCOM/noverify/src/linttest"
 	"github.com/VKCOM/noverify/src/meta"
+	"github.com/VKCOM/noverify/src/workspace"
 )
 
 // Tests in this file make it less likely that type solving will break
@@ -2161,8 +2162,8 @@ exprtype(Roo::$d, "string");
 func runExprTypeTest(t *testing.T, params *exprTypeTestParams) {
 	meta.ResetInfo()
 	if params.stubs != "" {
-		linter.InitStubs(func(ch chan linter.FileInfo) {
-			ch <- linter.FileInfo{
+		linter.InitStubs(func(ch chan workspace.FileInfo) {
+			ch <- workspace.FileInfo{
 				Filename: "stubs.php",
 				Contents: []byte(params.stubs),
 			}