Repository: VKCOM/noverify
Pull request #670 (2020-08-13 13:21:52)

cmd/php-guru: run indexing and use meta info in ir/normalization
Signed-off-by: Iskander Sharipov <quasilyte@gmail.com>


Patch:
diff --git a/src/cmd/flags.go b/src/cmd/flags.go
index b31274d..b65ea41 100644
--- a/src/cmd/flags.go
+++ b/src/cmd/flags.go
@@ -73,6 +73,16 @@ type cmdlineArguments struct {
 	disableCache bool
 }
 
+func DefaultCacheDir() string {
+	defaultCacheDir, err := os.UserCacheDir()
+	if err != nil {
+		defaultCacheDir = ""
+	} else {
+		defaultCacheDir = filepath.Join(defaultCacheDir, "noverify-cache")
+	}
+	return defaultCacheDir
+}
+
 func bindFlags(ruleSets []*rules.Set, args *cmdlineArguments) {
 	var enabledByDefault []string
 	declaredChecks := linter.GetDeclaredChecks()
@@ -85,13 +95,6 @@ func bindFlags(ruleSets []*rules.Set, args *cmdlineArguments) {
 		enabledByDefault = append(enabledByDefault, rset.Names...)
 	}
 
-	defaultCacheDir, err := os.UserCacheDir()
-	if err != nil {
-		defaultCacheDir = ""
-	} else {
-		defaultCacheDir = filepath.Join(defaultCacheDir, "noverify-cache")
-	}
-
 	flag.Usage = func() {
 		out := flag.CommandLine.Output()
 		fmt.Fprintf(out, "Usage of noverify:\n")
@@ -169,7 +172,7 @@ func bindFlags(ruleSets []*rules.Set, args *cmdlineArguments) {
 	flag.BoolVar(&linter.LangServer, "lang-server", false, "Run language server for VS Code")
 
 	flag.StringVar(&linter.StubsDir, "stubs-dir", "", "phpstorm-stubs directory")
-	flag.StringVar(&linter.CacheDir, "cache-dir", defaultCacheDir, "Directory for linter cache (greatly improves indexing speed)")
+	flag.StringVar(&linter.CacheDir, "cache-dir", DefaultCacheDir(), "Directory for linter cache (greatly improves indexing speed)")
 	flag.BoolVar(&args.disableCache, "disable-cache", false, "If set, cache is not used and cache-dir is ignored")
 
 	flag.StringVar(&args.unusedVarPattern, "unused-var-regex", `^_$`,
diff --git a/src/cmd/php-guru/dupcode/fileindexer.go b/src/cmd/php-guru/dupcode/fileindexer.go
index daffca1..d65dc3c 100644
--- a/src/cmd/php-guru/dupcode/fileindexer.go
+++ b/src/cmd/php-guru/dupcode/fileindexer.go
@@ -10,6 +10,7 @@ type fileIndexer struct {
 	fileContents []byte
 	filename     string
 	args         *arguments
+	normLevel    normalizationLevel
 }
 
 func (indexer *fileIndexer) CollectFuncs(root *ir.Root) {
@@ -67,9 +68,9 @@ func (indexer *fileIndexer) walkFunc(n *ir.FunctionStmt) {
 }
 
 func (indexer *fileIndexer) collectFunc(n ir.Node, className, funcName string, params, stmts []ir.Node) {
-	if indexer.args.norm != 0 {
+	if indexer.normLevel != normNone {
 		conf := normalize.Config{
-			NormalizeMore: indexer.args.norm > 1,
+			NormalizeMore: indexer.normLevel > normSafe,
 		}
 		stmts = normalize.FuncBody(conf, params, stmts)
 	}
diff --git a/src/cmd/php-guru/dupcode/main.go b/src/cmd/php-guru/dupcode/main.go
index cf630d5..ca1d825 100644
--- a/src/cmd/php-guru/dupcode/main.go
+++ b/src/cmd/php-guru/dupcode/main.go
@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"sync"
 
+	"github.com/VKCOM/noverify/src/cmd"
 	"github.com/VKCOM/noverify/src/cmd/php-guru/guru"
 	"github.com/VKCOM/noverify/src/ir/irconv"
 	"github.com/VKCOM/noverify/src/php/parseutil"
@@ -22,6 +23,19 @@ import (
 
 // TODO: handle methods better (don't give up on self-references).
 
+type normalizationLevel int
+
+const (
+	// -norm=0 is "no normalization".
+	normNone normalizationLevel = iota
+	// -norm=1 enables basic normalization.
+	normFast
+	// -norm=2 also does indexing, so normalization can do more.
+	normSafe
+	// -norm=3 also enables risky normalizations.
+	normMore
+)
+
 func Main(ctx *guru.Context) (int, error) {
 	var args arguments
 	flag.BoolVar(&args.checkPrivate, "private", false,
@@ -30,10 +44,12 @@ func Main(ctx *guru.Context) (int, error) {
 		"whether auto-generated files should be analyzed")
 	flag.StringVar(&args.exclude, "exclude", "",
 		"regexp that excludes files from the analysis")
+	flag.StringVar(&args.cacheDir, "cache-dir", cmd.DefaultCacheDir(),
+		"Directory for linter cache (greatly improves indexing speed)")
 	flag.UintVar(&args.minComplexity, "min-complexity", 20,
 		"min function complexity level threshold")
-	flag.UintVar(&args.norm, "norm", 1,
-		"code normalization level: 0 for none, 1 for safe, 2 if you feel lucky")
+	flag.UintVar(&args.norm, "norm", uint(normFast),
+		"code normalization level: 0 for none, 1 for fast-only, 2 for safe, 3 if you feel lucky")
 
 	flag.Parse()
 
@@ -45,9 +61,16 @@ func Main(ctx *guru.Context) (int, error) {
 			return 1, fmt.Errorf("parse -exclude: %v", err)
 		}
 	}
+	var normLevel normalizationLevel
 	switch args.norm {
-	case 0, 1, 2:
-		// OK.
+	case 0:
+		normLevel = normNone
+	case 1:
+		normLevel = normFast
+	case 2:
+		normLevel = normSafe
+	case 3:
+		normLevel = normMore
 	default:
 		return 1, fmt.Errorf("invalid -norm level %d", args.norm)
 	}
@@ -55,6 +78,9 @@ func Main(ctx *guru.Context) (int, error) {
 	targets := flag.Args()
 
 	filter := workspace.NewFilenameFilter(exclude)
+	if normLevel > normFast {
+		runIndexing(args.cacheDir, targets, filter)
+	}
 	readFileNamesFunc := workspace.ReadFilenames(targets, filter)
 	filenamesCh := make(chan workspace.FileInfo, 512)
 	go func() {
@@ -88,6 +114,7 @@ func Main(ctx *guru.Context) (int, error) {
 					fileContents: data,
 					filename:     f.Filename,
 					args:         &args,
+					normLevel:    normLevel,
 				}
 				indexer.CollectFuncs(rootIR)
 			}
@@ -111,6 +138,7 @@ type arguments struct {
 	checkPrivate  bool
 	checkAutogen  bool
 	exclude       string
+	cacheDir      string
 	minComplexity uint
 	norm          uint
 }
diff --git a/src/cmd/php-guru/dupcode/utils.go b/src/cmd/php-guru/dupcode/utils.go
index 7b6a8ff..7e77e0f 100644
--- a/src/cmd/php-guru/dupcode/utils.go
+++ b/src/cmd/php-guru/dupcode/utils.go
@@ -1,8 +1,13 @@
 package dupcode
 
 import (
+	"github.com/VKCOM/noverify/src/cmd"
+	"github.com/VKCOM/noverify/src/cmd/stubs"
 	"github.com/VKCOM/noverify/src/ir"
 	"github.com/VKCOM/noverify/src/ir/irutil"
+	"github.com/VKCOM/noverify/src/linter"
+	"github.com/VKCOM/noverify/src/meta"
+	"github.com/VKCOM/noverify/src/workspace"
 )
 
 func findNode(root ir.Node, pred func(n ir.Node) bool) bool {
@@ -35,3 +40,20 @@ func hasModifier(list []*ir.Identifier, key string) bool {
 	}
 	return false
 }
+
+func runIndexing(cacheDir string, targets []string, filter *workspace.FilenameFilter) {
+	linter.CacheDir = cacheDir
+	linter.AnalysisFiles = targets
+
+	// If we don't do this, the program will hang.
+	go linter.MemoryLimiterThread()
+
+	// Handle stubs.
+	filenames := stubs.AssetNames()
+	cmd.LoadEmbeddedStubs(filenames)
+
+	// Handle workspace files.
+	linter.ParseFilenames(workspace.ReadFilenames(targets, filter), nil)
+
+	meta.SetIndexingComplete(true)
+}
diff --git a/src/ir/normalize/normalize.go b/src/ir/normalize/normalize.go
index 63896d4..9fa4cc8 100644
--- a/src/ir/normalize/normalize.go
+++ b/src/ir/normalize/normalize.go
@@ -8,10 +8,14 @@ import (
 	"github.com/VKCOM/noverify/src/ir"
 	"github.com/VKCOM/noverify/src/ir/irfmt"
 	"github.com/VKCOM/noverify/src/ir/irutil"
+	"github.com/VKCOM/noverify/src/meta"
+	"github.com/VKCOM/noverify/src/solver"
 )
 
 type Config struct {
-	CurrentClass  string
+	CurrentClass string
+	Namespace    string
+
 	NormalizeMore bool
 }
 
@@ -33,8 +37,8 @@ func FuncBody(conf Config, params, statements []ir.Node) []ir.Node {
 }
 
 type normalizer struct {
-	out      []ir.Node
 	conf     Config
+	out      []ir.Node
 	varNames map[string]int
 	globals  map[string]struct{}
 }
@@ -132,8 +136,8 @@ func (norm *normalizer) sortStatements(statements []ir.Node) []ir.Node {
 
 func (norm *normalizer) LeaveNode(n ir.Node) {}
 
-func (norm *normalizer) EnterNode(w ir.Node) bool {
-	switch n := w.(type) {
+func (norm *normalizer) EnterNode(n ir.Node) bool {
+	switch n := n.(type) {
 	case *ir.GlobalStmt:
 		for _, v := range n.Vars {
 			v, ok := v.(*ir.SimpleVar)
@@ -305,7 +309,7 @@ func (norm *normalizer) normalizedExpr(e ir.Node) ir.Node {
 			break
 		}
 		if !strings.ContainsAny(e.Value, `'\`) {
-			e.Value = `'` + strings.Trim(e.Value, `"`) + `'`
+			e.Value = `'` + irutil.Unquote(e.Value) + `'`
 		}
 
 	case *ir.TernaryExpr:
@@ -333,6 +337,23 @@ func (norm *normalizer) normalizedExpr(e ir.Node) ir.Node {
 				e.Constant = &ir.Name{Value: `false`}
 			}
 		}
+		// Named const => its value
+		if !meta.IsIndexingComplete() {
+			return e
+		}
+		info, ok := norm.getConstant(e.Constant)
+		if !ok {
+			return e
+		}
+		value := info.Value.Value
+		switch info.Value.Type {
+		case meta.Integer:
+			return &ir.Lnumber{Value: fmt.Sprint(value)}
+		case meta.String:
+			return &ir.String{Value: `'` + value.(string) + `'`}
+		default:
+			return e
+		}
 
 	case *ir.FunctionCallExpr:
 		funcName, ok := e.Function.(*ir.Name)
@@ -422,6 +443,13 @@ func (norm *normalizer) renameVar(v *ir.SimpleVar) {
 	v.Name = fmt.Sprintf("v%d", id)
 }
 
+func (norm *normalizer) getConstant(n ir.Node) (meta.ConstantInfo, bool) {
+	var cs meta.ClassParseState
+	cs.Namespace = norm.conf.Namespace
+	_, info, ok := solver.GetConstant(&cs, n)
+	return info, ok
+}
+
 var (
 	nullConstNode   = &ir.ConstFetchExpr{Constant: &ir.Name{Value: "null"}}
 	emptyStringNode = &ir.String{Value: `''`}


Test patch:
diff --git a/src/ir/normalize/normalize_test.go b/src/ir/normalize/normalize_test.go
index de1768f..4343152 100644
--- a/src/ir/normalize/normalize_test.go
+++ b/src/ir/normalize/normalize_test.go
@@ -8,6 +8,8 @@ import (
 	"github.com/VKCOM/noverify/src/ir"
 	"github.com/VKCOM/noverify/src/ir/irconv"
 	"github.com/VKCOM/noverify/src/ir/irfmt"
+	"github.com/VKCOM/noverify/src/linttest"
+	"github.com/VKCOM/noverify/src/meta"
 	"github.com/VKCOM/noverify/src/php/parser/php7"
 	"github.com/VKCOM/noverify/src/php/parseutil"
 )
@@ -91,11 +93,46 @@ func TestNormalizeStmtList(t *testing.T) {
 	}
 }
 
+type normalizationTest struct {
+	orig string
+	want string
+}
+
+func runNormalizeTests(t *testing.T, tests []normalizationTest) {
+	t.Helper()
+
+	conf := Config{CurrentClass: `Foo`}
+	for _, test := range tests {
+		n, _, err := parseutil.ParseStmt([]byte(test.orig))
+		if err != nil {
+			t.Errorf("parse `%s`: %v", test.orig, err)
+			return
+		}
+		irNode := irconv.ConvertNode(n)
+		normalized := FuncBody(conf, nil, []ir.Node{irNode})
+		have := strings.TrimSuffix(irfmt.Node(normalized[0]), ";")
+		if have != test.want {
+			t.Errorf("normalize `%s`:\nhave: %s\nwant: %s",
+				test.orig, have, test.want)
+			return
+		}
+		n2, _, err := parseutil.ParseStmt([]byte(have))
+		if err != nil {
+			t.Errorf("parse normalized `%s`: %v", test.orig, err)
+			return
+		}
+		irNode2 := irconv.ConvertNode(n2)
+		normalized2 := FuncBody(conf, nil, []ir.Node{irNode2})
+		have2 := strings.TrimSuffix(irfmt.Node(normalized2[0]), ";")
+		if have != have2 {
+			t.Errorf("second normalization of `%s`:\nhave: %s\nwant: %s",
+				test.orig, have2, have)
+		}
+	}
+}
+
 func TestNormalizeExpr(t *testing.T) {
-	tests := []struct {
-		orig string
-		want string
-	}{
+	runNormalizeTests(t, []normalizationTest{
 		{`new T`, `new T()`},
 
 		{`"$x"`, `'' . $v0`},
@@ -129,36 +166,23 @@ func TestNormalizeExpr(t *testing.T) {
 
 		{`$x ? $x : $y`, `$v0 ?: $v1`},
 		{`f() ? f() : $y`, `f() ? f() : $v0`},
-	}
+	})
+}
 
-	conf := Config{CurrentClass: `Foo`}
-	for _, test := range tests {
-		n, _, err := parseutil.ParseStmt([]byte(test.orig))
-		if err != nil {
-			t.Errorf("parse `%s`: %v", test.orig, err)
-			continue
-		}
-		irNode := irconv.ConvertNode(n)
-		normalized := FuncBody(conf, nil, []ir.Node{irNode})
-		have := strings.TrimSuffix(irfmt.Node(normalized[0]), ";")
-		if have != test.want {
-			t.Errorf("normalize `%s`:\nhave: %s\nwant: %s",
-				test.orig, have, test.want)
-			continue
-		}
-		n2, _, err := parseutil.ParseStmt([]byte(have))
-		if err != nil {
-			t.Errorf("parse normalized `%s`: %v", test.orig, err)
-			continue
-		}
-		irNode2 := irconv.ConvertNode(n2)
-		normalized2 := FuncBody(conf, nil, []ir.Node{irNode2})
-		have2 := strings.TrimSuffix(irfmt.Node(normalized2[0]), ";")
-		if have != have2 {
-			t.Errorf("second normalization of `%s`:\nhave: %s\nwant: %s",
-				test.orig, have2, have)
-		}
-	}
+func TestNormalizeExprAfterIndexing(t *testing.T) {
+	linttest.ParseTestFile(t, "defs.php", `<?php
+const ZERO = 0;
+const HELLO_WORLD = 'hello, world';
+const LOCALHOST = "127.0.0.1";
+`)
+	meta.SetIndexingComplete(true)
+	defer meta.SetIndexingComplete(false)
+
+	runNormalizeTests(t, []normalizationTest{
+		{`ZERO`, `0`},
+		{`HELLO_WORLD`, `'hello, world'`},
+		{`LOCALHOST`, `'127.0.0.1'`},
+	})
 }
 
 func parseStmtList(s string) ([]ir.Node, error) {