Repository: VKCOM/noverify
Pull request #588 (2020-07-16 10:09:28)

parseutil: new package added
Move []byte->AST parsing utils from phpgrep to a separate
package as it's convenient to have such functions for tests
that want to parse some expressions without having to
mess with <?php tags and explicit parser creation.


Patch:
diff --git a/src/langsrv/langsrv.go b/src/langsrv/langsrv.go
index 94da0c7..8190b41 100644
--- a/src/langsrv/langsrv.go
+++ b/src/langsrv/langsrv.go
@@ -818,6 +818,11 @@ func getInstanceProperties(className string) (res []string) {
 }
 
 func getMethodCompletionItems(st *meta.ClassParseState, str string, sc *meta.Scope) (result []vscode.CompletionItem) {
+	// It looks like this function could use parseutil.ParseStmt which
+	// handles <?php tag and returns stmt[0], but I wouldn't dare
+	// to re-write it without testing.
+	// TODO: use parseutil.ParseStmt here.
+
 	strTemp := "<?php " + strings.TrimSuffix(str, "->") + ";"
 	parser := php7.NewParser([]byte(strTemp))
 	parser.Parse()
diff --git a/src/php/parseutil/parseutil.go b/src/php/parseutil/parseutil.go
new file mode 100644
index 0000000..28bace4
--- /dev/null
+++ b/src/php/parseutil/parseutil.go
@@ -0,0 +1,58 @@
+package parseutil
+
+import (
+	"bytes"
+	"errors"
+
+	"github.com/VKCOM/noverify/src/php/parser/node"
+	"github.com/VKCOM/noverify/src/php/parser/node/stmt"
+	"github.com/VKCOM/noverify/src/php/parser/php7"
+)
+
+// Parse combines ParseFile and ParseStmt.
+//
+// If input code is a proper PHP file, it's parsed with ParseFile.
+// Otherwise it parsed with ParseStmt.
+//
+// Useful for testing.
+func Parse(code []byte) (node.Node, []byte, error) {
+	if bytes.HasPrefix(code, []byte("<?")) || bytes.HasPrefix(code, []byte("<?php")) {
+		n, err := ParseFile(code)
+		return n, code, err
+	}
+	return ParseStmt(code)
+}
+
+// ParseFile parses PHP file sources and returns its AST root.
+//
+// Useful for testing.
+func ParseFile(code []byte) (*node.Root, error) {
+	p := php7.NewParser(code)
+	p.WithFreeFloating()
+	p.Parse()
+	if len(p.GetErrors()) != 0 {
+		return nil, errors.New(p.GetErrors()[0].String())
+	}
+	return p.GetRootNode(), nil
+}
+
+// ParseStmt parses a single PHP statement (which can be an expression).
+//
+// Due to the fact that we extend the input slice with <?php tags
+// so our parser can handle it, updated slice is returned.
+// Result node source positions are precise only for that updated slice.
+//
+// Useful for testing.
+func ParseStmt(code []byte) (node.Node, []byte, error) {
+	code = append([]byte("<?php "), code...)
+	code = append(code, ';')
+	root, err := ParseFile(code)
+	if err != nil {
+		return nil, code, err
+	}
+	stmts := root.Stmts
+	if len(stmts) == 0 {
+		return &stmt.Nop{}, code, nil
+	}
+	return root.Stmts[0], code, nil
+}
diff --git a/src/phpgrep/compile.go b/src/phpgrep/compile.go
index 44d96b6..b4ba9c4 100644
--- a/src/phpgrep/compile.go
+++ b/src/phpgrep/compile.go
@@ -7,6 +7,7 @@ import (
 	"github.com/VKCOM/noverify/src/php/parser/node/scalar"
 	"github.com/VKCOM/noverify/src/php/parser/node/stmt"
 	"github.com/VKCOM/noverify/src/php/parser/walker"
+	"github.com/VKCOM/noverify/src/php/parseutil"
 )
 
 type compiler struct {
@@ -15,7 +16,7 @@ type compiler struct {
 }
 
 func compile(opts *Compiler, pattern []byte) (*Matcher, error) {
-	root, src, err := parsePHP7(pattern)
+	root, src, err := parseutil.Parse(pattern)
 	if err != nil {
 		return nil, err
 	}
diff --git a/src/phpgrep/utils.go b/src/phpgrep/utils.go
index 378cc8b..6a65ab1 100644
--- a/src/phpgrep/utils.go
+++ b/src/phpgrep/utils.go
@@ -1,16 +1,12 @@
 package phpgrep
 
 import (
-	"bytes"
-	"errors"
-
 	"github.com/VKCOM/noverify/src/php/parser/node"
 	"github.com/VKCOM/noverify/src/php/parser/node/expr"
 	"github.com/VKCOM/noverify/src/php/parser/node/expr/assign"
 	"github.com/VKCOM/noverify/src/php/parser/node/expr/binary"
 	"github.com/VKCOM/noverify/src/php/parser/node/scalar"
 	"github.com/VKCOM/noverify/src/php/parser/node/stmt"
-	"github.com/VKCOM/noverify/src/php/parser/php7"
 	"github.com/VKCOM/noverify/src/php/parser/position"
 )
 
@@ -165,34 +161,3 @@ func matchMetaVar(n node.Node, s string) bool {
 		return false
 	}
 }
-
-func parsePHP7(code []byte) (node.Node, []byte, error) {
-	if bytes.HasPrefix(code, []byte("<?")) || bytes.HasPrefix(code, []byte("<?php")) {
-		n, err := parsePHP7root(code)
-		return n, code, err
-	}
-	return parsePHP7expr(code)
-}
-
-func parsePHP7expr(code []byte) (node.Node, []byte, error) {
-	code = append([]byte("<?php "), code...)
-	code = append(code, ';')
-	root, err := parsePHP7root(code)
-	if err != nil {
-		return nil, code, err
-	}
-	stmts := root.(*node.Root).Stmts
-	if len(stmts) == 0 {
-		return &stmt.Nop{}, code, nil
-	}
-	return root.(*node.Root).Stmts[0], code, nil
-}
-
-func parsePHP7root(code []byte) (node.Node, error) {
-	p := php7.NewParser(code)
-	p.Parse()
-	if len(p.GetErrors()) != 0 {
-		return nil, errors.New(p.GetErrors()[0].String())
-	}
-	return p.GetRootNode(), nil
-}
diff --git a/src/rules/parser.go b/src/rules/parser.go
index 0f206d9..05e7571 100644
--- a/src/rules/parser.go
+++ b/src/rules/parser.go
@@ -1,7 +1,6 @@
 package rules
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -14,7 +13,7 @@ import (
 	"github.com/VKCOM/noverify/src/php/parser/freefloating"
 	"github.com/VKCOM/noverify/src/php/parser/node"
 	"github.com/VKCOM/noverify/src/php/parser/node/stmt"
-	"github.com/VKCOM/noverify/src/php/parser/php7"
+	"github.com/VKCOM/noverify/src/php/parseutil"
 	"github.com/VKCOM/noverify/src/phpdoc"
 	"github.com/VKCOM/noverify/src/phpgrep"
 )
@@ -56,13 +55,10 @@ func (p *parser) parse(filename string, r io.Reader) (*Set, error) {
 	if err != nil {
 		return res, err
 	}
-	q := php7.NewParser(sources)
-	q.WithFreeFloating()
-	q.Parse()
-	if errs := q.GetErrors(); len(errs) != 0 {
-		return res, errors.New(errs[0].String())
+	root, err := parseutil.ParseFile(sources)
+	if err != nil {
+		return res, err
 	}
-	root := q.GetRootNode()
 
 	// Convert PHP file into the rule set.
 	p.filename = filename


Test patch:
diff --git a/src/phpgrep/matcher_test.go b/src/phpgrep/matcher_test.go
index b3f25d5..72f7cea 100644
--- a/src/phpgrep/matcher_test.go
+++ b/src/phpgrep/matcher_test.go
@@ -8,10 +8,11 @@ import (
 	"github.com/VKCOM/noverify/src/php/astutil"
 	"github.com/VKCOM/noverify/src/php/parser/node"
 	"github.com/VKCOM/noverify/src/php/parser/node/stmt"
+	"github.com/VKCOM/noverify/src/php/parseutil"
 )
 
 func mustParse(t testing.TB, code string) node.Node {
-	n, _, err := parsePHP7([]byte(code))
+	n, _, err := parseutil.Parse([]byte(code))
 	if err != nil {
 		t.Fatalf("parse `%s`: %v", code, err)
 	}