Repository: goplus/gop
Pull request #2139 (2025-02-28 13:17:39)

tpl/matcher


Patch:
diff --git a/demo/gop-parser/_spec/expr.gop b/demo/gop-parser/_spec/expr.gop
index eb547527..6fde8d7e 100644
--- a/demo/gop-parser/_spec/expr.gop
+++ b/demo/gop-parser/_spec/expr.gop
@@ -11,28 +11,29 @@ node (
 	BinaryExpr Expr
 )
 
-expr = termExpr | (X:expr binOp Y:expr as BinaryExpr {
+expr = x:termExpr *(binOp:("+" | "-") Y:termExpr as BinaryExpr {
+	X: x
 	OpPos: binOp.Pos
 	Op: binOp.Tok
 })
 
-binOp = "+" | "-"
-
-termExpr = unaryExpr | (X:termExpr termOp Y:termExpr as BinaryExpr {
-	OpPos: termOp.Pos
-	Op: termOp.Tok
+termExpr = x:unaryExpr *(binOp:("*" | "/" | "%") Y:unaryExpr as BinaryExpr {
+	X: x
+	OpPos: binOp.Pos
+	Op: binOp.Tok
 })
 
-termOp = "*" | "/" | "%"
-
-unaryExpr = primaryExpr | (unaryOp X:unaryExpr as UnaryExpr {
-	OpPos: unaryOp.Pos
-	Op: unaryOp.Tok
+unaryExpr = primaryExpr | (uOp:("+" | "-" | "!" | "^") X:primaryExpr as UnaryExpr {
+	OpPos: uOp.Pos
+	Op: uOp.Tok
 })
 
-unaryOp = "+" | "-" | "!" | "^"
-
-primaryExpr = operand | CallExpr
+primaryExpr = operand *(lp:"(" ?(Args:(expr % ",") ?ell:"..." ?",") rp:")" as CallExpr {
+	Fun: operand
+	Lparen: lp.Pos
+	Ellipsis: ell != nil ? ell.Pos : 0
+	Rparen: rp.Pos
+})
 
 operand = BasicLit | Ident | ParenExpr
 
@@ -51,10 +52,3 @@ ParenExpr = lp:"(" X:expr rp:")" {
 	Lparen: lp.Pos
 	Rparen: rp.Pos
 }
-
-CallExpr = primaryExpr lp:"(" ?(Args:(expr % ",") ?ell:"..." ?",") rp:")" {
-	Fun: primaryExpr
-	Lparen: lp.Pos
-	Ellipsis: ell != nil ? ell.Pos : 0
-	Rparen: rp.Pos
-}
diff --git a/tpl/matcher/match.go b/tpl/matcher/match.go
new file mode 100644
index 00000000..56447019
--- /dev/null
+++ b/tpl/matcher/match.go
@@ -0,0 +1,233 @@
+/*
+ * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package matcher
+
+import (
+	"github.com/goplus/gop/tpl/token"
+	"github.com/goplus/gop/tpl/types"
+)
+
+// -----------------------------------------------------------------------------
+// Matcher
+
+// Context represents the context of a matching process.
+type Context struct {
+}
+
+// Matcher represents a matcher.
+type Matcher interface {
+	Match(src []*types.Token, ctx *Context) (n int, result any, err error)
+}
+
+// -----------------------------------------------------------------------------
+
+// Error represents a matching error.
+type Error struct {
+}
+
+func (p *Error) Error() string {
+	panic("todo")
+}
+
+// -----------------------------------------------------------------------------
+
+type gToken struct {
+	tok token.Token
+}
+
+func (p *gToken) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
+	if len(src) == 0 {
+		return 0, nil, &Error{} // TODO(xsw): err
+	}
+	t := src[0]
+	if t.Tok != p.tok {
+		return 0, nil, &Error{} // TODO(xsw): err
+	}
+	return 1, t, nil
+}
+
+// Token: ADD, SUB, IDENT, INT, FLOAT, CHAR, STRING, etc.
+func Token(tok token.Token) Matcher {
+	return &gToken{tok}
+}
+
+// -----------------------------------------------------------------------------
+
+type gLiteral struct {
+	tok token.Token
+	lit string
+}
+
+func (p *gLiteral) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
+	if len(src) == 0 {
+		return 0, nil, &Error{} // TODO(xsw): err
+	}
+	t := src[0]
+	if t.Tok != p.tok || t.Lit != p.lit {
+		return 0, nil, &Error{} // TODO(xsw): err
+	}
+	return 1, t, nil
+}
+
+// Literal: "abc", 'a', 123, 1.23, etc.
+func Literal(tok token.Token, lit string) Matcher {
+	return &gLiteral{tok, lit}
+}
+
+// -----------------------------------------------------------------------------
+
+type gChoice struct {
+	options []Matcher
+}
+
+func (p *gChoice) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
+	var nMax int
+	var errMax error
+	var multiErr = true
+
+	for _, g := range p.options {
+		if n, result, err = g.Match(src, ctx); err == nil {
+			return
+		}
+		if n >= nMax {
+			if n == nMax {
+				multiErr = true
+			} else {
+				nMax, errMax, multiErr = n, err, false
+			}
+		}
+	}
+	if multiErr {
+		errMax = &Error{} // TODO(xsw): err
+	}
+	return nMax, nil, errMax
+}
+
+// Choice: R1 | R2 | ... | Rn
+func Choice(options ...Matcher) Matcher {
+	return &gChoice{options}
+}
+
+// -----------------------------------------------------------------------------
+
+type gSequence struct {
+	items []Matcher
+}
+
+func (p *gSequence) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
+	rets := make([]any, len(p.items))
+	for i, g := range p.items {
+		n1, ret1, err1 := g.Match(src[n:], ctx)
+		if err1 != nil {
+			return n + n1, nil, err1
+		}
+		rets[i] = ret1
+		n += n1
+	}
+	result = rets
+	return
+}
+
+// Sequence: R1 R2 ... Rn
+func Sequence(items ...Matcher) Matcher {
+	return &gSequence{items}
+}
+
+// -----------------------------------------------------------------------------
+
+type gRepeat0 struct {
+	r Matcher
+}
+
+func (p *gRepeat0) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
+	g := p.r
+	rets := make([]any, 0, 2)
+	for {
+		n1, ret1, err1 := g.Match(src, ctx)
+		if err1 != nil {
+			result = rets
+			return
+		}
+		rets = append(rets, ret1)
+		n += n1
+		src = src[n1:]
+	}
+}
+
+// Repeat0: *R
+func Repeat0(r Matcher) Matcher {
+	return &gRepeat0{r}
+}
+
+// -----------------------------------------------------------------------------
+
+type gRepeat1 struct {
+	r Matcher
+}
+
+func (p *gRepeat1) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
+	g := p.r
+	n, ret0, err := g.Match(src, ctx)
+	if err != nil {
+		return
+	}
+
+	rets := make([]any, 1, 2)
+	rets[0] = ret0
+	for {
+		n1, ret1, err1 := g.Match(src[n:], ctx)
+		if err1 != nil {
+			result = rets
+			return
+		}
+		rets = append(rets, ret1)
+		n += n1
+	}
+}
+
+// Repeat1: +R
+func Repeat1(r Matcher) Matcher {
+	return &gRepeat1{r}
+}
+
+// -----------------------------------------------------------------------------
+
+type gRepeat01 struct {
+	r Matcher
+}
+
+func (p *gRepeat01) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
+	n, result, err = p.r.Match(src, ctx)
+	if err != nil {
+		return 0, nil, nil
+	}
+	return
+}
+
+// Repeat01: ?R
+func Repeat01(r Matcher) Matcher {
+	return &gRepeat01{r}
+}
+
+// -----------------------------------------------------------------------------
+
+// List: R1 % R2 is equivalent to R1 *(R2 R1)
+func List(a, b Matcher) Matcher {
+	return Sequence(a, Repeat0(Sequence(b, a)))
+}
+
+// -----------------------------------------------------------------------------
diff --git a/tpl/scanner/scanner.go b/tpl/scanner/scanner.go
index 097c68b2..c696875a 100644
--- a/tpl/scanner/scanner.go
+++ b/tpl/scanner/scanner.go
@@ -25,24 +25,9 @@ import (
 	"unicode/utf8"
 
 	"github.com/goplus/gop/tpl/token"
+	"github.com/goplus/gop/tpl/types"
 )
 
-// A Token is a lexical unit returned by Scan.
-type Token struct {
-	Tok token.Token
-	Pos token.Pos
-	Lit string
-}
-
-// End returns end position of this token.
-func (p *Token) End() token.Pos {
-	n := len(p.Lit)
-	if n == 0 {
-		n = p.Tok.Len()
-	}
-	return p.Pos + token.Pos(n)
-}
-
 // An ScanErrorHandler may be provided to Scanner.Init. If a syntax error is
 // encountered and a handler was installed, the handler is called with a
 // position and an error message. The position points to the beginning of
@@ -630,7 +615,7 @@ func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Tok
 // Scan adds line information to the file added to the file
 // set with Init. Token positions are relative to that file
 // and thus relative to the file set.
-func (s *Scanner) Scan() (t Token) {
+func (s *Scanner) Scan() (t types.Token) {
 scanAgain:
 	s.skipWhitespace()
 
diff --git a/tpl/types/types.go b/tpl/types/types.go
new file mode 100644
index 00000000..dbba974d
--- /dev/null
+++ b/tpl/types/types.go
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package types
+
+import (
+	"github.com/goplus/gop/tpl/token"
+)
+
+// A Token is a lexical unit returned by Scan.
+type Token struct {
+	Tok token.Token
+	Pos token.Pos
+	Lit string
+}
+
+// End returns end position of this token.
+func (p *Token) End() token.Pos {
+	n := len(p.Lit)
+	if n == 0 {
+		n = p.Tok.Len()
+	}
+	return p.Pos + token.Pos(n)
+}


Test patch:
diff --git a/tpl/scanner/scanner_test.go b/tpl/scanner/scanner_test.go
index 1dc9679a..86737411 100644
--- a/tpl/scanner/scanner_test.go
+++ b/tpl/scanner/scanner_test.go
@@ -20,8 +20,11 @@ import (
 	"testing"
 
 	"github.com/goplus/gop/tpl/token"
+	"github.com/goplus/gop/tpl/types"
 )
 
+type Token = types.Token
+
 type tokenTest struct {
 	Pos     token.Pos
 	Kind    token.Token
@@ -132,7 +135,7 @@ world !=
 		if c.Tok == token.EOF {
 			break
 		}
-		expect := Token{expected[i].Kind, expected[i].Pos, expected[i].Literal}
+		expect := Token{Tok: expected[i].Kind, Pos: expected[i].Pos, Lit: expected[i].Literal}
 		if c != expect {
 			t.Fatal("Scan failed:", c, expect)
 		}
@@ -155,7 +158,7 @@ world !=
 		if expected[i].Kind == token.COMMENT {
 			i++
 		}
-		expect := Token{expected[i].Kind, expected[i].Pos, expected[i].Literal}
+		expect := Token{Tok: expected[i].Kind, Pos: expected[i].Pos, Lit: expected[i].Literal}
 		if c != expect {
 			t.Fatal("Scan failed:", c.Pos, c.Lit, expect)
 		}
@@ -183,7 +186,7 @@ world !=
 		if expected[i].Kind == token.COMMENT {
 			i++
 		}
-		expect := Token{expected[i].Kind, expected[i].Pos, expected[i].Literal}
+		expect := Token{Tok: expected[i].Kind, Pos: expected[i].Pos, Lit: expected[i].Literal}
 		if c != expect {
 			t.Fatal("Scan failed:", c.Pos, c.Lit, expect)
 		}