Repository: goplus/gop
Pull request #2149 (2025-03-01 15:43:03)

tpl/cl: checkToken; demo: pseudo-2


Patch:
diff --git a/demo/gop-parser/pseudo/ast.gop b/demo/gop-parser/pseudo-1/ast.gop
similarity index 100%
rename from demo/gop-parser/pseudo/ast.gop
rename to demo/gop-parser/pseudo-1/ast.gop
diff --git a/demo/gop-parser/pseudo/gauss.pseudo b/demo/gop-parser/pseudo-1/gauss.pseudo
similarity index 100%
rename from demo/gop-parser/pseudo/gauss.pseudo
rename to demo/gop-parser/pseudo-1/gauss.pseudo
diff --git a/demo/gop-parser/pseudo/main.gop b/demo/gop-parser/pseudo-1/main.gop
similarity index 100%
rename from demo/gop-parser/pseudo/main.gop
rename to demo/gop-parser/pseudo-1/main.gop
diff --git a/demo/gop-parser/pseudo/parser.gop b/demo/gop-parser/pseudo-1/parser.gop
similarity index 100%
rename from demo/gop-parser/pseudo/parser.gop
rename to demo/gop-parser/pseudo-1/parser.gop
diff --git a/demo/gop-parser/pseudo/state.gox b/demo/gop-parser/pseudo-1/state.gox
similarity index 100%
rename from demo/gop-parser/pseudo/state.gox
rename to demo/gop-parser/pseudo-1/state.gox
diff --git a/demo/gop-parser/pseudo-2/gauss.pseudo b/demo/gop-parser/pseudo-2/gauss.pseudo
new file mode 100644
index 00000000..eef9e9a6
--- /dev/null
+++ b/demo/gop-parser/pseudo-2/gauss.pseudo
@@ -0,0 +1,10 @@
+DECLARE n, i, sum : INTEGER
+OUTPUT "please input n:"
+INPUT n
+i <- 1
+sum <- 0
+WHILE i <= n DO
+    sum <- sum + i
+    i <- i + 1
+ENDWHILE
+OUTPUT sum
diff --git a/demo/gop-parser/pseudo-2/main.gop b/demo/gop-parser/pseudo-2/main.gop
new file mode 100644
index 00000000..f7b67a0e
--- /dev/null
+++ b/demo/gop-parser/pseudo-2/main.gop
@@ -0,0 +1,46 @@
+import (
+	"gop/tpl"
+	"os"
+)
+
+if len(os.Args) < 2 {
+	println("Usage: pseudo <file>")
+	return
+}
+
+cl := tpl`
+doc = +(stmt ";")
+
+stmt = varStmt | constStmt | outputStmt | inputStmt | ifStmt | whileStmt | untilStmt | assignStmt
+
+varStmt = "DECLARE" IDENT % "," ":" typeExpr
+
+constStmt = "CONSTANT" IDENT "<-" expr
+
+assignStmt = IDENT "<-" expr
+
+outputStmt = "OUTPUT" expr
+
+inputStmt = "INPUT" IDENT % ","
+
+ifStmt = "IF" expr "THEN" ";" doc ?("ELSE" ";" doc) "ENDIF"
+
+whileStmt = "WHILE" expr "DO" ";" doc "ENDWHILE"
+
+untilStmt = "REPEAT" ";" doc "UNTIL" expr
+
+typeExpr = "INTEGER" | "REAL" | "STRING" | "BOOLEAN"
+
+expr = term2Expr % ("<" | "<=" | ">" | ">=" | "=" | "!=")
+
+term2Expr = term1Expr % ("+" | "-")
+
+term1Expr = unaryExpr % ("*" | "/")
+
+unaryExpr = operand | "-" unaryExpr
+
+operand = INT | FLOAT | STRING | IDENT | "(" expr ")"
+`!
+
+e := cl.parse(os.Args[1], nil, nil)!
+tpl.dump e
diff --git a/tpl/cl/compile.go b/tpl/cl/compile.go
index 2d528b07..d196babb 100644
--- a/tpl/cl/compile.go
+++ b/tpl/cl/compile.go
@@ -23,7 +23,6 @@ import (
 	"github.com/goplus/gop/tpl/ast"
 	"github.com/goplus/gop/tpl/matcher"
 	"github.com/goplus/gop/tpl/token"
-	"github.com/qiniu/x/ctype"
 	"github.com/qiniu/x/errors"
 )
 
@@ -138,25 +137,25 @@ func compileExpr(expr ast.Expr, ctx *context) (matcher.Matcher, bool) {
 				break
 			}
 			if tail != "" || multibyte {
-				ctx.addErrorf(expr.Pos(), "invalid literal %s", lit)
+				ctx.addError(expr.Pos(), "invalid literal "+lit)
 				break
 			}
-			return tokenExpr(v, expr, ctx)
+			return tokenExpr(token.Token(v), expr, ctx)
 		case token.STRING:
 			v, e := strconv.Unquote(lit)
-			if e != nil {
-				ctx.addErrorf(expr.Pos(), "invalid literal %s: %v", lit, e)
+			if e != nil || len(v) == 0 {
+				ctx.addError(expr.Pos(), "invalid literal "+lit)
 				break
 			}
-			if len(v) == 1 {
-				return tokenExpr(rune(v[0]), expr, ctx)
-			}
-			if ctype.IsCSymbol(v) {
+			if c := v[0]; c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_' {
 				return matcher.Literal(token.IDENT, v), true
 			}
+			if t, ok := checkToken(v); ok {
+				return tokenExpr(t, expr, ctx)
+			}
 			fallthrough
 		default:
-			ctx.addErrorf(expr.Pos(), "invalid literal %s", lit)
+			ctx.addError(expr.Pos(), "invalid literal "+lit)
 		}
 	case *ast.Sequence:
 		items := make([]matcher.Matcher, len(expr.Items))
@@ -208,11 +207,24 @@ func compileExpr(expr ast.Expr, ctx *context) (matcher.Matcher, bool) {
 	return nil, false
 }
 
-func tokenExpr(v rune, expr *ast.BasicLit, ctx *context) (matcher.Matcher, bool) {
-	tok := token.Token(v)
+func tokenExpr(tok token.Token, expr *ast.BasicLit, ctx *context) (matcher.Matcher, bool) {
 	if tok.Len() > 0 {
 		return matcher.Token(tok), true
 	}
 	ctx.addErrorf(expr.Pos(), "invalid token: %s", expr.Value)
 	return nil, false
 }
+
+func checkToken(v string) (ret token.Token, ok bool) {
+	if len(v) == 1 {
+		return token.Token(v[0]), true
+	}
+	token.ForEach(0, func(tok token.Token, lit string) int {
+		if lit == v {
+			ret, ok = tok, true
+			return token.Break
+		}
+		return 0
+	})
+	return
+}
diff --git a/tpl/matcher/match.go b/tpl/matcher/match.go
index 82c4f470..d1abae90 100644
--- a/tpl/matcher/match.go
+++ b/tpl/matcher/match.go
@@ -70,11 +70,11 @@ type gToken struct {
 
 func (p *gToken) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
 	if len(src) == 0 {
-		return 0, nil, &Error{} // TODO(xsw): error
+		return 0, nil, ctx.NewErrorf(ctx.FileEnd, "expect `%s`, but got EOF", p.tok)
 	}
 	t := src[0]
 	if t.Tok != p.tok {
-		return 0, nil, &Error{}
+		return 0, nil, ctx.NewErrorf(t.Pos, "expect `%s`, but got `%s`", p.tok, t.Tok)
 	}
 	return 1, t, nil
 }
@@ -116,23 +116,28 @@ type gChoice struct {
 func (p *gChoice) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
 	var nMax int
 	var errMax error
-	var multiErr = true
+	// var multiErr = true
 
 	for _, g := range p.options {
 		if n, result, err = g.Match(src, ctx); err == nil {
 			return
 		}
 		if n >= nMax {
-			if n == nMax {
-				multiErr = true
-			} else {
-				nMax, errMax, multiErr = n, err, false
-			}
+			nMax, errMax = n, err
+			/*
+				if n == nMax {
+					multiErr = true
+				} else {
+					nMax, errMax, multiErr = n, err, false
+				}
+			*/
 		}
 	}
-	if multiErr {
-		errMax = ctx.NewError(src[nMax].End(), "TODO: error msg") // TODO(xsw)
-	}
+	/*
+		if multiErr {
+			errMax = ctx.NewError(src[nMax].End(), "TODO: error msg") // TODO(xsw)
+		}
+	*/
 	return nMax, nil, errMax
 }
 
diff --git a/tpl/token/token.go b/tpl/token/token.go
index 627c6ae4..55909117 100644
--- a/tpl/token/token.go
+++ b/tpl/token/token.go
@@ -122,7 +122,7 @@ const (
 	DEFINE   // :=
 	ELLIPSIS // ...
 
-	_ = operator_beg
+	operator_end
 )
 
 // -----------------------------------------------------------------------------
@@ -201,12 +201,22 @@ var tokens = [...]string{
 	ELLIPSIS: "...",
 }
 
-// ForEach iterates all tokens.
-func ForEach(f func(tok Token, lit string)) {
-	for i, s := range tokens {
-		if s != "" {
-			f(Token(i), s)
+const (
+	Break = -1
+)
+
+// ForEach iterates tokens.
+func ForEach(from Token, f func(tok Token, lit string) int) {
+	if from == 0 {
+		from = operator_beg + 1
+	}
+	for from < operator_end {
+		if s := tokens[from]; s != "" {
+			if f(Token(from), s) == Break {
+				break
+			}
 		}
+		from++
 	}
 }
 
diff --git a/tpl/tpl.go b/tpl/tpl.go
index 2cd29f72..a0bd5867 100644
--- a/tpl/tpl.go
+++ b/tpl/tpl.go
@@ -173,7 +173,9 @@ func Dump(result any) {
 func Fdump(w io.Writer, result any, prefix, indent string) {
 	switch result := result.(type) {
 	case *Token:
-		fmt.Fprint(w, prefix, result, "\n")
+		if result.Tok != token.SEMICOLON {
+			fmt.Fprint(w, prefix, result, "\n")
+		}
 	case []any:
 		if isPlain(result) {
 			fmt.Print(prefix, "[")


Test patch:
diff --git a/tpl/token/token_test.go b/tpl/token/token_test.go
index d3c3af86..8854afc4 100644
--- a/tpl/token/token_test.go
+++ b/tpl/token/token_test.go
@@ -34,10 +34,11 @@ func TestToken(t *testing.T) {
 		t.Fatal("TokLen +")
 	}
 	count := 0
-	ForEach(func(tok Token, name string) {
+	ForEach(0, func(tok Token, name string) int {
 		count++
+		return 0
 	})
-	if count != 59 {
+	if count != 25 {
 		t.Fatal("ForEach:", count)
 	}
 }