Repository: goplus/gop
Pull request #2152 (2025-03-01 16:29:23)

v1.3.5


Patch:
diff --git a/VERSION b/VERSION
deleted file mode 100644
index 968e750a..00000000
--- a/VERSION
+++ /dev/null
@@ -1,1 +0,0 @@
-v1.3.2
diff --git a/demo/gop-parser/pseudo/ast.gop b/demo/gop-parser/pseudo-1/ast.gop
similarity index 100%
rename from demo/gop-parser/pseudo/ast.gop
rename to demo/gop-parser/pseudo-1/ast.gop
diff --git a/demo/gop-parser/pseudo/gauss.pseudo b/demo/gop-parser/pseudo-1/gauss.pseudo
similarity index 100%
rename from demo/gop-parser/pseudo/gauss.pseudo
rename to demo/gop-parser/pseudo-1/gauss.pseudo
diff --git a/demo/gop-parser/pseudo/main.gop b/demo/gop-parser/pseudo-1/main.gop
similarity index 100%
rename from demo/gop-parser/pseudo/main.gop
rename to demo/gop-parser/pseudo-1/main.gop
diff --git a/demo/gop-parser/pseudo/parser.gop b/demo/gop-parser/pseudo-1/parser.gop
similarity index 100%
rename from demo/gop-parser/pseudo/parser.gop
rename to demo/gop-parser/pseudo-1/parser.gop
diff --git a/demo/gop-parser/pseudo/state.gox b/demo/gop-parser/pseudo-1/state.gox
similarity index 100%
rename from demo/gop-parser/pseudo/state.gox
rename to demo/gop-parser/pseudo-1/state.gox
diff --git a/demo/gop-parser/pseudo-2/gauss.pseudo b/demo/gop-parser/pseudo-2/gauss.pseudo
new file mode 100644
index 00000000..eef9e9a6
--- /dev/null
+++ b/demo/gop-parser/pseudo-2/gauss.pseudo
@@ -0,0 +1,10 @@
+DECLARE n, i, sum : INTEGER
+OUTPUT "please input n:"
+INPUT n
+i <- 1
+sum <- 0
+WHILE i <= n DO
+    sum <- sum + i
+    i <- i + 1
+ENDWHILE
+OUTPUT sum
diff --git a/demo/gop-parser/pseudo-2/main.gop b/demo/gop-parser/pseudo-2/main.gop
new file mode 100644
index 00000000..7c9ee963
--- /dev/null
+++ b/demo/gop-parser/pseudo-2/main.gop
@@ -0,0 +1,46 @@
+import (
+	"gop/tpl"
+	"os"
+)
+
+if len(os.Args) < 2 {
+	println("Usage: pseudo <file>")
+	return
+}
+
+cl := tpl`
+doc = +(stmt ";")
+
+stmt = varStmt | constStmt | outputStmt | inputStmt | ifStmt | whileStmt | untilStmt | assignStmt
+
+varStmt = "DECLARE" IDENT % "," ":" typeExpr
+
+constStmt = "CONSTANT" IDENT "<-" expr
+
+assignStmt = IDENT "<-" expr
+
+outputStmt = "OUTPUT" expr
+
+inputStmt = "INPUT" IDENT % ","
+
+ifStmt = "IF" expr "THEN" ";" doc ?("ELSE" ";" doc) "ENDIF"
+
+whileStmt = "WHILE" expr "DO" ";" doc "ENDWHILE"
+
+untilStmt = "REPEAT" ";" doc "UNTIL" expr
+
+typeExpr = "INTEGER" | "REAL" | "STRING" | "BOOLEAN"
+
+expr = term2Expr % ("<" | "<=" | ">" | ">=" | "=" | "<>")
+
+term2Expr = term1Expr % ("+" | "-")
+
+term1Expr = unaryExpr % ("*" | "/")
+
+unaryExpr = operand | "-" unaryExpr
+
+operand = INT | FLOAT | STRING | IDENT | "(" expr ")"
+`!
+
+e := cl.parse(os.Args[1], nil, nil)!
+tpl.dump e
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 3842de77..1994eb8c 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -994,7 +994,7 @@ scanAgain:
 			} else if s.ch == '>' { // <>
 				s.next()
 				tok = token.BIDIARROW
-			} else { // << <<=
+			} else { // < <= << <<=
 				tok = s.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
 			}
 		case '>':
diff --git a/tpl/cl/compile.go b/tpl/cl/compile.go
index 2d528b07..d196babb 100644
--- a/tpl/cl/compile.go
+++ b/tpl/cl/compile.go
@@ -23,7 +23,6 @@ import (
 	"github.com/goplus/gop/tpl/ast"
 	"github.com/goplus/gop/tpl/matcher"
 	"github.com/goplus/gop/tpl/token"
-	"github.com/qiniu/x/ctype"
 	"github.com/qiniu/x/errors"
 )
 
@@ -138,25 +137,25 @@ func compileExpr(expr ast.Expr, ctx *context) (matcher.Matcher, bool) {
 				break
 			}
 			if tail != "" || multibyte {
-				ctx.addErrorf(expr.Pos(), "invalid literal %s", lit)
+				ctx.addError(expr.Pos(), "invalid literal "+lit)
 				break
 			}
-			return tokenExpr(v, expr, ctx)
+			return tokenExpr(token.Token(v), expr, ctx)
 		case token.STRING:
 			v, e := strconv.Unquote(lit)
-			if e != nil {
-				ctx.addErrorf(expr.Pos(), "invalid literal %s: %v", lit, e)
+			if e != nil || len(v) == 0 {
+				ctx.addError(expr.Pos(), "invalid literal "+lit)
 				break
 			}
-			if len(v) == 1 {
-				return tokenExpr(rune(v[0]), expr, ctx)
-			}
-			if ctype.IsCSymbol(v) {
+			if c := v[0]; c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_' {
 				return matcher.Literal(token.IDENT, v), true
 			}
+			if t, ok := checkToken(v); ok {
+				return tokenExpr(t, expr, ctx)
+			}
 			fallthrough
 		default:
-			ctx.addErrorf(expr.Pos(), "invalid literal %s", lit)
+			ctx.addError(expr.Pos(), "invalid literal "+lit)
 		}
 	case *ast.Sequence:
 		items := make([]matcher.Matcher, len(expr.Items))
@@ -208,11 +207,24 @@ func compileExpr(expr ast.Expr, ctx *context) (matcher.Matcher, bool) {
 	return nil, false
 }
 
-func tokenExpr(v rune, expr *ast.BasicLit, ctx *context) (matcher.Matcher, bool) {
-	tok := token.Token(v)
+func tokenExpr(tok token.Token, expr *ast.BasicLit, ctx *context) (matcher.Matcher, bool) {
 	if tok.Len() > 0 {
 		return matcher.Token(tok), true
 	}
 	ctx.addErrorf(expr.Pos(), "invalid token: %s", expr.Value)
 	return nil, false
 }
+
+func checkToken(v string) (ret token.Token, ok bool) {
+	if len(v) == 1 {
+		return token.Token(v[0]), true
+	}
+	token.ForEach(0, func(tok token.Token, lit string) int {
+		if lit == v {
+			ret, ok = tok, true
+			return token.Break
+		}
+		return 0
+	})
+	return
+}
diff --git a/tpl/matcher/match.go b/tpl/matcher/match.go
index 82c4f470..40b9767a 100644
--- a/tpl/matcher/match.go
+++ b/tpl/matcher/match.go
@@ -70,11 +70,11 @@ type gToken struct {
 
 func (p *gToken) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
 	if len(src) == 0 {
-		return 0, nil, &Error{} // TODO(xsw): error
+		return 0, nil, ctx.NewErrorf(ctx.FileEnd, "expect `%s`, but got EOF", p.tok)
 	}
 	t := src[0]
 	if t.Tok != p.tok {
-		return 0, nil, &Error{}
+		return 0, nil, ctx.NewErrorf(t.Pos, "expect `%s`, but got `%s`", p.tok, t.Tok)
 	}
 	return 1, t, nil
 }
@@ -116,23 +116,28 @@ type gChoice struct {
 func (p *gChoice) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
 	var nMax int
 	var errMax error
-	var multiErr = true
+	// var multiErr = true
 
 	for _, g := range p.options {
 		if n, result, err = g.Match(src, ctx); err == nil {
 			return
 		}
 		if n >= nMax {
-			if n == nMax {
-				multiErr = true
-			} else {
-				nMax, errMax, multiErr = n, err, false
-			}
+			nMax, errMax = n, err
+			/*
+				if n == nMax {
+					multiErr = true
+				} else {
+					nMax, errMax, multiErr = n, err, false
+				}
+			*/
 		}
 	}
-	if multiErr {
-		errMax = ctx.NewError(src[nMax].End(), "TODO: error msg") // TODO(xsw)
-	}
+	/*
+		if multiErr {
+			errMax = ctx.NewError(src[nMax].End(), "TODO: error msg") // TODO(xsw)
+		}
+	*/
 	return nMax, nil, errMax
 }
 
@@ -148,7 +153,8 @@ type gSequence struct {
 }
 
 func (p *gSequence) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
-	rets := make([]any, len(p.items))
+	nitems := len(p.items)
+	rets := make([]any, nitems)
 	for i, g := range p.items {
 		n1, ret1, err1 := g.Match(src[n:], ctx)
 		if err1 != nil {
@@ -157,6 +163,12 @@ func (p *gSequence) Match(src []*types.Token, ctx *Context) (n int, result any,
 		rets[i] = ret1
 		n += n1
 	}
+	if nitems == 2 {
+		if _, ok := p.items[1].(*gRepeat0); ok && len(rets[1].([]any)) == 0 {
+			result = rets[0]
+			return
+		}
+	}
 	result = rets
 	return
 }
diff --git a/tpl/scanner/scanner.go b/tpl/scanner/scanner.go
index de65b44c..16087781 100644
--- a/tpl/scanner/scanner.go
+++ b/tpl/scanner/scanner.go
@@ -702,9 +702,14 @@ scanAgain:
 				insertSemi = true
 			}
 		case '-':
-			t.Tok = s.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
-			if t.Tok == token.DEC {
-				insertSemi = true
+			if s.ch == '>' { // ->
+				s.next()
+				t.Tok = token.SRARROW
+			} else { // -- -=
+				t.Tok = s.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
+				if t.Tok == token.DEC {
+					insertSemi = true
+				}
 			}
 		case '*':
 			t.Tok = s.switch2(token.MUL, token.MUL_ASSIGN)
@@ -751,16 +756,19 @@ scanAgain:
 		case '^':
 			t.Tok = s.switch2(token.XOR, token.XOR_ASSIGN)
 		case '<':
-			if s.ch == '-' {
+			if s.ch == '-' { // <-
 				s.next()
 				t.Tok = token.ARROW
-			} else {
+			} else if s.ch == '>' { // <>
+				s.next()
+				t.Tok = token.BIDIARROW
+			} else { // <= << <<=
 				t.Tok = s.switch4(token.LT, token.LE, '<', token.SHL, token.SHL_ASSIGN)
 			}
 		case '>':
 			t.Tok = s.switch4(token.GT, token.GE, '>', token.SHR, token.SHR_ASSIGN)
 		case '=':
-			t.Tok = s.switch2(token.ASSIGN, token.EQ)
+			t.Tok = s.switch3(token.ASSIGN, token.EQ, '>', token.DRARROW)
 		case '!':
 			t.Tok = s.switch2(token.NOT, token.NE)
 		case '&':
@@ -774,6 +782,8 @@ scanAgain:
 			t.Tok = s.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
 		case '?':
 			t.Tok = token.QUESTION
+		case '$':
+			t.Tok = token.ENV
 		case '~':
 			t.Tok = token.TILDE
 		case '@':
diff --git a/tpl/token/token.go b/tpl/token/token.go
index 627c6ae4..e03f3112 100644
--- a/tpl/token/token.go
+++ b/tpl/token/token.go
@@ -87,6 +87,7 @@ const (
 	QUESTION  = '?'
 	TILDE     = '~'
 	AT        = '@'
+	ENV       = '$'
 )
 
 const (
@@ -122,7 +123,11 @@ const (
 	DEFINE   // :=
 	ELLIPSIS // ...
 
-	_ = operator_beg
+	DRARROW   // =>
+	SRARROW   // ->
+	BIDIARROW // <>
+
+	operator_end
 )
 
 // -----------------------------------------------------------------------------
@@ -169,6 +174,7 @@ var tokens = [...]string{
 	QUESTION:  "?",
 	TILDE:     "~",
 	AT:        "@",
+	ENV:       "$",
 
 	SHL:     "<<",
 	SHR:     ">>",
@@ -199,14 +205,28 @@ var tokens = [...]string{
 	GE:       ">=",
 	DEFINE:   ":=",
 	ELLIPSIS: "...",
+
+	DRARROW:   "=>",
+	SRARROW:   "->",
+	BIDIARROW: "<>",
 }
 
-// ForEach iterates all tokens.
-func ForEach(f func(tok Token, lit string)) {
-	for i, s := range tokens {
-		if s != "" {
-			f(Token(i), s)
+const (
+	Break = -1
+)
+
+// ForEach iterates tokens.
+func ForEach(from Token, f func(tok Token, lit string) int) {
+	if from == 0 {
+		from = operator_beg + 1
+	}
+	for from < operator_end {
+		if s := tokens[from]; s != "" {
+			if f(Token(from), s) == Break {
+				break
+			}
 		}
+		from++
 	}
 }
 
diff --git a/tpl/tpl.go b/tpl/tpl.go
index fbdd79ab..a0bd5867 100644
--- a/tpl/tpl.go
+++ b/tpl/tpl.go
@@ -156,7 +156,9 @@ func isEOL(tok token.Token) bool {
 func isPlain(result []any) bool {
 	for _, v := range result {
 		if _, ok := v.(*Token); !ok {
-			return false
+			if a, ok := v.([]any); !ok || len(a) != 0 {
+				return false
+			}
 		}
 	}
 	return true
@@ -171,7 +173,9 @@ func Dump(result any) {
 func Fdump(w io.Writer, result any, prefix, indent string) {
 	switch result := result.(type) {
 	case *Token:
-		fmt.Fprint(w, prefix, result, "\n")
+		if result.Tok != token.SEMICOLON {
+			fmt.Fprint(w, prefix, result, "\n")
+		}
 	case []any:
 		if isPlain(result) {
 			fmt.Print(prefix, "[")


Test patch:
diff --git a/tpl/token/token_test.go b/tpl/token/token_test.go
index d3c3af86..a275f14f 100644
--- a/tpl/token/token_test.go
+++ b/tpl/token/token_test.go
@@ -34,10 +34,11 @@ func TestToken(t *testing.T) {
 		t.Fatal("TokLen +")
 	}
 	count := 0
-	ForEach(func(tok Token, name string) {
+	ForEach(0, func(tok Token, name string) int {
 		count++
+		return 0
 	})
-	if count != 59 {
+	if count != 28 {
 		t.Fatal("ForEach:", count)
 	}
 }