Repository: goplus/gop
Pull request #2136 (2025-02-28 00:07:56)

tpl spec


Patch:
diff --git a/demo/gop-parser/_spec/tpl.gop b/demo/gop-parser/_spec/tpl.gop
new file mode 100644
index 00000000..e7e1453e
--- /dev/null
+++ b/demo/gop-parser/_spec/tpl.gop
@@ -0,0 +1,9 @@
+doc = +rule
+
+rule = IDENT '=' expr ';'
+
+expr = +term % '|'
+
+term = factor *('%' factor)
+
+factor = IDENT | CHAR | STRING | ('*' | '+' | '?') factor | '(' expr ')'
diff --git a/tpl/ast/ast.go b/tpl/ast/ast.go
index 24325764..6ab4ae57 100644
--- a/tpl/ast/ast.go
+++ b/tpl/ast/ast.go
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package tpl
+package ast
 
 import (
 	"github.com/goplus/gop/tpl/token"
@@ -90,8 +90,8 @@ func (p *Ident) exprNode()      {}
 
 // BasicLit: STRING | CHAR
 type BasicLit struct {
-	ValuePos token.Pos // literal position
-	Kind     token.Tok // token.STRING or token.CHAR
+	ValuePos token.Pos   // literal position
+	Kind     token.Token // token.STRING or token.CHAR
 	Value    string
 }
 
@@ -125,9 +125,9 @@ func (p *Sequence) exprNode()      {}
 
 // UnaryExpr: *R, +R or ?R
 type UnaryExpr struct {
-	OpPos token.Pos // operator position
-	Op    token.Tok // operator: token.MUL, token.ADD or token.QUESTION
-	X     Expr      // operand
+	OpPos token.Pos   // operator position
+	Op    token.Token // operator: token.MUL, token.ADD or token.QUESTION
+	X     Expr        // operand
 }
 
 func (p *UnaryExpr) Pos() token.Pos { return p.OpPos }
@@ -138,10 +138,10 @@ func (p *UnaryExpr) exprNode()      {}
 
 // BinaryExpr: R1 % R2
 type BinaryExpr struct {
-	X     Expr      // left operand
-	OpPos token.Pos // operator position
-	Op    token.Tok // operator: token.REM (list operator)
-	Y     Expr      // right operand
+	X     Expr        // left operand
+	OpPos token.Pos   // operator position
+	Op    token.Token // operator: token.REM (list operator)
+	Y     Expr        // right operand
 }
 
 func (p *BinaryExpr) Pos() token.Pos { return p.X.Pos() }
diff --git a/tpl/scanner/scanner.go b/tpl/scanner/scanner.go
index 0d1bc55b..097c68b2 100644
--- a/tpl/scanner/scanner.go
+++ b/tpl/scanner/scanner.go
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package tpl
+package scanner
 
 import (
 	"bytes"
@@ -29,7 +29,7 @@ import (
 
 // A Token is a lexical unit returned by Scan.
 type Token struct {
-	Tok token.Tok
+	Tok token.Token
 	Pos token.Pos
 	Lit string
 }
@@ -38,7 +38,7 @@ type Token struct {
 func (p *Token) End() token.Pos {
 	n := len(p.Lit)
 	if n == 0 {
-		n = token.TokLen(p.Tok)
+		n = p.Tok.Len()
 	}
 	return p.Pos + token.Pos(n)
 }
@@ -312,7 +312,7 @@ func (s *Scanner) scanMantissa(base int) {
 	}
 }
 
-func (s *Scanner) scanNumber(seenDecimalPoint bool) (token.Tok, string) {
+func (s *Scanner) scanNumber(seenDecimalPoint bool) (token.Token, string) {
 	// digitVal(s.ch) < 10
 	offs := s.offset
 	tok := token.INT
@@ -567,7 +567,7 @@ func (s *Scanner) skipWhitespace() {
 // respectively. Otherwise, the result is tok0 if there was no other
 // matching character, or tok2 if the matching character was ch2.
 
-func (s *Scanner) switch2(tok0, tok1 token.Tok) token.Tok {
+func (s *Scanner) switch2(tok0, tok1 token.Token) token.Token {
 	if s.ch == '=' {
 		s.next()
 		return tok1
@@ -575,7 +575,7 @@ func (s *Scanner) switch2(tok0, tok1 token.Tok) token.Tok {
 	return tok0
 }
 
-func (s *Scanner) switch3(tok0, tok1 token.Tok, ch2 rune, tok2 token.Tok) token.Tok {
+func (s *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token {
 	if s.ch == '=' {
 		s.next()
 		return tok1
@@ -587,7 +587,7 @@ func (s *Scanner) switch3(tok0, tok1 token.Tok, ch2 rune, tok2 token.Tok) token.
 	return tok0
 }
 
-func (s *Scanner) switch4(tok0, tok1 token.Tok, ch2 rune, tok2, tok3 token.Tok) token.Tok {
+func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token {
 	if s.ch == '=' {
 		s.next()
 		return tok1
diff --git a/tpl/token/token.go b/tpl/token/token.go
index e0edc641..627c6ae4 100644
--- a/tpl/token/token.go
+++ b/tpl/token/token.go
@@ -22,10 +22,10 @@ import (
 
 // -----------------------------------------------------------------------------
 
-type Tok uint
+type Token uint
 
-func (tok Tok) String() (s string) {
-	if tok < Tok(len(tokens)) {
+func (tok Token) String() (s string) {
+	if tok < Token(len(tokens)) {
 		s = tokens[tok]
 	}
 	if s == "" {
@@ -34,10 +34,20 @@ func (tok Tok) String() (s string) {
 	return
 }
 
+// Len returns
+// 1) len of is token literal, if token is an operator.
+// 2) 0 for else.
+func (tok Token) Len() int {
+	if tok > ' ' && tok <= Token(len(tokens)) {
+		return len(tokens[tok])
+	}
+	return 0
+}
+
 // -----------------------------------------------------------------------------
 
 const (
-	ILLEGAL Tok = iota
+	ILLEGAL Token = iota
 	EOF
 	COMMENT
 
@@ -80,7 +90,7 @@ const (
 )
 
 const (
-	operator_beg Tok = 0x80 + iota
+	operator_beg Token = 0x80 + iota
 
 	SHL     // <<
 	SHR     // >>
@@ -191,21 +201,11 @@ var tokens = [...]string{
 	ELLIPSIS: "...",
 }
 
-// TokLen returns:
-// 1) len of is token literal, if token is an operator.
-// 2) 0 for else.
-func TokLen(tok Tok) int {
-	if tok > ' ' && tok <= Tok(len(tokens)) {
-		return len(tokens[tok])
-	}
-	return 0
-}
-
 // ForEach iterates all tokens.
-func ForEach(f func(tok Tok, lit string)) {
+func ForEach(f func(tok Token, lit string)) {
 	for i, s := range tokens {
 		if s != "" {
-			f(Tok(i), s)
+			f(Token(i), s)
 		}
 	}
 }


Test patch:
diff --git a/tpl/scanner/scanner_test.go b/tpl/scanner/scanner_test.go
index 2415d961..1dc9679a 100644
--- a/tpl/scanner/scanner_test.go
+++ b/tpl/scanner/scanner_test.go
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package tpl
+package scanner
 
 import (
 	"testing"
@@ -24,7 +24,7 @@ import (
 
 type tokenTest struct {
 	Pos     token.Pos
-	Kind    token.Tok
+	Kind    token.Token
 	Literal string
 }
 
@@ -159,10 +159,10 @@ world !=
 		if c != expect {
 			t.Fatal("Scan failed:", c.Pos, c.Lit, expect)
 		}
-		switch token.TokLen(c.Tok) {
+		switch c.Tok.Len() {
 		case 0, 1, 2, 3:
 		default:
-			t.Fatal("TokenLen failed:", token.TokLen(c.Tok))
+			t.Fatal("TokenLen failed:", c.Tok.Len())
 		}
 		i++
 	}
diff --git a/tpl/token/token_test.go b/tpl/token/token_test.go
index 98240f01..d3c3af86 100644
--- a/tpl/token/token_test.go
+++ b/tpl/token/token_test.go
@@ -20,72 +20,24 @@ import (
 	"testing"
 )
 
-func TestTok(t *testing.T) {
+func TestToken(t *testing.T) {
 	if IDENT.String() != "IDENT" {
 		t.Fatal("IDENT")
 	}
-	if Tok(' ').String() != "token(32)" {
+	if Token(' ').String() != "token(32)" {
 		t.Fatal("token(32)")
 	}
-	if TokLen(IDENT) != 0 {
+	if IDENT.Len() != 0 {
 		t.Fatal("TokLen IDENT")
 	}
-	if TokLen('+') != 1 {
+	if Token('+').Len() != 1 {
 		t.Fatal("TokLen +")
 	}
-}
-
-/*
-func TestTokens(tt *testing.T) {
-	for i, t := range tokens {
-		if t == "" {
-			continue
-		}
-		if i > 20 {
-			n := len(t)
-			if n < 3 {
-				tt.Fatal("invalid token:", i, t)
-			}
-			if i < 0x80 {
-				if t[0] != '\'' || t[n-1] != '\'' {
-					tt.Fatal("invalid token:", i, t)
-				}
-				if n != 3 || t[1] != byte(i) {
-					tt.Fatal("invalid token:", i, t)
-				}
-			} else {
-				if t[0] != '"' || t[n-1] != '"' {
-					tt.Fatal("invalid token:", i, t)
-				}
-			}
-		}
-	}
-}
-
-func TestToken(t *testing.T) {
-	if operator_beg != 0x80 {
-		t.Fatal("operator_beg != 0x80", int(operator_beg))
-	}
-
-	s := new(Scanner)
-	if s.Ttol(LPAREN) != "'('" {
-		t.Fatal("LPAREN")
-	}
-	if s.Ttol(IDENT) != "IDENT" {
-		t.Fatal("IDENT")
-	}
-	if s.Ttol(ADD_ASSIGN) != "\"+=\"" {
-		t.Fatal("ADD_ASSIGN")
-	}
-
-	if s.Ltot("\"++\"") != INC {
-		t.Fatal("++")
-	}
-	if s.Ltot("')'") != RPAREN {
-		t.Fatal(")")
-	}
-	if s.Ltot("COMMENT") != COMMENT {
-		t.Fatal("COMMENT")
+	count := 0
+	ForEach(func(tok Token, name string) {
+		count++
+	})
+	if count != 59 {
+		t.Fatal("ForEach:", count)
 	}
 }
-*/