Repository: goplus/gop
Pull request #2225 (2025-03-30 13:31:55)

tpl/scanner: CSTRING, PYSTRING, POW


Patch:
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 962e7150..8a439329 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -937,7 +937,7 @@ scanAgain:
 			if s.ch == '>' { // ->
 				s.next()
 				tok = token.SRARROW
-			} else { // --, -=
+			} else { // -- -=
 				tok = s.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
 				if tok == token.DEC {
 					insertSemi = true
diff --git a/token/token.go b/token/token.go
index c4d7f891..631dffb0 100644
--- a/token/token.go
+++ b/token/token.go
@@ -148,7 +148,7 @@ const (
 	additional_literal_end = 97
 
 	ENV  = additional_end2 // ${name}
-	UNIT = additional_end3 // 1m, 2s, 3ms, 4us, 5ns, 6m, 7h, 8d, 9w, 10y
+	UNIT = additional_end3 // 1m, 2.3s, 3ms, 4us, 5ns, 6.5m, 7h, 8d, 9w, 10y
 
 	PYSTRING = additional_literal_beg // py"Hello"
 
diff --git a/tpl/scanner/scanner.go b/tpl/scanner/scanner.go
index ca290c1d..9566256d 100644
--- a/tpl/scanner/scanner.go
+++ b/tpl/scanner/scanner.go
@@ -649,8 +649,20 @@ scanAgain:
 	switch ch := s.ch; {
 	case isLetter(ch):
 		t.Lit = s.scanIdentifier()
-		t.Tok = token.IDENT
-		insertSemi = true
+		if (t.Lit == "c" || t.Lit == "C") && s.ch == '"' { // c"..."
+			s.next()
+			insertSemi = true
+			t.Tok = token.CSTRING
+			t.Lit = s.scanString()
+		} else if t.Lit == "py" && s.ch == '"' { // py"..."
+			s.next()
+			insertSemi = true
+			t.Tok = token.PYSTRING
+			t.Lit = s.scanString()
+		} else {
+			insertSemi = true
+			t.Tok = token.IDENT
+		}
 	case '0' <= ch && ch <= '9':
 		insertSemi = true
 		t.Tok, t.Lit = s.scanNumber(false)
@@ -733,8 +745,8 @@ scanAgain:
 					insertSemi = true
 				}
 			}
-		case '*':
-			t.Tok = s.switch2(token.MUL, token.MUL_ASSIGN)
+		case '*': // ** *=
+			t.Tok = s.switch3(token.MUL, token.MUL_ASSIGN, '*', token.POW)
 		case '/':
 			if s.ch == '/' || s.ch == '*' {
 				// comment
diff --git a/tpl/token/token.go b/tpl/token/token.go
index dcb61a8e..18dfdadf 100644
--- a/tpl/token/token.go
+++ b/tpl/token/token.go
@@ -55,14 +55,16 @@ const (
 	literal_beg
 	// Identifiers and basic type literals
 	// (these tokens stand for classes of literals)
-	IDENT  // main
-	INT    // 12345
-	FLOAT  // 123.45
-	IMAG   // 123.45i
-	CHAR   // 'a'
-	STRING // "abc"
-	RAT    // 3r, 3.4r
-	UNIT   // 1m, 2.3s, 3ms, 4us, 5ns, 6.5m, 7h, 8d, 9w, 10y
+	IDENT    // main
+	INT      // 12345
+	FLOAT    // 123.45
+	IMAG     // 123.45i
+	CHAR     // 'a'
+	STRING   // "abc"
+	CSTRING  // c"Hello"
+	PYSTRING // py"Hello"
+	RAT      // 3r, 3.4r
+	UNIT     // 1m, 2.3s, 3ms, 4us, 5ns, 6.5m, 7h, 8d, 9w, 10y
 	literal_end
 
 	ADD = '+'
@@ -133,6 +135,7 @@ const (
 	DRARROW   // =>
 	SRARROW   // ->
 	BIDIARROW // <>
+	POW       // **
 
 	operator_end
 )
@@ -145,14 +148,16 @@ var tokens = [...]string{
 	EOF:     "EOF",
 	COMMENT: "COMMENT",
 
-	IDENT:  "IDENT",
-	INT:    "INT",
-	FLOAT:  "FLOAT",
-	IMAG:   "IMAG",
-	CHAR:   "CHAR",
-	STRING: "STRING",
-	RAT:    "RAT",
-	UNIT:   "UNIT",
+	IDENT:    "IDENT",
+	INT:      "INT",
+	FLOAT:    "FLOAT",
+	IMAG:     "IMAG",
+	CHAR:     "CHAR",
+	STRING:   "STRING",
+	CSTRING:  "CSTRING",
+	PYSTRING: "PYSTRING",
+	RAT:      "RAT",
+	UNIT:     "UNIT",
 
 	ADD: "+",
 	SUB: "-",
@@ -218,6 +223,7 @@ var tokens = [...]string{
 	DRARROW:   "=>",
 	SRARROW:   "->",
 	BIDIARROW: "<>",
+	POW:       "**",
 }
 
 const (


Test patch:
diff --git a/tpl/scanner/_testdata/cstr/tpl.expect b/tpl/scanner/_testdata/cstr/tpl.expect
index aee852ee..16ea4142 100644
--- a/tpl/scanner/_testdata/cstr/tpl.expect
+++ b/tpl/scanner/_testdata/cstr/tpl.expect
@@ -1,14 +1,12 @@
 1 IDENT c
 2 . 
 3 IDENT printf
-10 IDENT c
-11 STRING "Hello"
+10 CSTRING "Hello"
 18 ; 
 
 19 IDENT std
 22 . 
 23 IDENT print
-29 IDENT py
-31 STRING "Hello"
+29 PYSTRING "Hello"
 38 ; 
 
diff --git a/tpl/scanner/_testdata/pow/go.expect b/tpl/scanner/_testdata/pow/go.expect
new file mode 100644
index 00000000..ceb36fbd
--- /dev/null
+++ b/tpl/scanner/_testdata/pow/go.expect
@@ -0,0 +1,14 @@
+1 IDENT echo
+6 INT 2
+7 * 
+8 * 
+9 INT 3
+10 ; 
+
+11 IDENT echo
+16 STRING `abc`
+21 [ 
+22 INT 0
+23 ] 
+24 ; 
+
diff --git a/tpl/scanner/_testdata/pow/gop.expect b/tpl/scanner/_testdata/pow/gop.expect
new file mode 100644
index 00000000..ceb36fbd
--- /dev/null
+++ b/tpl/scanner/_testdata/pow/gop.expect
@@ -0,0 +1,14 @@
+1 IDENT echo
+6 INT 2
+7 * 
+8 * 
+9 INT 3
+10 ; 
+
+11 IDENT echo
+16 STRING `abc`
+21 [ 
+22 INT 0
+23 ] 
+24 ; 
+
diff --git a/tpl/scanner/_testdata/pow/in.gop b/tpl/scanner/_testdata/pow/in.gop
new file mode 100644
index 00000000..a80a7327
--- /dev/null
+++ b/tpl/scanner/_testdata/pow/in.gop
@@ -0,0 +1,2 @@
+echo 2**3
+echo `abc`[0]
diff --git a/tpl/scanner/_testdata/pow/tpl.expect b/tpl/scanner/_testdata/pow/tpl.expect
new file mode 100644
index 00000000..467e196e
--- /dev/null
+++ b/tpl/scanner/_testdata/pow/tpl.expect
@@ -0,0 +1,13 @@
+1 IDENT echo
+6 INT 2
+7 ** 
+9 INT 3
+10 ; 
+
+11 IDENT echo
+16 STRING `abc`
+21 [ 
+22 INT 0
+23 ] 
+24 ; 
+
diff --git a/tpl/token/token_test.go b/tpl/token/token_test.go
index 5ed7a5f9..1f214bea 100644
--- a/tpl/token/token_test.go
+++ b/tpl/token/token_test.go
@@ -27,7 +27,7 @@ func TestToken(t *testing.T) {
 	}
 	for i := Token(0); i < literal_end; i++ {
 		s := i.String()
-		if s == "RAT" || s == "UNIT" {
+		if s == "RAT" || s == "UNIT" || s == "CSTRING" || s == "PYSTRING" {
 			continue
 		}
 		if s != token.Token(i).String() {
@@ -51,7 +51,7 @@ func TestToken(t *testing.T) {
 		count++
 		return 0
 	})
-	if count != 28 {
+	if count != 29 {
 		t.Fatal("ForEach:", count)
 	}
 	NewFileSet()