Repository: cloudspannerecosystem/memefish (go)

Issue:
Sync lexer punctuations with ZetaSQL
Some punctuations are not yet handled in `memefish.Lexer`.
It cause panic when lexing even if parsing is not needed.

I think it is safe to add all punctuations in ZetaSQL.

https://github.com/google/zetasql/blob/194cd32b5d766d60e3ca442651d792c7fe54ea74/zetasql/parser/flex_tokenizer.l#L513-L552

- [x] `(`
- [x] `[`
- [x] `{`
- [x] `)`
- [x] `]`
- [x] `}`
- [x] `*`
- [x] `,`
- [x] `=`
- [ ] `+=` `KW_ADD_ASSIGN`
  - It seems to be used in [`options_assignment_operator`](https://sourcegraph.com/github.com/google/zetasql@194cd32b5d766d60e3ca442651d792c7fe54ea74/-/blob/zetasql/parser/bison_parser.y?L10257-10271)
- [ ] `-=` `KW_SUB_ASSIGN`
  -  It seems to be used in [`options_assignment_operator`](https://sourcegraph.com/github.com/google/zetasql@194cd32b5d766d60e3ca442651d792c7fe54ea74/-/blob/zetasql/parser/bison_parser.y?L10257-10271)
- [x] `!=`
- [x] `<=`
- [x] `<<`
- [x] `=>`
- [ ] `->` `KW_LAMBDA_ARROW`
  -  #153 
- [x] `<`
- [x] `>`
- [x] `>=`
- [x] `||`
- [x] `|`
- [x] `^`
- [x] `&`
- [x] `+`
- [x] `-`
- [x] `/`
- [x] `~`
- [ ] `?`
  - [Positional parameter](https://cloud.google.com/bigquery/docs/parameterized-queries) is available in BigQuery, but Cloud Spanner has not yet supported.
    > Positional parameters are not supported
- [ ] `!`
  - It seems not to be used.
- [ ] `%`
  - GQL #99 
- [ ] `|>` `KW_PIPE`
  - Pipe syntax
- [x] `@`
- [ ] `@@` `KW_DOUBLE_AT`
  - [System variables in BigQuery](https://cloud.google.com/bigquery/docs/reference/system-variables)
- [x] `.`
- [ ] `:`
  - `NEW` constructors #119 
  - GQL #99 
- [ ] `\` `BACKSLASH`
  - [Only for lenient macro expansion](https://sourcegraph.com/github.com/google/zetasql@194cd32b5d766d60e3ca442651d792c7fe54ea74/-/blob/zetasql/parser/bison_parser.y?L245)
- [x] `;`
- [ ] `${unquoted_identifier}` [`MACRO_INVOCATION`](https://sourcegraph.com/search?q=repo:%5Egithub%5C.com/google/zetasql%24%40194cd32b5d766d60e3ca442651d792c7fe54ea74+MACRO_INVOCATION&patternType=keyword&sm=0)
  - [internal names in ZetaSQL](https://sourcegraph.com/search?q=repo:%5Egithub%5C.com/google/zetasql%24+%27%24%27&patternType=keyword&sm=0)?
- [ ] `${decimal_digits}` [`MACRO_ARGUMENT_REFERENCE`](https://sourcegraph.com/search?q=repo:%5Egithub%5C.com/google/zetasql%24%40194cd32b5d766d60e3ca442651d792c7fe54ea74+MACRO_ARGUMENT_REFERENCE&patternType=keyword&sm=0)
- [ ] `$`
  - It seems not used.

Except `MACRO_INVOCATION` and `MACRO_ARGUMENT_REFERENCE` would be better to be implemented.

This makes `Lexer` more robust even to new lexical structures that are being tested privately.


Pull request:
Sync lexer punctuations with ZetaSQL
This PR adds all ZetaSQL punctuations as memefish tokens.

`MACRO_*` are not yet implemented because they are not yet used and can be tokenized as `"$" Ident`, `"$" IntLiteral`, but they are not trivial to implement.

fixes #181

Patch:
diff --git a/lexer.go b/lexer.go
index 26d679b..632e07f 100644
--- a/lexer.go
+++ b/lexer.go
@@ -109,7 +109,9 @@ func (l *Lexer) consumeToken() {
 	}
 
 	switch l.peek(0) {
-	case '(', ')', '{', '}', ';', ',', '[', ']', '~', '*', '/', '&', '^', '+', '-':
+	case '(', ')', '{', '}', ';', ',', '[', ']', '~', '*', '/', '&', '^', '%', ':',
+		// Belows are not yet used in Spanner.
+		'?', '\\', '$':
 		l.Token.Kind = token.TokenKind([]byte{l.skip()})
 		return
 	case '.':
@@ -151,6 +153,32 @@ func (l *Lexer) consumeToken() {
 			l.Token.Kind = ">"
 		}
 		return
+	case '+':
+		switch {
+		// KW_ADD_ASSIGN in ZetaSQL
+		case l.peekIs(1, '='):
+			l.skipN(2)
+			l.Token.Kind = "+="
+		default:
+			l.skip()
+			l.Token.Kind = "+"
+		}
+		return
+	case '-':
+		switch {
+		// KW_SUB_ASSIGN in ZetaSQL
+		case l.peekIs(1, '='):
+			l.skipN(2)
+			l.Token.Kind = "-="
+		// KW_LAMBDA_ARROW in ZetaSQL
+		case l.peekIs(1, '>'):
+			l.skipN(2)
+			l.Token.Kind = "->"
+		default:
+			l.skip()
+			l.Token.Kind = "-"
+		}
+		return
 	case '=':
 		switch {
 		case l.peekIs(1, '>'):
@@ -163,6 +191,9 @@ func (l *Lexer) consumeToken() {
 		return
 	case '|':
 		switch {
+		case l.peekIs(1, '>'):
+			l.skipN(2)
+			l.Token.Kind = "|>"
 		case l.peekIs(1, '|'):
 			l.skipN(2)
 			l.Token.Kind = "||"
@@ -177,7 +208,16 @@ func (l *Lexer) consumeToken() {
 			l.Token.Kind = "!="
 			return
 		}
+		l.skip()
+		l.Token.Kind = "!"
+		return
 	case '@':
+		// KW_DOUBLE_AT is not yet used in Cloud Spanner, but used in BigQuery.
+		if l.peekIs(1, '@') {
+			l.skipN(2)
+			l.Token.Kind = "@@"
+			return
+		}
 		if l.peekOk(1) && char.IsIdentStart(l.peek(1)) {
 			i := 1
 			for l.peekOk(i) && char.IsIdentPart(l.peek(i)) {
diff --git a/tools/parse/main.go b/tools/parse/main.go
index 1cb36bf..17d4a6a 100644
--- a/tools/parse/main.go
+++ b/tools/parse/main.go
@@ -51,7 +51,7 @@ func main() {
 	}
 
 	flag.Parse()
-	if flag.NArg() < 1 {
+	if flag.NArg() != 1 {
 		flag.Usage()
 		os.Exit(1)
 		return


Test patch:
diff --git a/lexer_test.go b/lexer_test.go
index 4919c0d..6ad623e 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -11,35 +11,48 @@ import (
 	. "github.com/cloudspannerecosystem/memefish/token"
 )
 
+// Keep same order https://github.com/google/zetasql/blob/master/zetasql/parser/flex_tokenizer.l
 var symbols = []string{
-	".",
-	",",
-	";",
 	"(",
-	")",
-	"{",
-	"}",
 	"[",
+	"{",
+	")",
 	"]",
-	"@",
-	"~",
-	"+",
-	"-",
+	"}",
 	"*",
-	"/",
-	"&",
-	"^",
-	"|",
-	"||",
+	",",
 	"=",
-	"<",
-	"<<",
+	"+=",
+	"-=",
+	"!=",
 	"<=",
-	"<>",
+	"<<",
+	"=>",
+	"->",
+	"<",
 	">",
-	">>",
 	">=",
-	"!=",
+	"||",
+	"|",
+	"^",
+	"&",
+	"+",
+	"-",
+	"/",
+	"~",
+	"?",
+	"!",
+	"%",
+	"|>",
+	"@",
+	"@@",
+	".",
+	":",
+	"\\",
+	";",
+	"$",
+	"<>", // <> is not a valid token in ZetaSQL, but it is a token in memefish
+	">>", // >> is not a valid token in ZetaSQL, but it is a token in memefish.
 }
 
 var lexerTestCases = []struct {
@@ -132,7 +145,7 @@ var lexerWrongTestCase = []struct {
 	pos     Pos
 	message string
 }{
-	{"?", 0, "illegal input character: '?'"},
+	{"\b", 0, "illegal input character: '\\b'"},
 	{`"foo`, 0, "unclosed string literal"},
 	{`R"foo`, 1, "unclosed raw string literal"},
 	{"'foo\n", 0, "unclosed string literal: newline appears in non triple-quoted"},
diff --git a/split_test.go b/split_test.go
index 14d838f..0e7533b 100644
--- a/split_test.go
+++ b/split_test.go
@@ -69,8 +69,6 @@ func TestSplitRawStatements(t *testing.T) {
 			want: []*memefish.RawStatement{
 				{Statement: "SELECT `1;2;3`", End: token.Pos(14)},
 			}},
-		// $` may become a valid token in the future, but it's reasonable to check its current behavior.
-		{desc: "unknown token", input: "SELECT $;", errRe: regexp.MustCompile(`illegal input character: '\$'`)},
 	} {
 		t.Run(test.desc, func(t *testing.T) {
 			stmts, err := memefish.SplitRawStatements("", test.input)