Repository: cloudspannerecosystem/memefish
Pull request #121 (2024-09-29 09:32:59)

Support proto/enum as typename
This PR adds `NamedType`.

it is required for `PROTO`/`ENUM` support:

* [CAST AS ENUM](https://cloud.google.com/spanner/docs/reference/standard-sql/conversion_functions#cast_as_enum)
* [CAST AS PROTO](https://cloud.google.com/spanner/docs/reference/standard-sql/conversion_functions#cast_as_proto)
* [Use a protocol buffer message in a column definition](https://cloud.google.com/spanner/docs/reference/standard-sql/protocol-buffers#use_a_protocol_buffer_message_in_a_column_definition)

makenowjust: I'm concerned that it seems to be abusing `Path` for the field of `NamedType`. `Path` is a type for path expressions.

Patch:
diff --git a/ast/ast.go b/ast/ast.go
index 4614f24..56e8bf1 100644
--- a/ast/ast.go
+++ b/ast/ast.go
@@ -185,6 +185,7 @@ type Type interface {
 func (SimpleType) isType() {}
 func (ArrayType) isType()  {}
 func (StructType) isType() {}
+func (NamedType) isType()  {}
 
 // IntValue represents integer values in SQL.
 type IntValue interface {
@@ -302,6 +303,7 @@ type SchemaType interface {
 func (ScalarSchemaType) isSchemaType() {}
 func (SizedSchemaType) isSchemaType()  {}
 func (ArraySchemaType) isSchemaType()  {}
+func (NamedType) isSchemaType()        {}
 
 // IndexAlteration represents ALTER INDEX action.
 type IndexAlteration interface {
@@ -1392,6 +1394,18 @@ type StructField struct {
 	Type  Type
 }
 
+// NamedType is named type node.
+// It is currently PROTO or ENUM.
+// Name is full qualified name, but it can be len(Name) == 1 if it doesn't contain ".".
+//
+//	{{.Path | sqlJoin "."}}
+type NamedType struct {
+	// pos = Name.pos
+	// end = Name.end
+
+	Path []*Ident // len(Path) > 0
+}
+
 // ================================================================================
 //
 // Cast for Special Cases
diff --git a/ast/pos.go b/ast/pos.go
index 4e3ebb3..98431a5 100644
--- a/ast/pos.go
+++ b/ast/pos.go
@@ -466,6 +466,9 @@ func (f *StructField) End() token.Pos {
 	return f.Type.End()
 }
 
+func (n *NamedType) Pos() token.Pos { return n.Path[0].Pos() }
+func (n *NamedType) End() token.Pos { return n.Path[len(n.Path)-1].End() }
+
 // ================================================================================
 //
 // Cast for Special Cases
diff --git a/ast/sql.go b/ast/sql.go
index ad5e719..71889a1 100644
--- a/ast/sql.go
+++ b/ast/sql.go
@@ -720,6 +720,17 @@ func (f *StructField) SQL() string {
 	return sql
 }
 
+func (n *NamedType) SQL() string {
+	var sql string
+	for i, elem := range n.Path {
+		if i > 0 {
+			sql += "."
+		}
+		sql += elem.SQL()
+	}
+	return sql
+}
+
 // ================================================================================
 //
 // Cast for Special Cases
diff --git a/parser.go b/parser.go
index b13a461..2252041 100644
--- a/parser.go
+++ b/parser.go
@@ -1888,9 +1888,17 @@ func (p *Parser) lookaheadSubQuery() bool {
 //
 // ================================================================================
 
+func (p *Parser) parseNamedType() *ast.NamedType {
+	path := p.parseIdentOrPath()
+	return &ast.NamedType{Path: path}
+}
+
 func (p *Parser) parseType() ast.Type {
 	switch p.Token.Kind {
 	case token.TokenIdent:
+		if !p.lookaheadSimpleType() {
+			return p.parseNamedType()
+		}
 		return p.parseSimpleType()
 	case "ARRAY":
 		return p.parseArrayType()
@@ -1911,6 +1919,7 @@ var simpleTypes = []string{
 	"NUMERIC",
 	"STRING",
 	"BYTES",
+	"JSON",
 }
 
 func (p *Parser) parseSimpleType() *ast.SimpleType {
@@ -2018,6 +2027,21 @@ func (p *Parser) lookaheadType() bool {
 	return p.Token.Kind == token.TokenIdent || p.Token.Kind == "ARRAY" || p.Token.Kind == "STRUCT"
 }
 
+func (p *Parser) lookaheadSimpleType() bool {
+	if p.Token.Kind != token.TokenIdent {
+		return false
+	}
+
+	id := p.Token
+
+	for _, name := range simpleTypes {
+		if id.IsIdent(name) {
+			return true
+		}
+	}
+	return false
+}
+
 // ================================================================================
 //
 // DDL
@@ -3279,6 +3303,9 @@ func (p *Parser) tryParseTablePrivilegeColumns() ([]*ast.Ident, token.Pos) {
 func (p *Parser) parseSchemaType() ast.SchemaType {
 	switch p.Token.Kind {
 	case token.TokenIdent:
+		if !p.lookaheadSimpleType() {
+			return p.parseNamedType()
+		}
 		return p.parseScalarSchemaType()
 	case "ARRAY":
 		pos := p.expect("ARRAY").Pos
diff --git a/split.go b/split.go
index 9f0193c..a22a4fd 100644
--- a/split.go
+++ b/split.go
@@ -2,14 +2,20 @@ package memefish
 
 import "github.com/cloudspannerecosystem/memefish/token"
 
+type RawStatement struct {
+	Pos, End  token.Pos
+	Statement string
+}
+
 // SplitRawStatements splits an input string to statement strings at terminating semicolons without parsing.
+// It preserves all comments.
 // Statements are terminated by `;`, `<eof>` or `;<eof>` and the minimum output will be []string{""}.
 // See [terminating semicolons].
 // This function won't panic but return error if lexer become error state.
-// filepath can be used in error message.
+// filepath can be empty, it is only used in error message.
 //
 // [terminating semicolons]: https://cloud.google.com/spanner/docs/reference/standard-sql/lexical#terminating_semicolons
-func SplitRawStatements(filepath, s string) ([]string, error) {
+func SplitRawStatements(filepath, s string) ([]*RawStatement, error) {
 	lex := &Lexer{
 		File: &token.File{
 			FilePath: filepath,
@@ -17,11 +23,15 @@ func SplitRawStatements(filepath, s string) ([]string, error) {
 		},
 	}
 
-	var result []string
+	var result []*RawStatement
 	var firstPos token.Pos
 	for {
 		if lex.Token.Kind == ";" {
-			result = append(result, s[firstPos:lex.Token.Pos])
+			result = append(result, &RawStatement{
+				Pos:       firstPos,
+				End:       lex.Token.Pos,
+				Statement: s[firstPos:lex.Token.Pos],
+			})
 			if err := lex.NextToken(); err != nil {
 				return nil, err
 			}
@@ -36,13 +46,19 @@ func SplitRawStatements(filepath, s string) ([]string, error) {
 
 		if lex.Token.Kind == token.TokenEOF {
 			if lex.Token.Pos != firstPos {
-				result = append(result, s[firstPos:lex.Token.Pos])
+				result = append(result, &RawStatement{
+					Pos:       firstPos,
+					End:       lex.Token.Pos,
+					Statement: s[firstPos:lex.Token.Pos],
+				})
 			}
 			break
 		}
 	}
 	if len(result) == 0 {
-		return []string{""}, nil
+		return []*RawStatement{
+			{Statement: ""},
+		}, nil
 	}
 	return result, nil
 }


Test patch:
diff --git a/split_test.go b/split_test.go
index d9b8a5b..14d838f 100644
--- a/split_test.go
+++ b/split_test.go
@@ -2,6 +2,7 @@ package memefish_test
 
 import (
 	"github.com/cloudspannerecosystem/memefish"
+	"github.com/cloudspannerecosystem/memefish/token"
 	"github.com/google/go-cmp/cmp"
 	"regexp"
 	"testing"
@@ -12,23 +13,62 @@ func TestSplitRawStatements(t *testing.T) {
 		desc  string
 		input string
 		errRe *regexp.Regexp
-		want  []string
+		want  []*memefish.RawStatement
 	}{
 		// SplitRawStatements treats only lexical structures, so the test cases can be invalid statements.
-		{desc: "empty input", input: "", want: []string{""}},
-		{desc: "single statement ends with semicolon", input: `SELECT "123";`, want: []string{`SELECT "123"`}},
-		{desc: "single statement ends with EOF", input: `SELECT "123"`, want: []string{`SELECT "123"`}},
-		{desc: "two statement ends with semicolon", input: `SELECT "123"; SELECT "456";`, want: []string{`SELECT "123"`, `SELECT "456"`}},
-		{desc: "two statement ends with EOF", input: `SELECT "123"; SELECT "456"`, want: []string{`SELECT "123"`, `SELECT "456"`}},
-		{desc: "second statement is empty", input: `SELECT 1; ;`, want: []string{`SELECT 1`, ``}},
-		{desc: "two statement with new lines", input: "SELECT 1;\n SELECT 2;\n", want: []string{"SELECT 1", "SELECT 2"}},
+		{desc: "empty input", input: "", want: []*memefish.RawStatement{{Statement: ""}}},
+		{desc: "single statement ends with semicolon", input: `SELECT "123";`,
+			want: []*memefish.RawStatement{
+				{Statement: `SELECT "123"`, End: token.Pos(12)},
+			}},
+		{desc: "single statement ends with EOF", input: `SELECT "123"`,
+			want: []*memefish.RawStatement{
+				{Statement: `SELECT "123"`, End: token.Pos(12)},
+			}},
+		{desc: "two statement ends with semicolon", input: `SELECT "123"; SELECT "456";`,
+			want: []*memefish.RawStatement{
+				{Statement: `SELECT "123"`, End: token.Pos(12)},
+				{Statement: `SELECT "456"`, Pos: token.Pos(14), End: token.Pos(26)},
+			}},
+		{desc: "two statement ends with EOF", input: `SELECT "123"; SELECT "456"`,
+			want: []*memefish.RawStatement{
+				{Statement: `SELECT "123"`, End: token.Pos(12)},
+				{Statement: `SELECT "456"`, Pos: token.Pos(14), End: token.Pos(26)},
+			}},
+		{desc: "second statement is empty", input: `SELECT 1; ;`,
+			want: []*memefish.RawStatement{
+				{Statement: `SELECT 1`, End: token.Pos(8)},
+				{Statement: ``, Pos: token.Pos(10), End: token.Pos(10)},
+			}},
+		{desc: "two statement with new lines", input: "SELECT 1;\n SELECT 2;\n",
+			want: []*memefish.RawStatement{
+				{Statement: "SELECT 1", End: token.Pos(8)},
+				{Statement: "SELECT 2", Pos: token.Pos(11), End: token.Pos(19)},
+			}},
 		{desc: "single statement with line comment", input: `SELECT 1//
-`, want: []string{"SELECT 1//\n"}},
-		{desc: "semicolon in line comment", input: "SELECT 1 //;\n + 2", want: []string{"SELECT 1 //;\n + 2"}},
-		{desc: "semicolon in multi-line comment", input: "SELECT 1 /*;\n*/ + 2", want: []string{"SELECT 1 /*;\n*/ + 2"}},
-		{desc: "semicolon in double-quoted string", input: `SELECT "1;2;3";`, want: []string{`SELECT "1;2;3"`}},
-		{desc: "semicolon in single-quoted string", input: `SELECT '1;2;3';`, want: []string{`SELECT '1;2;3'`}},
-		{desc: "semicolon in back-quote", input: "SELECT `1;2;3`;", want: []string{"SELECT `1;2;3`"}},
+`, want: []*memefish.RawStatement{
+			{Statement: "SELECT 1//\n", End: token.Pos(11)},
+		}},
+		{desc: "semicolon in line comment", input: "SELECT 1 //;\n + 2",
+			want: []*memefish.RawStatement{
+				{Statement: "SELECT 1 //;\n + 2", End: token.Pos(17)},
+			}},
+		{desc: "semicolon in multi-line comment", input: "SELECT 1 /*;\n*/ + 2",
+			want: []*memefish.RawStatement{
+				{Statement: "SELECT 1 /*;\n*/ + 2", End: token.Pos(19)},
+			}},
+		{desc: "semicolon in double-quoted string", input: `SELECT "1;2;3";`,
+			want: []*memefish.RawStatement{
+				{Statement: `SELECT "1;2;3"`, End: token.Pos(14)},
+			}},
+		{desc: "semicolon in single-quoted string", input: `SELECT '1;2;3';`,
+			want: []*memefish.RawStatement{
+				{Statement: `SELECT '1;2;3'`, End: token.Pos(14)},
+			}},
+		{desc: "semicolon in back-quote", input: "SELECT `1;2;3`;",
+			want: []*memefish.RawStatement{
+				{Statement: "SELECT `1;2;3`", End: token.Pos(14)},
+			}},
 		// $` may become a valid token in the future, but it's reasonable to check its current behavior.
 		{desc: "unknown token", input: "SELECT $;", errRe: regexp.MustCompile(`illegal input character: '\$'`)},
 	} {
diff --git a/testdata/input/ddl/create_table_types.sql b/testdata/input/ddl/create_table_types.sql
index a2d6879..d1d9c6f 100644
--- a/testdata/input/ddl/create_table_types.sql
+++ b/testdata/input/ddl/create_table_types.sql
@@ -10,5 +10,6 @@ create table types (
   bs bytes(256),
   bsmax bytes(max),
   ab array<bool>,
-  abs array<bytes(max)>
+  abs array<bytes(max)>,
+  p examples.ProtoType,
 ) primary key (i)
diff --git a/testdata/input/expr/cast_as_typename.sql b/testdata/input/expr/cast_as_typename.sql
new file mode 100644
index 0000000..ae539b2
--- /dev/null
+++ b/testdata/input/expr/cast_as_typename.sql
@@ -0,0 +1,1 @@
+CAST('order_number: "123"' AS examples.shipping.`Order`)
\ No newline at end of file
diff --git a/testdata/result/ddl/create_table_types.sql.txt b/testdata/result/ddl/create_table_types.sql.txt
index efee150..f7f536d 100644
--- a/testdata/result/ddl/create_table_types.sql.txt
+++ b/testdata/result/ddl/create_table_types.sql.txt
@@ -11,13 +11,14 @@ create table types (
   bs bytes(256),
   bsmax bytes(max),
   ab array<bool>,
-  abs array<bytes(max)>
+  abs array<bytes(max)>,
+  p examples.ProtoType,
 ) primary key (i)
 
 --- AST
 &ast.CreateTable{
   Create:      0,
-  Rparen:      227,
+  Rparen:      252,
   IfNotExists: false,
   Name:        &ast.Ident{
     NamePos: 13,
@@ -250,14 +251,40 @@ create table types (
       GeneratedExpr: (*ast.GeneratedColumnExpr)(nil),
       Options:       (*ast.ColumnDefOptions)(nil),
     },
+    &ast.ColumnDef{
+      Null: -1,
+      Name: &ast.Ident{
+        NamePos: 214,
+        NameEnd: 215,
+        Name:    "p",
+      },
+      Type: &ast.NamedType{
+        Path: []*ast.Ident{
+          &ast.Ident{
+            NamePos: 216,
+            NameEnd: 224,
+            Name:    "examples",
+          },
+          &ast.Ident{
+            NamePos: 225,
+            NameEnd: 234,
+            Name:    "ProtoType",
+          },
+        },
+      },
+      NotNull:       false,
+      DefaultExpr:   (*ast.ColumnDefaultExpr)(nil),
+      GeneratedExpr: (*ast.GeneratedColumnExpr)(nil),
+      Options:       (*ast.ColumnDefOptions)(nil),
+    },
   },
   TableConstraints: []*ast.TableConstraint(nil),
   PrimaryKeys:      []*ast.IndexKey{
     &ast.IndexKey{
       DirPos: -1,
       Name:   &ast.Ident{
-        NamePos: 226,
-        NameEnd: 227,
+        NamePos: 251,
+        NameEnd: 252,
         Name:    "i",
       },
       Dir: "",
@@ -268,4 +295,4 @@ create table types (
 }
 
 --- SQL
-CREATE TABLE types (b BOOL, i INT64, f32 FLOAT32, f FLOAT64, d DATE, t TIMESTAMP, s STRING(256), smax STRING(MAX), bs BYTES(256), bsmax BYTES(MAX), ab ARRAY<BOOL>, abs ARRAY<BYTES(MAX)>) PRIMARY KEY (i)
+CREATE TABLE types (b BOOL, i INT64, f32 FLOAT32, f FLOAT64, d DATE, t TIMESTAMP, s STRING(256), smax STRING(MAX), bs BYTES(256), bsmax BYTES(MAX), ab ARRAY<BOOL>, abs ARRAY<BYTES(MAX)>, p examples.ProtoType) PRIMARY KEY (i)
diff --git a/testdata/result/expr/cast_as_typename.sql.txt b/testdata/result/expr/cast_as_typename.sql.txt
new file mode 100644
index 0000000..ec1e5fe
--- /dev/null
+++ b/testdata/result/expr/cast_as_typename.sql.txt
@@ -0,0 +1,34 @@
+--- cast_as_typename.sql
+CAST('order_number: "123"' AS examples.shipping.`Order`)
+--- AST
+&ast.CastExpr{
+  Cast:   0,
+  Rparen: 55,
+  Expr:   &ast.StringLiteral{
+    ValuePos: 5,
+    ValueEnd: 26,
+    Value:    "order_number: \"123\"",
+  },
+  Type: &ast.NamedType{
+    Path: []*ast.Ident{
+      &ast.Ident{
+        NamePos: 30,
+        NameEnd: 38,
+        Name:    "examples",
+      },
+      &ast.Ident{
+        NamePos: 39,
+        NameEnd: 47,
+        Name:    "shipping",
+      },
+      &ast.Ident{
+        NamePos: 48,
+        NameEnd: 55,
+        Name:    "Order",
+      },
+    },
+  },
+}
+
+--- SQL
+CAST("order_number: \"123\"" AS examples.shipping.`Order`)
diff --git a/testdata/result/statement/create_table_types.sql.txt b/testdata/result/statement/create_table_types.sql.txt
index efee150..f7f536d 100644
--- a/testdata/result/statement/create_table_types.sql.txt
+++ b/testdata/result/statement/create_table_types.sql.txt
@@ -11,13 +11,14 @@ create table types (
   bs bytes(256),
   bsmax bytes(max),
   ab array<bool>,
-  abs array<bytes(max)>
+  abs array<bytes(max)>,
+  p examples.ProtoType,
 ) primary key (i)
 
 --- AST
 &ast.CreateTable{
   Create:      0,
-  Rparen:      227,
+  Rparen:      252,
   IfNotExists: false,
   Name:        &ast.Ident{
     NamePos: 13,
@@ -250,14 +251,40 @@ create table types (
       GeneratedExpr: (*ast.GeneratedColumnExpr)(nil),
       Options:       (*ast.ColumnDefOptions)(nil),
     },
+    &ast.ColumnDef{
+      Null: -1,
+      Name: &ast.Ident{
+        NamePos: 214,
+        NameEnd: 215,
+        Name:    "p",
+      },
+      Type: &ast.NamedType{
+        Path: []*ast.Ident{
+          &ast.Ident{
+            NamePos: 216,
+            NameEnd: 224,
+            Name:    "examples",
+          },
+          &ast.Ident{
+            NamePos: 225,
+            NameEnd: 234,
+            Name:    "ProtoType",
+          },
+        },
+      },
+      NotNull:       false,
+      DefaultExpr:   (*ast.ColumnDefaultExpr)(nil),
+      GeneratedExpr: (*ast.GeneratedColumnExpr)(nil),
+      Options:       (*ast.ColumnDefOptions)(nil),
+    },
   },
   TableConstraints: []*ast.TableConstraint(nil),
   PrimaryKeys:      []*ast.IndexKey{
     &ast.IndexKey{
       DirPos: -1,
       Name:   &ast.Ident{
-        NamePos: 226,
-        NameEnd: 227,
+        NamePos: 251,
+        NameEnd: 252,
         Name:    "i",
       },
       Dir: "",
@@ -268,4 +295,4 @@ create table types (
 }
 
 --- SQL
-CREATE TABLE types (b BOOL, i INT64, f32 FLOAT32, f FLOAT64, d DATE, t TIMESTAMP, s STRING(256), smax STRING(MAX), bs BYTES(256), bsmax BYTES(MAX), ab ARRAY<BOOL>, abs ARRAY<BYTES(MAX)>) PRIMARY KEY (i)
+CREATE TABLE types (b BOOL, i INT64, f32 FLOAT32, f FLOAT64, d DATE, t TIMESTAMP, s STRING(256), smax STRING(MAX), bs BYTES(256), bsmax BYTES(MAX), ab ARRAY<BOOL>, abs ARRAY<BYTES(MAX)>, p examples.ProtoType) PRIMARY KEY (i)