Repository: cloudspannerecosystem/memefish
Pull request #102 (2024-09-20 09:42:27)

Add SplitRawStatements()
It is known that statement separating without parsing syntax is needed for Cloud Spanner related OSSs.

https://github.com/cloudspannerecosystem/wrench/issues/83
https://github.com/cloudspannerecosystem/yo/pull/116

memefish is good place for statement separator, because the statement separator must know about lexical structure like strings and comments 
and cloudspannerecosystem is reliable org.

Test cases should be added as
* [spanner-cli](https://github.com/cloudspannerecosystem/spanner-cli/blob/abc1cf9fced7a7d74c8344d3197f419ea5c9d95d/separator_test.go)
* [gsqlsep](https://github.com/apstndb/gsqlsep/blob/432be37d515a12c4665ca4aa963a6f4382dc750e/separator_test.go)
 
Note: I think it is no need to support custom separator like spanner-cli's `\G`.


Patch:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 6c020fe..2fc34ce 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -9,11 +9,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
         with:
           submodules: true
       - name: Set up Hugo
-        uses: peaceiris/actions-hugo@v2
+        uses: peaceiris/actions-hugo@v3
         with:
           hugo-version: '0.133.0'
           extended: true
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index c253430..d34a542 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -12,13 +12,13 @@ jobs:
     steps:
 
     - name: Set up Go 1.19
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: 1.19
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Get dependencies
       run: |
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 20efd0d..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,4 +0,0 @@
-[submodule "hugo-whisper-theme"]
-	path = docs/themes/hugo-whisper-theme
-	url = https://github.com/deining/hugo-whisper-theme.git
-	branch = deprecation-warning
diff --git a/Makefile b/Makefile
index 62b994f..2ac24ad 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,7 @@ docs:
 	@echo "  (x x) < memefish: docs"
 	@echo "  /|||\\"
 	@echo
-	cd docs && hugo
+	cd docs && hugo mod get -u && hugo
 
 .PHONY: ci
 ci: lint test
diff --git a/docs/config.toml b/docs/config.toml
deleted file mode 100644
index 2d9b45d..0000000
--- a/docs/config.toml
+++ /dev/null
@@ -1,36 +0,0 @@
-baseurl = "https://cloudspannerecosystem.github.io/memefish"
-title = "méméfish"
-theme = "hugo-whisper-theme"
-languagecode = "en"
-defaultcontentlanguage = "en"
-
-paginate = 20
-canonifyurls = true
-
-pygmentsstyle = "bw"
-pygmentscodefences = true
-pygmentscodefencesguesssyntax = true
-
-summaryLength = 30
-
-[params]
-  homepage_button_link = "/docs"
-  homepage_button_text = "Read The Docs"
-  homepage_intro = "Spanner SQL parser for Go"
-
-  [params.logo]
-    mobile = "/images/memefish.svg"
-    standard  = "/images/memefish.svg"
-
-[[menu.main]]
-    name = "Docs"
-    weight = 1
-    url  = "/docs/"
-[[menu.main]]
-    name = "GoDoc"
-    weight = 3
-    url = "https://pkg.go.dev/github.com/cloudspannerecosystem/memefish"
-[[menu.main]]
-    name = "GitHub"
-    weight = 3
-    url = "https://github.com/cloudspannerecosystem/memefish"
diff --git a/docs/content/docs/_index.md b/docs/content/_index.md
similarity index 92%
rename from docs/content/docs/_index.md
rename to docs/content/_index.md
index bd5681e..e4cadb7 100644
--- a/docs/content/docs/_index.md
+++ b/docs/content/_index.md
@@ -1,9 +1,9 @@
 ---
-title: README
+title: Home
 ---
 
 <p align="center">
-  <img src="images/memefish.png" width="220px">
+  <img src="images/memefish.svg" width="220px">
 </p>
 
 # méméfish
diff --git a/docs/content/docs/example-parse/index.md b/docs/content/example-parse/_index.md
similarity index 100%
rename from docs/content/docs/example-parse/index.md
rename to docs/content/example-parse/_index.md
diff --git a/docs/go.mod b/docs/go.mod
new file mode 100644
index 0000000..89b35ca
--- /dev/null
+++ b/docs/go.mod
@@ -0,0 +1,5 @@
+module github.com/couldspannerecosystem/memefish/docs
+
+go 1.21.0
+
+require github.com/thingsym/hugo-theme-techdoc v1.0.1 // indirect
diff --git a/docs/go.sum b/docs/go.sum
new file mode 100644
index 0000000..cd9e238
--- /dev/null
+++ b/docs/go.sum
@@ -0,0 +1,2 @@
+github.com/thingsym/hugo-theme-techdoc v1.0.1 h1:YesixDLh7bRLFpcoyqflL3KUA+VFFw4l/9g/PuRHa4M=
+github.com/thingsym/hugo-theme-techdoc v1.0.1/go.mod h1:LTThnoCpG8nsum5q10OLrG1LdJNI3BXzV4Qr557cmAY=
diff --git a/docs/hugo.toml b/docs/hugo.toml
new file mode 100644
index 0000000..c881e52
--- /dev/null
+++ b/docs/hugo.toml
@@ -0,0 +1,52 @@
+theme = "github.com/thingsym/hugo-theme-techdoc"
+
+baseURL = "https://cloudspannerecosystem.dev/memefish/"
+title = "méméfish"
+languageCode = "en-US"
+defaultContentLanguage = "en"
+
+canonifyurls = true
+
+pygmentsstyle = "bw"
+pygmentscodefences = true
+pygmentscodefencesguesssyntax = true
+
+summaryLength = 30
+
+[markup]
+  defaultMarkdownHandler = "goldmark"
+  [markup.goldmark.renderer]
+    unsafe= true
+  [markup.tableOfContents]
+    startLevel = 2
+    endLevel = 6
+    ordered = false
+
+[params]
+  description = "Spanner SQL parser for Go"
+  github_repository = "https://github.com/cloudspannerecosystem/memefish"
+  version = ""
+
+  github_doc_repository = "https://github.com/cloudspannerecosystem/memefish"
+  github_doc_repository_branch = "main"
+  github_doc_repository_path = "docs"
+
+  menu_style = "open-menu"
+
+[menu]
+  [[menu.main]]
+    name = "Docs"
+    weight = 1
+    url  = "/"
+  [[menu.main]]
+    name = "GoDoc"
+    weight = 2
+    url = "https://pkg.go.dev/github.com/cloudspannerecosystem/memefish"
+  [[menu.main]]
+    name = "GitHub"
+    weight = 3
+    url = "https://github.com/cloudspannerecosystem/memefish"
+
+[module]
+  [[module.imports]]
+    path = "github.com/thingsym/hugo-theme-techdoc"
diff --git a/docs/layouts/_default/list.html b/docs/layouts/_default/list.html
deleted file mode 100644
index b23d375..0000000
--- a/docs/layouts/_default/list.html
+++ /dev/null
@@ -1,12 +0,0 @@
-{{ define "header_css" }}{{ end }}
-{{ define "body_classes" }}page-default-list{{ end }}
-{{ define "header_classes" }}{{ end }}
-
-{{ define "main" }}
-
-<h1 class="title">{{ .Title }}</h1>
-<div class="content">
-  {{ .Content }}
-</div>
-
-{{ end }}
diff --git a/docs/layouts/partials/sub-footer.html b/docs/layouts/partials/sub-footer.html
deleted file mode 100644
index 47be19c..0000000
--- a/docs/layouts/partials/sub-footer.html
+++ /dev/null
@@ -1,16 +0,0 @@
-<div class="sub-footer">
-    <div class="container">
-      <div class="row">
-        <div class="col-12">
-          <div class="sub-footer-inner">
-            <ul>
-              <li class="zerostatic"><a href="https://github.com/JugglerX/hugo-whisper-theme">Hugo Whisper Theme</a> by <a href="https://www.zerostatic.io">www.zerostatic.io</a></li>
-              <li>/</li>
-              <li>(C) 2019-2023 TSUYUSATO "<a href="https://github.com/MakeNowJust">MakeNowJust</a>" Kitsune</li>
-            </ul>
-          </div>
-        </div>
-      </div>
-    </div>
-  </div>
-  
\ No newline at end of file
diff --git a/docs/static/images/terminal.svg b/docs/static/images/terminal.svg
deleted file mode 120000
index ae07bd8..0000000
--- a/docs/static/images/terminal.svg
+++ /dev/null
@@ -1,1 +0,0 @@
-../../../images/terminal.svg
\ No newline at end of file
diff --git a/docs/themes/hugo-whisper-theme b/docs/themes/hugo-whisper-theme
deleted file mode 160000
index 8c982a7..0000000
--- a/docs/themes/hugo-whisper-theme
+++ /dev/null
@@ -1,1 +0,0 @@
-Subproject commit 8c982a7455d544881d3bea6e137a6b574b9cb176
diff --git a/go.mod b/go.mod
index cfc6216..0defff2 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/MakeNowJust/heredoc/v2 v2.0.1
-	github.com/google/go-cmp v0.5.9
+	github.com/google/go-cmp v0.6.0
 	github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc
 	github.com/pmezard/go-difflib v1.0.0
 )
diff --git a/go.sum b/go.sum
index aef80fa..3b8b22a 100644
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,8 @@ github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZ
 github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc h1:XLjmW07gT7cG/wb6mavIrvAIWBYaTacPo8UOnxGSspA=
diff --git a/renovate.json b/renovate.json
new file mode 100644
index 0000000..5db72dd
--- /dev/null
+++ b/renovate.json
@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}
diff --git a/split.go b/split.go
new file mode 100644
index 0000000..9f0193c
--- /dev/null
+++ b/split.go
@@ -0,0 +1,48 @@
+package memefish
+
+import "github.com/cloudspannerecosystem/memefish/token"
+
+// SplitRawStatements splits an input string to statement strings at terminating semicolons without parsing.
+// Statements are terminated by `;`, `<eof>` or `;<eof>` and the minimum output will be []string{""}.
+// See [terminating semicolons].
+// This function won't panic but return error if lexer become error state.
+// filepath can be used in error message.
+//
+// [terminating semicolons]: https://cloud.google.com/spanner/docs/reference/standard-sql/lexical#terminating_semicolons
+func SplitRawStatements(filepath, s string) ([]string, error) {
+	lex := &Lexer{
+		File: &token.File{
+			FilePath: filepath,
+			Buffer:   s,
+		},
+	}
+
+	var result []string
+	var firstPos token.Pos
+	for {
+		if lex.Token.Kind == ";" {
+			result = append(result, s[firstPos:lex.Token.Pos])
+			if err := lex.NextToken(); err != nil {
+				return nil, err
+			}
+			firstPos = lex.Token.Pos
+			continue
+		}
+
+		err := lex.NextToken()
+		if err != nil {
+			return nil, err
+		}
+
+		if lex.Token.Kind == token.TokenEOF {
+			if lex.Token.Pos != firstPos {
+				result = append(result, s[firstPos:lex.Token.Pos])
+			}
+			break
+		}
+	}
+	if len(result) == 0 {
+		return []string{""}, nil
+	}
+	return result, nil
+}


Test patch:
diff --git a/split_test.go b/split_test.go
new file mode 100644
index 0000000..d9b8a5b
--- /dev/null
+++ b/split_test.go
@@ -0,0 +1,57 @@
+package memefish_test
+
+import (
+	"github.com/cloudspannerecosystem/memefish"
+	"github.com/google/go-cmp/cmp"
+	"regexp"
+	"testing"
+)
+
+func TestSplitRawStatements(t *testing.T) {
+	for _, test := range []struct {
+		desc  string
+		input string
+		errRe *regexp.Regexp
+		want  []string
+	}{
+		// SplitRawStatements treats only lexical structures, so the test cases can be invalid statements.
+		{desc: "empty input", input: "", want: []string{""}},
+		{desc: "single statement ends with semicolon", input: `SELECT "123";`, want: []string{`SELECT "123"`}},
+		{desc: "single statement ends with EOF", input: `SELECT "123"`, want: []string{`SELECT "123"`}},
+		{desc: "two statement ends with semicolon", input: `SELECT "123"; SELECT "456";`, want: []string{`SELECT "123"`, `SELECT "456"`}},
+		{desc: "two statement ends with EOF", input: `SELECT "123"; SELECT "456"`, want: []string{`SELECT "123"`, `SELECT "456"`}},
+		{desc: "second statement is empty", input: `SELECT 1; ;`, want: []string{`SELECT 1`, ``}},
+		{desc: "two statement with new lines", input: "SELECT 1;\n SELECT 2;\n", want: []string{"SELECT 1", "SELECT 2"}},
+		{desc: "single statement with line comment", input: `SELECT 1//
+`, want: []string{"SELECT 1//\n"}},
+		{desc: "semicolon in line comment", input: "SELECT 1 //;\n + 2", want: []string{"SELECT 1 //;\n + 2"}},
+		{desc: "semicolon in multi-line comment", input: "SELECT 1 /*;\n*/ + 2", want: []string{"SELECT 1 /*;\n*/ + 2"}},
+		{desc: "semicolon in double-quoted string", input: `SELECT "1;2;3";`, want: []string{`SELECT "1;2;3"`}},
+		{desc: "semicolon in single-quoted string", input: `SELECT '1;2;3';`, want: []string{`SELECT '1;2;3'`}},
+		{desc: "semicolon in back-quote", input: "SELECT `1;2;3`;", want: []string{"SELECT `1;2;3`"}},
+		// $` may become a valid token in the future, but it's reasonable to check its current behavior.
+		{desc: "unknown token", input: "SELECT $;", errRe: regexp.MustCompile(`illegal input character: '\$'`)},
+	} {
+		t.Run(test.desc, func(t *testing.T) {
+			stmts, err := memefish.SplitRawStatements("", test.input)
+			if err != nil {
+				if test.errRe == nil {
+					t.Errorf("should success, but %v", err)
+					return
+				}
+				if !test.errRe.MatchString(err.Error()) {
+					t.Errorf("error message should match %q, but %q", test.errRe, err)
+					return
+				}
+			}
+			if err == nil && test.errRe != nil {
+				t.Errorf("success, but should fail %q", test.errRe)
+				return
+			}
+			if diff := cmp.Diff(stmts, test.want); diff != "" {
+				t.Errorf("differs: %v", diff)
+				return
+			}
+		})
+	}
+}