Repository: goplus/gop
Pull request #2224 (2025-03-30 12:31:45)

scannertest


Patch:
diff --git a/token/token.go b/token/token.go
index aaa2a07a..c4d7f891 100644
--- a/token/token.go
+++ b/token/token.go
@@ -179,6 +179,7 @@ var tokens = [...]string{
 	CSTRING:  "CSTRING",
 	PYSTRING: "PYSTRING",
 	RAT:      "RAT",
+	UNIT:     "UNIT",
 
 	ADD: "+",
 	SUB: "-",


Test patch:
diff --git a/tpl/scanner/_testdata/cstr/go.expect b/tpl/scanner/_testdata/cstr/go.expect
new file mode 100644
index 00000000..aee852ee
--- /dev/null
+++ b/tpl/scanner/_testdata/cstr/go.expect
@@ -0,0 +1,14 @@
+1 IDENT c
+2 . 
+3 IDENT printf
+10 IDENT c
+11 STRING "Hello"
+18 ; 
+
+19 IDENT std
+22 . 
+23 IDENT print
+29 IDENT py
+31 STRING "Hello"
+38 ; 
+
diff --git a/tpl/scanner/_testdata/cstr/gop.expect b/tpl/scanner/_testdata/cstr/gop.expect
new file mode 100644
index 00000000..16ea4142
--- /dev/null
+++ b/tpl/scanner/_testdata/cstr/gop.expect
@@ -0,0 +1,12 @@
+1 IDENT c
+2 . 
+3 IDENT printf
+10 CSTRING "Hello"
+18 ; 
+
+19 IDENT std
+22 . 
+23 IDENT print
+29 PYSTRING "Hello"
+38 ; 
+
diff --git a/tpl/scanner/_testdata/cstr/in.gop b/tpl/scanner/_testdata/cstr/in.gop
new file mode 100644
index 00000000..ba2ad819
--- /dev/null
+++ b/tpl/scanner/_testdata/cstr/in.gop
@@ -0,0 +1,2 @@
+c.printf c"Hello"
+std.print py"Hello"
diff --git a/tpl/scanner/_testdata/cstr/tpl.expect b/tpl/scanner/_testdata/cstr/tpl.expect
new file mode 100644
index 00000000..aee852ee
--- /dev/null
+++ b/tpl/scanner/_testdata/cstr/tpl.expect
@@ -0,0 +1,14 @@
+1 IDENT c
+2 . 
+3 IDENT printf
+10 IDENT c
+11 STRING "Hello"
+18 ; 
+
+19 IDENT std
+22 . 
+23 IDENT print
+29 IDENT py
+31 STRING "Hello"
+38 ; 
+
diff --git a/tpl/scanner/_testdata/rat/go.expect b/tpl/scanner/_testdata/rat/go.expect
new file mode 100644
index 00000000..e3c89db5
--- /dev/null
+++ b/tpl/scanner/_testdata/rat/go.expect
@@ -0,0 +1,12 @@
+1 IDENT echo
+6 INT 1
+7 / 
+8 INT 3
+9 IDENT r
+10 ; 
+
+11 IDENT echo
+16 FLOAT 1.23
+20 IDENT r
+21 ; 
+
diff --git a/tpl/scanner/_testdata/rat/gop.expect b/tpl/scanner/_testdata/rat/gop.expect
new file mode 100644
index 00000000..5f75eba6
--- /dev/null
+++ b/tpl/scanner/_testdata/rat/gop.expect
@@ -0,0 +1,10 @@
+1 IDENT echo
+6 INT 1
+7 / 
+8 RAT 3r
+10 ; 
+
+11 IDENT echo
+16 RAT 1.23r
+21 ; 
+
diff --git a/tpl/scanner/_testdata/rat/in.gop b/tpl/scanner/_testdata/rat/in.gop
new file mode 100644
index 00000000..f008bdd9
--- /dev/null
+++ b/tpl/scanner/_testdata/rat/in.gop
@@ -0,0 +1,2 @@
+echo 1/3r
+echo 1.23r
diff --git a/tpl/scanner/_testdata/rat/tpl.expect b/tpl/scanner/_testdata/rat/tpl.expect
new file mode 100644
index 00000000..5f75eba6
--- /dev/null
+++ b/tpl/scanner/_testdata/rat/tpl.expect
@@ -0,0 +1,10 @@
+1 IDENT echo
+6 INT 1
+7 / 
+8 RAT 3r
+10 ; 
+
+11 IDENT echo
+16 RAT 1.23r
+21 ; 
+
diff --git a/tpl/scanner/_testdata/unit/go.expect b/tpl/scanner/_testdata/unit/go.expect
new file mode 100644
index 00000000..833b113d
--- /dev/null
+++ b/tpl/scanner/_testdata/unit/go.expect
@@ -0,0 +1,33 @@
+1 import import
+8 STRING "time"
+14 ; 
+
+16 IDENT echo
+21 STRING "Let's wait for a second"
+46 ; 
+
+47 IDENT time
+51 . 
+52 IDENT sleep
+58 INT 1
+59 IDENT s
+60 ; 
+
+61 IDENT time
+65 . 
+66 IDENT sleep
+72 FLOAT 1.
+74 IDENT s
+75 ; 
+
+76 IDENT time
+80 . 
+81 IDENT sleep
+87 FLOAT 1.2
+90 IDENT s
+91 ; 
+
+92 IDENT echo
+97 STRING "Hello, world"
+111 ; 
+
diff --git a/tpl/scanner/_testdata/unit/gop.expect b/tpl/scanner/_testdata/unit/gop.expect
new file mode 100644
index 00000000..17ac9eb1
--- /dev/null
+++ b/tpl/scanner/_testdata/unit/gop.expect
@@ -0,0 +1,33 @@
+1 import import
+8 STRING "time"
+14 ; 
+
+16 IDENT echo
+21 STRING "Let's wait for a second"
+46 ; 
+
+47 IDENT time
+51 . 
+52 IDENT sleep
+58 INT 1
+59 UNIT s
+60 ; 
+
+61 IDENT time
+65 . 
+66 IDENT sleep
+72 FLOAT 1.
+74 UNIT s
+75 ; 
+
+76 IDENT time
+80 . 
+81 IDENT sleep
+87 FLOAT 1.2
+90 UNIT s
+91 ; 
+
+92 IDENT echo
+97 STRING "Hello, world"
+111 ; 
+
diff --git a/tpl/scanner/_testdata/unit/in.gop b/tpl/scanner/_testdata/unit/in.gop
new file mode 100644
index 00000000..23fa95de
--- /dev/null
+++ b/tpl/scanner/_testdata/unit/in.gop
@@ -0,0 +1,7 @@
+import "time"
+
+echo "Let's wait for a second"
+time.sleep 1s
+time.sleep 1.s
+time.sleep 1.2s
+echo "Hello, world"
diff --git a/tpl/scanner/_testdata/unit/tpl.expect b/tpl/scanner/_testdata/unit/tpl.expect
new file mode 100644
index 00000000..70503895
--- /dev/null
+++ b/tpl/scanner/_testdata/unit/tpl.expect
@@ -0,0 +1,33 @@
+1 IDENT import
+8 STRING "time"
+14 ; 
+
+16 IDENT echo
+21 STRING "Let's wait for a second"
+46 ; 
+
+47 IDENT time
+51 . 
+52 IDENT sleep
+58 INT 1
+59 UNIT s
+60 ; 
+
+61 IDENT time
+65 . 
+66 IDENT sleep
+72 FLOAT 1.
+74 UNIT s
+75 ; 
+
+76 IDENT time
+80 . 
+81 IDENT sleep
+87 FLOAT 1.2
+90 UNIT s
+91 ; 
+
+92 IDENT echo
+97 STRING "Hello, world"
+111 ; 
+
diff --git a/tpl/scanner/scandir_test.go b/tpl/scanner/scandir_test.go
new file mode 100644
index 00000000..6d0eae31
--- /dev/null
+++ b/tpl/scanner/scandir_test.go
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package scanner_test
+
+import (
+	"bytes"
+	"io"
+	"log"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/goplus/gop/tpl/scanner/scannertest"
+)
+
+func testScan(
+	t *testing.T, pkgDir string, in []byte,
+	expFile string, scan func(w io.Writer, in []byte)) {
+	expect, _ := os.ReadFile(pkgDir + "/" + expFile)
+	var b bytes.Buffer
+	scan(&b, in)
+	out := b.Bytes()
+	if scannertest.Diff(t, pkgDir+"/result.txt", out, expect) {
+		t.Fatal(expFile, ": unexpect result")
+	}
+}
+
+func testFrom(t *testing.T, pkgDir, sel string) {
+	if sel != "" && !strings.Contains(pkgDir, sel) {
+		return
+	}
+	t.Helper()
+	log.Println("Scanning", pkgDir)
+	in, err := os.ReadFile(pkgDir + "/in.gop")
+	if err != nil {
+		t.Fatal("Scanning", pkgDir, "-", err)
+	}
+	testScan(t, pkgDir, in, "tpl.expect", scannertest.Scan)
+	testScan(t, pkgDir, in, "go.expect", scannertest.GoScan)
+	testScan(t, pkgDir, in, "gop.expect", scannertest.GopScan)
+}
+
+func testFromDir(t *testing.T, sel, relDir string) {
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatal("Getwd failed:", err)
+	}
+	dir = path.Join(dir, relDir)
+	fis, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal("ReadDir failed:", err)
+	}
+	for _, fi := range fis {
+		name := fi.Name()
+		if strings.HasPrefix(name, "_") {
+			continue
+		}
+		t.Run(name, func(t *testing.T) {
+			pkgDir := dir + "/" + name
+			testFrom(t, pkgDir, sel)
+		})
+	}
+}
+
+func TestFromTestdata(t *testing.T) {
+	testFromDir(t, "", "./_testdata")
+}
diff --git a/tpl/scanner/scannertest/scannertest.go b/tpl/scanner/scannertest/scannertest.go
new file mode 100644
index 00000000..4d9729bb
--- /dev/null
+++ b/tpl/scanner/scannertest/scannertest.go
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package scannertest
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/goplus/gop/tpl/scanner"
+	"github.com/goplus/gop/tpl/token"
+
+	goscanner "go/scanner"
+	gotoken "go/token"
+
+	gopscanner "github.com/goplus/gop/scanner"
+	goptoken "github.com/goplus/gop/token"
+)
+
+// -----------------------------------------------------------------------------
+
+func Diff(t *testing.T, outfile string, dst, src []byte) bool {
+	line := 1
+	offs := 0 // line offset
+	for i := 0; i < len(dst) && i < len(src); i++ {
+		d := dst[i]
+		s := src[i]
+		if d != s {
+			os.WriteFile(outfile, dst, 0644)
+			t.Errorf("dst:%d: %s\n", line, dst[offs:])
+			t.Errorf("src:%d: %s\n", line, src[offs:])
+			return true
+		}
+		if s == '\n' {
+			line++
+			offs = i + 1
+		}
+	}
+	if len(dst) != len(src) {
+		os.WriteFile(outfile, dst, 0644)
+		t.Errorf("len(dst) = %d, len(src) = %d\ndst = %q\nsrc = %q", len(dst), len(src), dst, src)
+		return true
+	}
+	return false
+}
+
+// -----------------------------------------------------------------------------
+
+// Scan scans the input and writes the tokens to the writer.
+func Scan(w io.Writer, in []byte) {
+	var s scanner.Scanner
+	fset := gotoken.NewFileSet()
+	f := fset.AddFile("", -1, len(in))
+	s.Init(f, in, nil, scanner.ScanComments)
+	for {
+		t := s.Scan()
+		if t.Tok == token.EOF {
+			break
+		}
+		fmt.Fprintln(w, t.Pos, t.Tok, t.Lit)
+	}
+}
+
+// -----------------------------------------------------------------------------
+
+// GopScan scans the input using the Go+ standard library scanner and
+// writes the tokens to the writer.
+func GopScan(w io.Writer, in []byte) {
+	var s gopscanner.Scanner
+	fset := gotoken.NewFileSet()
+	f := fset.AddFile("", -1, len(in))
+	s.Init(f, in, nil, gopscanner.ScanComments)
+	for {
+		pos, tok, lit := s.Scan()
+		if tok == goptoken.EOF {
+			break
+		}
+		fmt.Fprintln(w, pos, tok, lit)
+	}
+}
+
+// -----------------------------------------------------------------------------
+
+// GoScan scans the input using the Go standard library scanner and
+// writes the tokens to the writer.
+func GoScan(w io.Writer, in []byte) {
+	var s goscanner.Scanner
+	fset := gotoken.NewFileSet()
+	f := fset.AddFile("", -1, len(in))
+	s.Init(f, in, nil, goscanner.ScanComments)
+	for {
+		pos, tok, lit := s.Scan()
+		if tok == gotoken.EOF {
+			break
+		}
+		fmt.Fprintln(w, pos, tok, lit)
+	}
+}
+
+// -----------------------------------------------------------------------------