Repository: goplus/gop
Pull request #2141 (2025-03-01 07:34:26)

gop/tpl


Patch:
diff --git a/parser/interface.go b/parser/interface.go
index 7fd616af..831ac793 100644
--- a/parser/interface.go
+++ b/parser/interface.go
@@ -23,6 +23,7 @@ import (
 	"go/scanner"
 
 	"github.com/goplus/gop/ast"
+	"github.com/goplus/gop/parser/iox"
 	"github.com/goplus/gop/token"
 )
 
@@ -83,7 +84,7 @@ func parseFile(fset *token.FileSet, filename string, src interface{}, mode Mode)
 	}
 
 	// get source
-	text, err := readSourceLocal(filename, src)
+	text, err := iox.ReadSourceLocal(filename, src)
 	if err != nil {
 		return
 	}
@@ -133,7 +134,7 @@ func parseFile(fset *token.FileSet, filename string, src interface{}, mode Mode)
 // are returned via a scanner.ErrorList which is sorted by source position.
 func ParseExprFrom(fset *token.FileSet, filename string, src any, mode Mode) (expr ast.Expr, err error) {
 	// get source
-	text, err := readSourceLocal(filename, src)
+	text, err := iox.ReadSourceLocal(filename, src)
 	if err != nil {
 		return
 	}
diff --git a/parser/iox/io.go b/parser/iox/io.go
new file mode 100644
index 00000000..3aa29653
--- /dev/null
+++ b/parser/iox/io.go
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package iox
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"os"
+)
+
+// -----------------------------------------------------------------------------
+
+var (
+	ErrInvalidSource = errors.New("invalid source")
+)
+
+func ReadSource(src any) ([]byte, error) {
+	switch s := src.(type) {
+	case string:
+		return []byte(s), nil
+	case []byte:
+		return s, nil
+	case *bytes.Buffer:
+		// is io.Reader, but src is already available in []byte form
+		if s != nil {
+			return s.Bytes(), nil
+		}
+	case io.Reader:
+		return io.ReadAll(s)
+	}
+	return nil, ErrInvalidSource
+}
+
+// If src != nil, readSource converts src to a []byte if possible;
+// otherwise it returns an error. If src == nil, readSource returns
+// the result of reading the file specified by filename.
+func ReadSourceLocal(filename string, src any) ([]byte, error) {
+	if src != nil {
+		return ReadSource(src)
+	}
+	return os.ReadFile(filename)
+}
+
+// -----------------------------------------------------------------------------
diff --git a/parser/parser_gop.go b/parser/parser_gop.go
index e7b3b02b..6309871b 100644
--- a/parser/parser_gop.go
+++ b/parser/parser_gop.go
@@ -17,11 +17,8 @@
 package parser
 
 import (
-	"bytes"
 	"errors"
-	"io"
 	"io/fs"
-	"os"
 	"path"
 	"strings"
 
@@ -30,6 +27,7 @@ import (
 
 	"github.com/goplus/gop/ast"
 	"github.com/goplus/gop/parser/fsx"
+	"github.com/goplus/gop/parser/iox"
 	"github.com/goplus/gop/token"
 )
 
@@ -340,40 +338,9 @@ func ParseFSFile(fset *token.FileSet, fs FileSystem, filename string, src interf
 	return parseFile(fset, filename, code, mode)
 }
 
-var (
-	errInvalidSource = errors.New("invalid source")
-)
-
-func readSource(src interface{}) ([]byte, error) {
-	switch s := src.(type) {
-	case string:
-		return []byte(s), nil
-	case []byte:
-		return s, nil
-	case *bytes.Buffer:
-		// is io.Reader, but src is already available in []byte form
-		if s != nil {
-			return s.Bytes(), nil
-		}
-	case io.Reader:
-		return io.ReadAll(s)
-	}
-	return nil, errInvalidSource
-}
-
-// If src != nil, readSource converts src to a []byte if possible;
-// otherwise it returns an error. If src == nil, readSource returns
-// the result of reading the file specified by filename.
-func readSourceLocal(filename string, src interface{}) ([]byte, error) {
-	if src != nil {
-		return readSource(src)
-	}
-	return os.ReadFile(filename)
-}
-
 func readSourceFS(fs FileSystem, filename string, src interface{}) ([]byte, error) {
 	if src != nil {
-		return readSource(src)
+		return iox.ReadSource(src)
 	}
 	return fs.ReadFile(filename)
 }
diff --git a/tpl/cl/compile.go b/tpl/cl/compile.go
index 290853b3..e1d9ed7b 100644
--- a/tpl/cl/compile.go
+++ b/tpl/cl/compile.go
@@ -35,7 +35,7 @@ var (
 // Result represents the result of compiling a set of rules.
 type Result struct {
 	Doc   *matcher.Var
-	rules map[string]*matcher.Var
+	Rules map[string]*matcher.Var
 }
 
 type context struct {
diff --git a/tpl/parser/parser.go b/tpl/parser/parser.go
index fce1cd70..23cf5ee0 100644
--- a/tpl/parser/parser.go
+++ b/tpl/parser/parser.go
@@ -17,11 +17,7 @@
 package parser
 
 import (
-	"bytes"
-	"errors"
-	"io"
-	"os"
-
+	"github.com/goplus/gop/parser/iox"
 	"github.com/goplus/gop/tpl/ast"
 	"github.com/goplus/gop/tpl/scanner"
 	"github.com/goplus/gop/tpl/token"
@@ -36,7 +32,7 @@ type Mode uint
 
 // ParseFile parses a file and returns the AST.
 func ParseFile(fset *token.FileSet, filename string, src any, _ Mode) (f *ast.File, err error) {
-	b, err := readSourceLocal(filename, src)
+	b, err := iox.ReadSourceLocal(filename, src)
 	if err != nil {
 		return nil, err
 	}
@@ -56,39 +52,6 @@ func ParseFile(fset *token.FileSet, filename string, src any, _ Mode) (f *ast.Fi
 
 // -----------------------------------------------------------------------------
 
-var (
-	errInvalidSource = errors.New("invalid source")
-)
-
-func readSource(src any) ([]byte, error) {
-	switch s := src.(type) {
-	case string:
-		return []byte(s), nil
-	case []byte:
-		return s, nil
-	case *bytes.Buffer:
-		// is io.Reader, but src is already available in []byte form
-		if s != nil {
-			return s.Bytes(), nil
-		}
-	case io.Reader:
-		return io.ReadAll(s)
-	}
-	return nil, errInvalidSource
-}
-
-// If src != nil, readSource converts src to a []byte if possible;
-// otherwise it returns an error. If src == nil, readSource returns
-// the result of reading the file specified by filename.
-func readSourceLocal(filename string, src any) ([]byte, error) {
-	if src != nil {
-		return readSource(src)
-	}
-	return os.ReadFile(filename)
-}
-
-// -----------------------------------------------------------------------------
-
 // parser represents a parser.
 type parser struct {
 	scanner scanner.Scanner
@@ -106,7 +69,7 @@ type parser struct {
 func (p *parser) init(fset *token.FileSet, filename string, src []byte) {
 	p.file = fset.AddFile(filename, -1, len(src))
 	eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) }
-	p.scanner.Init(p.file, src, eh, scanner.InsertSemis)
+	p.scanner.Init(p.file, src, eh, 0)
 	p.next() // initialize first token
 }
 
diff --git a/tpl/scanner/scanner.go b/tpl/scanner/scanner.go
index c696875a..de65b44c 100644
--- a/tpl/scanner/scanner.go
+++ b/tpl/scanner/scanner.go
@@ -100,8 +100,8 @@ const (
 	// ScanComments means returning comments as COMMENT tokens
 	ScanComments ScanMode = 1 << iota
 
-	// InsertSemis means automatically insert semicolons
-	InsertSemis
+	// NoInsertSemis means don't automatically insert semicolons
+	NoInsertSemis
 )
 
 // Init prepares the scanner s to tokenize the text src by setting the
@@ -788,7 +788,7 @@ scanAgain:
 			t.Lit = string(ch)
 		}
 	}
-	if s.mode&InsertSemis != 0 {
+	if s.mode&NoInsertSemis == 0 {
 		s.insertSemi = insertSemi
 	}
 
diff --git a/tpl/tpl.go b/tpl/tpl.go
new file mode 100644
index 00000000..50f1b1c5
--- /dev/null
+++ b/tpl/tpl.go
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tpl
+
+import (
+	"github.com/goplus/gop/parser/iox"
+	"github.com/goplus/gop/tpl/cl"
+	"github.com/goplus/gop/tpl/matcher"
+	"github.com/goplus/gop/tpl/parser"
+	"github.com/goplus/gop/tpl/scanner"
+	"github.com/goplus/gop/tpl/token"
+	"github.com/goplus/gop/tpl/types"
+)
+
+// -----------------------------------------------------------------------------
+
+// Compiler represents a TPL compiler.
+type Compiler struct {
+	cl.Result
+}
+
+// New creates a new TPL compiler.
+func New(filename string, src any, fset *token.FileSet) (ret Compiler, err error) {
+	if fset == nil {
+		fset = token.NewFileSet()
+	}
+	f, err := parser.ParseFile(fset, filename, src, 0)
+	if err != nil {
+		return
+	}
+	ret.Result, err = cl.New(fset, f)
+	return
+}
+
+// -----------------------------------------------------------------------------
+
+type Scanner interface {
+	Scan() types.Token
+	Init(file *token.File, src []byte, err scanner.ScanErrorHandler, mode scanner.ScanMode)
+}
+
+type Config struct {
+	Fset             *token.FileSet
+	Scanner          Scanner
+	ScanMode         scanner.ScanMode
+	ScanErrorHandler scanner.ScanErrorHandler
+}
+
+func (p *Compiler) Eval(filename string, src any, conf *Config) (result any, err error) {
+	b, err := iox.ReadSourceLocal(filename, src)
+	if err != nil {
+		return
+	}
+	if conf == nil {
+		conf = &Config{}
+	}
+	fset := conf.Fset
+	if fset == nil {
+		fset = token.NewFileSet()
+	}
+	s := conf.Scanner
+	if s == nil {
+		s = new(scanner.Scanner)
+	}
+	f := fset.AddFile(filename, fset.Base(), len(b))
+	s.Init(f, b, conf.ScanErrorHandler, conf.ScanMode)
+	toks := make([]*types.Token, 0, len(b)>>3)
+	for {
+		t := s.Scan()
+		if t.Tok == token.EOF {
+			break
+		}
+		toks = append(toks, &t)
+	}
+	ctx := &matcher.Context{
+		Fset:    fset,
+		FileEnd: token.Pos(f.Base() + len(b)),
+	}
+	n, result, err := p.Doc.Match(toks, ctx)
+	if err != nil {
+		return
+	}
+	if n < len(toks) {
+		err = ctx.NewErrorf(toks[n].Pos, "unexpected token: %v", toks[n])
+	}
+	return
+}
+
+// -----------------------------------------------------------------------------


Test patch:
diff --git a/parser/parserdir_test.go b/parser/parserdir_test.go
index 086a27fc..8c4d29ed 100644
--- a/parser/parserdir_test.go
+++ b/parser/parserdir_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/goplus/gop/ast"
 	"github.com/goplus/gop/parser/fsx"
 	"github.com/goplus/gop/parser/fsx/memfs"
+	"github.com/goplus/gop/parser/iox"
 	"github.com/goplus/gop/parser/parsertest"
 	"github.com/goplus/gop/scanner"
 	"github.com/goplus/gop/token"
@@ -319,17 +320,17 @@ func TestParseExprFrom(t *testing.T) {
 
 func TestReadSource(t *testing.T) {
 	buf := bytes.NewBuffer(nil)
-	if _, err := readSource(buf); err != nil {
+	if _, err := iox.ReadSource(buf); err != nil {
 		t.Fatal("readSource failed:", err)
 	}
 	sr := strings.NewReader("")
-	if _, err := readSource(sr); err != nil {
+	if _, err := iox.ReadSource(sr); err != nil {
 		t.Fatal("readSource strings.Reader failed:", err)
 	}
-	if _, err := readSource(0); err == nil {
+	if _, err := iox.ReadSource(0); err == nil {
 		t.Fatal("readSource int failed: no error?")
 	}
-	if _, err := readSourceLocal("/foo/bar/not-exists", nil); err == nil {
+	if _, err := iox.ReadSourceLocal("/foo/bar/not-exists", nil); err == nil {
 		t.Fatal("readSourceLocal int failed: no error?")
 	}
 }
@@ -343,7 +344,7 @@ func TestParseFiles(t *testing.T) {
 
 func TestIparseFileInvalidSrc(t *testing.T) {
 	fset := token.NewFileSet()
-	if _, err := parseFile(fset, "/foo/bar/not-exists", 1, PackageClauseOnly); err != errInvalidSource {
+	if _, err := parseFile(fset, "/foo/bar/not-exists", 1, PackageClauseOnly); err != iox.ErrInvalidSource {
 		t.Fatal("ParseFile failed: not errInvalidSource?")
 	}
 }
diff --git a/tpl/scanner/scanner_test.go b/tpl/scanner/scanner_test.go
index 86737411..ed0bcc2b 100644
--- a/tpl/scanner/scanner_test.go
+++ b/tpl/scanner/scanner_test.go
@@ -128,7 +128,7 @@ world !=
 	fset := token.NewFileSet()
 	file := fset.AddFile("", -1, len(grammar))
 
-	s.Init(file, []byte(grammar), nil /* no error handler */, ScanComments|InsertSemis)
+	s.Init(file, []byte(grammar), nil /* no error handler */, ScanComments)
 	i := 0
 	for {
 		c := s.Scan()
@@ -145,7 +145,7 @@ world !=
 		t.Fatalf("len(expected) != i: %d, %d\n", len(expected), i)
 	}
 
-	s.Init(file, []byte(grammar), nil, 0)
+	s.Init(file, []byte(grammar), nil, NoInsertSemis)
 	i = 0
 	for {
 		c := s.Scan()
@@ -176,7 +176,7 @@ world !=
 		t.Fatalf("len(expected) != i: %d, %d\n", len(expected), i)
 	}
 
-	s.Init(file, []byte(grammar), nil, InsertSemis)
+	s.Init(file, []byte(grammar), nil, 0)
 	i = 0
 	for {
 		c := s.Scan()