Repository: tableauio/tableau (go)

Issue:
protogen(yaml): only read schema documents for templating data document
Design goal: support Jinja2 (or other templating libs) templating of data document
```yaml
---
"@sheet": "@TABLEAU"
ServiceConf:
  Template: true
---
# define schema
"@sheet": "@ServiceConf"
ID: uint32
Name: string
---
"@sheet": ServiceConf
ID: {{ env.id }}
Name: {{ env.name}}
{% if env.name == 'prod' %}
Enabled: true
{% else %}
Enabled: false
{% endif %}
```

Pull request:
protogen(yaml): only read schema documents for templating data document
- close #118

Patch:
diff --git a/internal/importer/yaml.go b/internal/importer/yaml.go
index 72e8369..6845bb3 100644
--- a/internal/importer/yaml.go
+++ b/internal/importer/yaml.go
@@ -1,10 +1,12 @@
 package importer
 
 import (
+	"bufio"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -18,31 +20,36 @@ type YAMLImporter struct {
 }
 
 func NewYAMLImporter(filename string, sheetNames []string, parser book.SheetParser, mode ImporterMode, cloned bool) (*YAMLImporter, error) {
-	book, err := readYAMLBook(filename, parser)
-	if err != nil {
-		return nil, errors.WithMessagef(err, "failed to read csv book: %s", filename)
-	}
-
+	var book *book.Book
+	var err error
 	if mode == Protogen {
+		book, err = readYAMLBookWithOnlySchemaSheet(filename, parser)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "failed to read csv book: %s", filename)
+		}
 		if err := book.ParseMetaAndPurge(); err != nil {
 			return nil, errors.WithMessage(err, "failed to parse metasheet")
 		}
+	} else {
+		book, err = readYAMLBook(filename, parser)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "failed to read csv book: %s", filename)
+		}
 	}
 
 	// log.Debugf("book: %+v", book)
-
 	return &YAMLImporter{
 		Book: book,
 	}, nil
 }
 
 func readYAMLBook(filename string, parser book.SheetParser) (*book.Book, error) {
+	bookName := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
+	newBook := book.NewBook(bookName, filename, parser)
 	file, err := os.Open(filename)
 	if err != nil {
 		return nil, err
 	}
-	bookName := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
-	newBook := book.NewBook(bookName, filename, parser)
 	// parse all documents in a file
 	decoder := yaml.NewDecoder(file)
 	for i := 0; ; i++ {
@@ -62,7 +69,36 @@ func readYAMLBook(filename string, parser book.SheetParser) (*book.Book, error)
 		}
 		newBook.AddSheet(sheet)
 	}
+	return newBook, nil
+}
+
+func readYAMLBookWithOnlySchemaSheet(filename string, parser book.SheetParser) (*book.Book, error) {
+	bookName := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
+	newBook := book.NewBook(bookName, filename, parser)
 
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	rawDocs, err := extractRawYAMLDocuments(string(content))
+	if err != nil {
+		return nil, err
+	}
+	for i, rawDoc := range rawDocs {
+		if !isSchemaSheet(rawDoc) {
+			continue
+		}
+		var doc yaml.Node
+		err := yaml.Unmarshal([]byte(rawDoc), &doc)
+		if err != nil {
+			return nil, err
+		}
+		sheet, err := parseYAMLSheet(&doc, i)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "file: %s", filename)
+		}
+		newBook.AddSheet(sheet)
+	}
 	return newBook, nil
 }
 
@@ -165,3 +201,38 @@ func parseYAMLNode(node *yaml.Node, bnode *book.Node) error {
 		return errors.Errorf("unknown yaml node(%d:%d) kind: %v, value: %v", node.Line, node.Column, node.Kind, node.Value)
 	}
 }
+
+var yamlSheetNameRegexp *regexp.Regexp
+
+func init() {
+	yamlSheetNameRegexp = regexp.MustCompile(`"@sheet"\s*:\s*(.+)`) // e.g.: "@sheet": "@EnvConf"
+}
+
+const yamlDocumentSeparator = "---"
+
+// extractRawYAMLDocuments extracts raw YAML into separate documents.
+func extractRawYAMLDocuments(content string) ([]string, error) {
+	rawDocuments := strings.Split(content, yamlDocumentSeparator)
+	var documents []string
+	for _, doc := range rawDocuments {
+		trimmedDoc := strings.TrimSpace(doc)
+		if trimmedDoc != "" {
+			documents = append(documents, trimmedDoc)
+		}
+	}
+	return documents, nil
+}
+
+func isSchemaSheet(rawDoc string) bool {
+	scanner := bufio.NewScanner(strings.NewReader(rawDoc))
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := yamlSheetNameRegexp.FindStringSubmatch(line)
+		if len(matches) == 2 {
+			sheetName := strings.Trim(matches[1], `"`)
+			// log.Debugf("sheet: %s", sheetName)
+			return strings.HasPrefix(sheetName, "@")
+		}
+	}
+	return false
+}


Test patch:
diff --git a/internal/importer/testdata/TestTemplate.yaml b/internal/importer/testdata/TestTemplate.yaml
new file mode 100644
index 0000000..d889f48
--- /dev/null
+++ b/internal/importer/testdata/TestTemplate.yaml
@@ -0,0 +1,18 @@
+---
+"@sheet": "@TABLEAU"
+ServiceConf:
+  Template: true
+---
+# define schema
+"@sheet": "@ServiceConf"
+ID: uint32
+Name: string
+---
+"@sheet": ServiceConf
+ID: {{ env.id }}
+Name: {{ env.name}}
+{% if env.name == 'prod' %}
+Enabled: true
+{% else %}
+Enabled: false
+{% endif %}
\ No newline at end of file
diff --git a/internal/importer/yaml_test.go b/internal/importer/yaml_test.go
index c9cd11e..1e100cb 100644
--- a/internal/importer/yaml_test.go
+++ b/internal/importer/yaml_test.go
@@ -1,13 +1,12 @@
 package importer
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"testing"
 
-	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
 	"github.com/tableauio/tableau/internal/importer/book"
+	"google.golang.org/protobuf/proto"
 	"gopkg.in/yaml.v3"
 )
 
@@ -16,58 +15,58 @@ func Test_inspectYAMLNode(t *testing.T) {
 	data := []byte(`
 ---
 "@sheet": "@TABLEAU"
-LiteConf:
-LoaderConf:
-  OrderedMap: true
+ServiceConf:
+  Template: true
 ---
-ScalarList: [1, 2, 3]
-Servers:
-  gamesvr:
-    Name: gamesvr
-    Confs:
-      ItemConf:
-        Async: true
-      DropConf:
-        Async: true
-  mailsvr:
-    Name: mailsvr
-    Confs:
-      ItemConf:
-        Async: true
-      DropConf:
-        Async: true
+# define schema
+"@sheet": "@ServiceConf"
+ID: uint32
+Name: string
+---
+"@sheet": ServiceConf
+ID: {{ env.id }}
+Name: {{ env.name}}
+{% if env.name == 'prod' %}
+Enabled: true
+{% else %}
+Enabled: false
+{% endif %}
 `)
 
-	// Create a new decoder
-	dec := yaml.NewDecoder(bytes.NewReader(data))
-	for i := 0; ; i++ {
-		var node yaml.Node
-		// Decode one document at a time
-		err := dec.Decode(&node)
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			} else {
-				t.Fatalf("error: %v", err)
-			}
+	rawDocs, err := extractRawYAMLDocuments(string(data))
+	require.NoError(t, err)
+	for i, rawDoc := range rawDocs {
+		if !isSchemaSheet(rawDoc) {
+			continue
 		}
+		var node yaml.Node
+		err := yaml.Unmarshal([]byte(rawDoc), &node)
+		require.NoError(t, err)
 		sheet, err := parseYAMLSheet(&node, i)
-		if err != nil {
-			t.Fatalf("%+v", err)
-		}
+		require.NoError(t, err)
 		fmt.Println(sheet.String())
 	}
 }
 
-func Test_readYAMLBook(t *testing.T) {
+type TestSheetParser struct {
+}
+
+func (p *TestSheetParser) Parse(protomsg proto.Message, sheet *book.Sheet) error {
+	return nil
+}
+
+func TestNewYAMLImporter(t *testing.T) {
 	type args struct {
-		filename string
-		parser   book.SheetParser
+		filename   string
+		sheetNames []string
+		parser     book.SheetParser
+		mode       ImporterMode
+		cloned     bool
 	}
 	tests := []struct {
 		name    string
 		args    args
-		want    *book.Book
+		want    *YAMLImporter
 		wantErr bool
 	}{
 		{
@@ -77,17 +76,25 @@ func Test_readYAMLBook(t *testing.T) {
 				parser:   nil,
 			},
 		},
+		{
+			name: "TestTemplate.yaml",
+			args: args{
+				filename: "testdata/TestTemplate.yaml",
+				mode:     Protogen,
+				parser:   &TestSheetParser{},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := readYAMLBook(tt.args.filename, tt.args.parser)
+			got, err := NewYAMLImporter(tt.args.filename, tt.args.sheetNames, tt.args.parser, tt.args.mode, tt.args.cloned)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("readYAMLBook() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("NewYAMLImporter() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			fmt.Println(got.String())
 			// if !reflect.DeepEqual(got, tt.want) {
-			// 	t.Errorf("readYAMLBook() = %v, want %v", got, tt.want)
+			// 	t.Errorf("NewYAMLImporter() = %v, want %v", got, tt.want)
 			// }
 		})
 	}