Repository: tableauio/tableau
Pull request #26 (2022-10-18 04:06:38)

feat(xml): strip template content only when `Protogen` mode

codecov[bot]: # [Codecov](https://codecov.io/gh/tableauio/tableau/pull/26?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) Report
> Merging [#26](https://codecov.io/gh/tableauio/tableau/pull/26?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) (a961e6c) into [master](https://codecov.io/gh/tableauio/tableau/commit/1d05542181578a2c646a7886e51470046b3acbdd?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) (1d05542) will **increase** coverage by `0.41%`.
> The diff coverage is `61.11%`.

```diff
@@            Coverage Diff             @@
##           master      #26      +/-   ##
==========================================
+ Coverage   19.27%   19.68%   +0.41%     
==========================================
  Files          19       19              
  Lines        2880     2900      +20     
==========================================
+ Hits          555      571      +16     
- Misses       2281     2285       +4     
  Partials       44       44              
```

| Flag | Coverage Δ | |
|---|---|---|
| unittests | `19.68% <61.11%> (+0.41%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Impacted Files](https://codecov.io/gh/tableauio/tableau/pull/26?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) | Coverage Δ | |
|---|---|---|
| [internal/importer/importer.go](https://codecov.io/gh/tableauio/tableau/pull/26/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-aW50ZXJuYWwvaW1wb3J0ZXIvaW1wb3J0ZXIuZ28=) | `0.00% <0.00%> (ø)` | |
| [internal/importer/options.go](https://codecov.io/gh/tableauio/tableau/pull/26/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-aW50ZXJuYWwvaW1wb3J0ZXIvb3B0aW9ucy5nbw==) | `0.00% <0.00%> (ø)` | |
| [internal/protogen/protogen.go](https://codecov.io/gh/tableauio/tableau/pull/26/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-aW50ZXJuYWwvcHJvdG9nZW4vcHJvdG9nZW4uZ28=) | `0.00% <0.00%> (ø)` | |
| [internal/importer/xml.go](https://codecov.io/gh/tableauio/tableau/pull/26/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-aW50ZXJuYWwvaW1wb3J0ZXIveG1sLmdv) | `54.92% <70.37%> (+1.13%)` | :arrow_up: |
| [test/functest/util.go](https://codecov.io/gh/tableauio/tableau/pull/26/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-dGVzdC9mdW5jdGVzdC91dGlsLmdv) | `71.73% <100.00%> (+0.62%)` | :arrow_up: |

:mega: We’re building smart automated test selection to slash your CI/CD build times. [Learn more](https://about.codecov.io/iterative-testing/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio)


Patch:
diff --git a/internal/confgen/confgen.go b/internal/confgen/confgen.go
index 88b6618..284daf2 100644
--- a/internal/confgen/confgen.go
+++ b/internal/confgen/confgen.go
@@ -228,7 +228,7 @@ func (gen *Generator) convert(fd protoreflect.FileDescriptor, worksheetName stri
 	rewrittenWorkbookName := fs.RewriteSubdir(workbook.Name, gen.InputOpt.SubdirRewrites)
 	wbPath := filepath.Join(gen.InputDir, rewrittenWorkbookName)
 	log.Debugf("proto: %s, workbook options: %s", fd.Path(), workbook)
-	imp, err := importer.New(wbPath, importer.Sheets(sheets))
+	imp, err := importer.New(wbPath, importer.Sheets(sheets), importer.ImporterMode(importer.Confgen))
 	if err != nil {
 		return xerrors.WithMessageKV(err, xerrors.KeyModule, xerrors.ModuleConf, xerrors.KeyBookName, workbook.Name)
 	}
diff --git a/internal/importer/importer.go b/internal/importer/importer.go
index 9b7b5ff..46d92c9 100644
--- a/internal/importer/importer.go
+++ b/internal/importer/importer.go
@@ -40,7 +40,7 @@ func New(filename string, setters ...Option) (Importer, error) {
 	case format.CSV:
 		return NewCSVImporter(filename, opts.Sheets, opts.Parser)
 	case format.XML:
-		return NewXMLImporter(filename, opts.Sheets, opts.Parser)
+		return NewXMLImporter(filename, opts.Sheets, opts.Parser, opts.Mode)
 	default:
 		return nil, errors.Errorf("unsupported format: %v", fmt)
 	}
diff --git a/internal/importer/options.go b/internal/importer/options.go
index dba50c0..a7568a9 100644
--- a/internal/importer/options.go
+++ b/internal/importer/options.go
@@ -4,10 +4,20 @@ import (
 	"github.com/tableauio/tableau/internal/importer/book"
 )
 
+type Mode int
+
+// Importer mode
+const (
+	UnknownMode Mode = 0
+	Protogen    Mode = 1
+	Confgen     Mode = 2
+)
+
 type Options struct {
 	Sheets []string         // sheet names to import
 	Parser book.SheetParser // parser to parse the worksheet
 	TopN   uint             // read top N rows, 0 means read all rows
+	Mode   Mode             // importer mode, e.g.: Protogen
 }
 
 // Option is the functional option type.
@@ -31,6 +41,12 @@ func TopN(n uint) Option {
 	}
 }
 
+func ImporterMode(m Mode) Option {
+	return func(opts *Options) {
+		opts.Mode = m
+	}
+}
+
 func newDefaultOptions() *Options {
 	return &Options{}
 }
diff --git a/internal/importer/xml.go b/internal/importer/xml.go
index eaa3fb6..3630d0f 100644
--- a/internal/importer/xml.go
+++ b/internal/importer/xml.go
@@ -31,21 +31,32 @@ var metasheetRegexp *regexp.Regexp
 
 const (
 	xmlProlog             = `<?xml version='1.0' encoding='UTF-8'?>`
-	ungreedyPropGroup     = `(\|\{[^\{\}]+\})?`
 	atTableauDisplacement = `ATABLEAU`
-	metasheetItemBlock    = `(\s+<Item(\s+\S+\s*=\s*"\S+")+\s*/>\s+)*`
-	sheetBlock            = `<%v(>(.*\n)*</%v>|\s*/>)`
+	ungreedyPropGroup     = `(\|\{[^\{\}]+\})?`                        // e.g.: |{default:"100"}
+	metasheetItemBlock    = `(\s+<Item(\s+\S+\s*=\s*"\S+")+\s*/>\s+)*` // e.g.: <Item Sheet="XXXConf" Sep="|"/>
+	sheetBlock            = `<%v(>(.*\n)*</%v>|\s*/>)`                 // e.g.: <XXXConf>...</XXXConf>
 )
 
 func init() {
-	attrRegexp = regexp.MustCompile(`([0-9A-Za-z_]+)="` + types.TypeGroup + ungreedyPropGroup + `"`)
-	scalarListRegexp = regexp.MustCompile(`([A-Za-z_]+)([0-9]+)`)
+	attrRegexp = regexp.MustCompile(`([0-9A-Za-z_]+)\s*=\s*"` + types.TypeGroup + ungreedyPropGroup + `"`) // e.g.: Num = "int32|{range:"1,~"}"
+	scalarListRegexp = regexp.MustCompile(`([A-Za-z_]+)([0-9]+)`)                                          // e.g.: Para1, Para2, Para3, ...
+
+	// metasheet regexp, e.g.:
+	// <!--
+	// <@TABLEAU>
+	// 		<Item Sheet="Server" />
+	// </@TABLEAU>
+
+	// <Server>
+	// 		<Weight Num="map<uint32, Weight>"/>
+	// </Server>
+	// -->
 	metasheetRegexp = regexp.MustCompile(fmt.Sprintf(`<!--\s+(<%v(>`+metasheetItemBlock+`</%v>|\s*/>)(.*\n)+?)-->`, book.MetasheetName, book.MetasheetName))
 }
 
 // TODO: options
-func NewXMLImporter(filename string, sheets []string, parser book.SheetParser) (*XMLImporter, error) {
-	newBook, err := parseXML(filename, sheets, parser)
+func NewXMLImporter(filename string, sheets []string, parser book.SheetParser, mode Mode) (*XMLImporter, error) {
+	newBook, err := parseXML(filename, sheets, parser, mode)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse xml:%s", filename)
 	}
@@ -76,7 +87,7 @@ func splitRawXML(rawXML string) (metasheet, content string) {
 
 // parseXML parse sheets in the XML file named `filename` and return a book with multiple sheets
 // in TABLEAU grammar which can be exported to protobuf by excel parser.
-func parseXML(filename string, sheetNames []string, parser book.SheetParser) (*book.Book, error) {
+func parseXML(filename string, sheetNames []string, parser book.SheetParser, mode Mode) (*book.Book, error) {
 	log.Debugf("xml: %s", filename)
 	buf, err := os.ReadFile(filename)
 	if err != nil {
@@ -92,7 +103,7 @@ func parseXML(filename string, sheetNames []string, parser book.SheetParser) (*b
 	// The first pass
 	bookName := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
 	newBook := book.NewBook(bookName, filename, parser)
-	xmlMeta, err := readXMLFile(metasheet, content, newBook)
+	xmlMeta, err := readXMLFile(metasheet, content, newBook, mode)
 	if err != nil {
 		return nil, err
 	}
@@ -130,7 +141,7 @@ func parseXML(filename string, sheetNames []string, parser book.SheetParser) (*b
 // structure defined in xml.proto and put it into memory.
 //
 // readXMLFile read the raw xml rooted at `root`, specify which sheets to parse and return a XMLBook.
-func readXMLFile(metasheet, content string, newBook *book.Book) (*tableaupb.XMLBook, error) {
+func readXMLFile(metasheet, content string, newBook *book.Book, mode Mode) (*tableaupb.XMLBook, error) {
 	xmlMeta := &tableaupb.XMLBook{
 		SheetMap: make(map[string]int32),
 	}
@@ -161,16 +172,18 @@ func readXMLFile(metasheet, content string, newBook *book.Book) (*tableaupb.XMLB
 		}
 	}
 
-	// strip template sheets
-	for sheet, colMap := range metasheetMap {
-		if template, ok := colMap["Template"]; !ok || template != "true" {
-			continue
-		}
-		matches := matchSheetBlock(content, sheet)
-		if len(matches) == 0 {
-			continue
+	if mode == Protogen {
+		// strip template sheets
+		for sheet, colMap := range metasheetMap {
+			if template, ok := colMap["Template"]; !ok || template != "true" {
+				continue
+			}
+			matches := matchSheetBlock(content, sheet)
+			if len(matches) == 0 {
+				continue
+			}
+			content = strings.ReplaceAll(content, matches[0], "")
 		}
-		content = strings.ReplaceAll(content, matches[0], "")
 	}
 
 	// parse data content
diff --git a/internal/protogen/protogen.go b/internal/protogen/protogen.go
index 06c8866..b355b9b 100644
--- a/internal/protogen/protogen.go
+++ b/internal/protogen/protogen.go
@@ -268,7 +268,7 @@ func (gen *Generator) convertWithErrorModule(dir, filename string, checkProtoFil
 func (gen *Generator) convert(dir, filename string, checkProtoFileConflicts bool) (err error) {
 	absPath := filepath.Join(dir, filename)
 	parser := confgen.NewSheetParser(TableauProtoPackage, gen.LocationName, book.MetasheetOptions())
-	imp, err := importer.New(absPath, importer.Parser(parser), importer.TopN(defaultTopN))
+	imp, err := importer.New(absPath, importer.Parser(parser), importer.TopN(defaultTopN), importer.ImporterMode(importer.Protogen))
 	if err != nil {
 		return xerrors.WrapKV(err, xerrors.KeyBookName, absPath)
 	}


Test patch:
diff --git a/internal/importer/xml_test.go b/internal/importer/xml_test.go
index cc81477..f1e51d4 100644
--- a/internal/importer/xml_test.go
+++ b/internal/importer/xml_test.go
@@ -25,7 +25,7 @@ func Test_escapeAttrs(t *testing.T) {
 			args: args{
 				doc: `
 <Conf>
-    <Server Type="map<enum<.ServerType>, Server>" Value="int32"/>
+    <Server Type = "map<enum<.ServerType>, Server>" Value = "int32"/>
 </Conf>
 `,
 			},
@@ -107,7 +107,7 @@ func Test_isRepeated(t *testing.T) {
 `
 	metasheet, content := splitRawXML(doc)
 	newBook := book.NewBook(`Test.xml`, `Test.xml`, nil)
-	xmlMeta, _ := readXMLFile(metasheet, content, newBook)
+	xmlMeta, _ := readXMLFile(metasheet, content, newBook, Protogen)
 	sheet1 := getXMLSheet(xmlMeta, "MatchCfg")
 	node1 := FindMetaNode(sheet1, "MatchCfg/TeamRatingWeight/Weight")
 	node2 := FindMetaNode(sheet1, "MatchCfg/TeamRatingWeight/Weight/Param")
@@ -176,10 +176,10 @@ func Test_matchAttr(t *testing.T) {
 		{
 			name: "scalar type",
 			args: args{
-				s: `<AAA bb="bool" cc="int64" dd="enum<.EnumType>" >`,
+				s: `<AAA bb = "bool" cc = "int64" dd = "enum<.EnumType>" >`,
 			},
 			want: []string{
-				`bb="bool"`, `bb`, `bool`, ``,
+				`bb = "bool"`, `bb`, `bool`, ``,
 			},
 		},
 		{
@@ -219,7 +219,7 @@ func Test_isFirstChild(t *testing.T) {
 `
 	metasheet, content := splitRawXML(doc)
 	newBook := book.NewBook(`Test.xml`, `Test.xml`, nil)
-	xmlMeta, _ := readXMLFile(metasheet, content, newBook)
+	xmlMeta, _ := readXMLFile(metasheet, content, newBook, Protogen)
 	sheet1 := getXMLSheet(xmlMeta, "Server")
 	node1 := FindMetaNode(sheet1, "Server/MapConf/Weight")
 	type args struct {
@@ -303,7 +303,7 @@ func Test_fixNodeType(t *testing.T) {
 `
 	metasheet, content := splitRawXML(doc)
 	newBook := book.NewBook(`Test.xml`, `Test.xml`, nil)
-	xmlMeta, _ := readXMLFile(metasheet, content, newBook)
+	xmlMeta, _ := readXMLFile(metasheet, content, newBook, Protogen)
 	sheet1 := getXMLSheet(xmlMeta, "MatchCfg")
 	node1 := FindMetaNode(sheet1, "MatchCfg/MatchMode/MatchAI/AI")
 	node2 := FindMetaNode(sheet1, "MatchCfg/MapConf/Test/Weight")
@@ -721,7 +721,7 @@ func Test_readXMLFile(t *testing.T) {
 
 	type args struct {
 		metasheet, content string
-		newBook *book.Book
+		newBook            *book.Book
 	}
 	tests := []struct {
 		name    string
@@ -777,7 +777,7 @@ func Test_readXMLFile(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := readXMLFile(tt.args.metasheet, tt.args.content, tt.args.newBook)
+			got, err := readXMLFile(tt.args.metasheet, tt.args.content, tt.args.newBook, Protogen)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("readXMLFile() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -1076,7 +1076,7 @@ func Test_matchSheetBlock(t *testing.T) {
 <Client>
 	<Weight Num="1"/>
 </Client>`
-	
+
 	type args struct {
 		xml       string
 		sheetName string
@@ -1089,11 +1089,11 @@ func Test_matchSheetBlock(t *testing.T) {
 		{
 			name: "General",
 			args: args{
-				xml: doc,
+				xml:       doc,
 				sheetName: "Server",
 			},
 			want: []string{
-`<Server>
+				`<Server>
 	<Weight Num="1"/>
 	{{ if a == 1 }}
 	<Weight Num="2">
@@ -1103,7 +1103,7 @@ func Test_matchSheetBlock(t *testing.T) {
 		<Param value="1" />
 	</Weight>
 </Server>`,
-`>
+				`>
 	<Weight Num="1"/>
 	{{ if a == 1 }}
 	<Weight Num="2">
@@ -1113,7 +1113,7 @@ func Test_matchSheetBlock(t *testing.T) {
 		<Param value="1" />
 	</Weight>
 </Server>`,
-`	</Weight>
+				`	</Weight>
 `,
 			},
 		},
diff --git a/test/functest/conf/MetasheetConf.json b/test/functest/conf/MetasheetConf.json
deleted file mode 100644
index ec74a67..0000000
--- a/test/functest/conf/MetasheetConf.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-    "maxNum":  100,
-    "sepConf":  {
-        "numberList":  [
-            1,
-            2,
-            3
-        ]
-    }
-}
\ No newline at end of file
diff --git a/test/functest/conf/TemplateConf.json b/test/functest/conf/TemplateConf.json
deleted file mode 100644
index 4b3c1b7..0000000
--- a/test/functest/conf/TemplateConf.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "outputType":  null
-}
\ No newline at end of file
diff --git a/test/functest/util.go b/test/functest/util.go
index abc3832..0d01243 100644
--- a/test/functest/util.go
+++ b/test/functest/util.go
@@ -104,6 +104,9 @@ func genConf(t *testing.T) {
 						format.CSV,
 						format.XML,
 					},
+					ExcludedProtoFiles: []string{
+						"./_proto/xml__metasheet__metasheet.proto",
+					},
 				},
 				Output: &options.ConfOutputOption{
 					Pretty:          true,