Repository: tableauio/tableau
Pull request #227 (2025-04-21 13:38:25)

protogen: support multi modes in the first pass

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/tableauio/tableau/pull/227?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) Report
Attention: Patch coverage is `52.94118%` with `24 lines` in your changes missing coverage. Please review.
> Project coverage is 70.57%. Comparing base [(`c3c43b6`)](https://app.codecov.io/gh/tableauio/tableau/commit/c3c43b62b478a960f5ff1ddcad221b015d9b5d35?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) to head [(`77a68e0`)](https://app.codecov.io/gh/tableauio/tableau/commit/77a68e018bcdf0f25128da582143f289a6ff8e05?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio).

| [Files with missing lines](https://app.codecov.io/gh/tableauio/tableau/pull/227?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) | Patch % | Lines |
|---|---|---|
| [internal/protogen/protogen.go](https://app.codecov.io/gh/tableauio/tableau/pull/227?src=pr&el=tree&filepath=internal%2Fprotogen%2Fprotogen.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-aW50ZXJuYWwvcHJvdG9nZW4vcHJvdG9nZW4uZ28=) | 52.94% | [21 Missing and 3 partials :warning: ](https://app.codecov.io/gh/tableauio/tableau/pull/227?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #227      +/-   ##
==========================================
- Coverage   70.72%   70.57%   -0.16%     
==========================================
  Files          79       79              
  Lines       10194    10196       +2     
==========================================
- Hits         7210     7196      -14     
- Misses       2423     2440      +17     
+ Partials      561      560       -1     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/tableauio/tableau/pull/227?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>

Patch:
diff --git a/internal/protogen/protogen.go b/internal/protogen/protogen.go
index 6a265e6..897eba6 100644
--- a/internal/protogen/protogen.go
+++ b/internal/protogen/protogen.go
@@ -43,9 +43,8 @@ type Generator struct {
 	protofiles *protoregistry.Files // all parsed imported proto file descriptors.
 	typeInfos  *xproto.TypeInfos    // predefined type infos
 
-	cacheMu           sync.RWMutex                 // guard fields below
-	cachedImporters   map[string]importer.Importer // absolute file path -> importer
-	cachedBookParsers map[string]*tableParser      // absolute file path -> bookParser (only for tables currently)
+	cacheMu         sync.RWMutex                 // guard fields below
+	cachedImporters map[string]importer.Importer // absolute file path -> importer
 }
 
 func NewGenerator(protoPackage, indir, outdir string, setters ...options.Option) *Generator {
@@ -67,8 +66,7 @@ func NewGeneratorWithOptions(protoPackage, indir, outdir string, opts *options.O
 		protofiles: &protoregistry.Files{},
 		typeInfos:  xproto.NewTypeInfos(protoPackage),
 
-		cachedImporters:   make(map[string]importer.Importer),
-		cachedBookParsers: make(map[string]*tableParser),
+		cachedImporters: make(map[string]importer.Importer),
 	}
 
 	if opts.Proto.Input.MetasheetName != "" {
@@ -78,22 +76,36 @@ func NewGeneratorWithOptions(protoPackage, indir, outdir string, opts *options.O
 	return gen
 }
 
-func (gen *Generator) preprocess(delExisted bool) error {
+func (gen *Generator) preprocess(useGeneratedProtos, delExisted bool) error {
+	outdir := filepath.Join(gen.OutputDir, gen.OutputOpt.Subdir)
+	var protoFiles []string
+	protoFiles = append(protoFiles, gen.InputOpt.ProtoFiles...)
+	importProtos := make(map[string]bool)
+	for _, path := range gen.InputOpt.ProtoFiles {
+		importProtos[path] = true
+	}
+	if useGeneratedProtos {
+		files, err := os.ReadDir(outdir)
+		if err != nil {
+			return xerrors.WrapKV(err, xerrors.KeyOutdir, outdir)
+		}
+		for _, file := range files {
+			if strings.HasSuffix(file.Name(), ".proto") &&
+				!importProtos[file.Name()] {
+				protoFiles = append(protoFiles, file.Name())
+			}
+		}
+	}
 	// parse custom imported proto files
-	protofiles, err := xproto.ParseProtos(
+	protoRegistryFiles, err := xproto.ParseProtos(
 		gen.InputOpt.ProtoPaths,
-		gen.InputOpt.ProtoFiles...)
+		protoFiles...)
 	if err != nil {
 		return err
 	}
-	gen.protofiles = protofiles
-	gen.typeInfos = xproto.GetAllTypeInfo(protofiles, gen.ProtoPackage)
-
-	outputProtoDir := filepath.Join(gen.OutputDir, gen.OutputOpt.Subdir)
-	if err := prepareOutdir(outputProtoDir, gen.InputOpt.ProtoFiles, delExisted); err != nil {
-		return err
-	}
-	return nil
+	gen.protofiles = protoRegistryFiles
+	gen.typeInfos = xproto.GetAllTypeInfo(protoRegistryFiles, gen.ProtoPackage)
+	return prepareOutdir(outdir, gen.InputOpt.ProtoFiles, delExisted)
 }
 
 func (gen *Generator) Generate(relWorkbookPaths ...string) error {
@@ -104,11 +116,12 @@ func (gen *Generator) Generate(relWorkbookPaths ...string) error {
 }
 
 func (gen *Generator) GenAll() error {
-	if err := gen.preprocess(true); err != nil {
+	if err := gen.preprocess(false, true); err != nil {
 		return err
 	}
 	// first pass
-	if err := gen.processFirstPass(); err != nil {
+	log.Infof("%15s: parsing all books", "first-pass")
+	if err := gen.processFirstPass(true); err != nil {
 		return err
 	}
 	// second pass
@@ -116,16 +129,26 @@ func (gen *Generator) GenAll() error {
 }
 
 func (gen *Generator) GenWorkbook(relWorkbookPaths ...string) error {
-	if err := gen.preprocess(false); err != nil {
-		return err
-	}
 	// first pass
-	if gen.InputOpt.FirstPassProcessAll {
-		if err := gen.processFirstPass(); err != nil {
+	switch gen.InputOpt.FirstPassMode {
+	case options.FirstPassModeNormal:
+		if err := gen.preprocess(false, false); err != nil {
+			return err
+		}
+		log.Infof("%15s: parsing all books", "first-pass")
+		if err := gen.processFirstPass(false); err != nil {
+			return err
+		}
+	case options.FirstPassModeAdvanced:
+		log.Infof("%15s: parsing previous generated proto files", "first-pass")
+		if err := gen.preprocess(true, false); err != nil {
 			return err
 		}
-	} else {
-		log.Infof("%15s: processing only specified books", "first-pass")
+	default:
+		if err := gen.preprocess(false, false); err != nil {
+			return err
+		}
+		log.Infof("%15s: parsing only specified books", "first-pass")
 		var eg errgroup.Group
 		for _, relWorkbookPath := range relWorkbookPaths {
 			absPath := filepath.Join(gen.InputDir, relWorkbookPath)
@@ -148,15 +171,14 @@ func (gen *Generator) GenWorkbook(relWorkbookPaths ...string) error {
 	return eg.Wait()
 }
 
-func (gen *Generator) processFirstPass() error {
+func (gen *Generator) processFirstPass(checkProtoFileConflicts bool) error {
 	// first pass
-	log.Infof("%15s: processing all books", "first-pass")
 	if len(gen.InputOpt.Subdirs) == 0 {
-		return gen.processDirFirstPass(gen.InputDir)
+		return gen.processDirFirstPass(gen.InputDir, checkProtoFileConflicts)
 	}
 	for _, subdir := range gen.InputOpt.Subdirs {
 		dir := filepath.Join(gen.InputDir, subdir)
-		if err := gen.processDirFirstPass(dir); err != nil {
+		if err := gen.processDirFirstPass(dir, checkProtoFileConflicts); err != nil {
 			return err
 		}
 	}
@@ -182,7 +204,7 @@ func (gen *Generator) processSecondPass() error {
 	return eg.Wait()
 }
 
-func (gen *Generator) processDirFirstPass(dir string) (err error) {
+func (gen *Generator) processDirFirstPass(dir string, checkProtoFileConflicts bool) (err error) {
 	var eg errgroup.Group
 	defer func() {
 		if err == nil {
@@ -201,7 +223,7 @@ func (gen *Generator) processDirFirstPass(dir string) (err error) {
 		if entry.IsDir() {
 			// scan and generate subdir recursively
 			subdir := filepath.Join(dir, entry.Name())
-			err = gen.processDirFirstPass(subdir)
+			err = gen.processDirFirstPass(subdir, checkProtoFileConflicts)
 			if err != nil {
 				return xerrors.WrapKV(err, xerrors.KeySubdir, subdir)
 			}
@@ -220,7 +242,7 @@ func (gen *Generator) processDirFirstPass(dir string) (err error) {
 				// is not a directory
 				log.Warnf("symlink: %s is not a directory, currently not processed", dstPath)
 			}
-			err = gen.processDirFirstPass(dstPath)
+			err = gen.processDirFirstPass(dstPath, checkProtoFileConflicts)
 			if err != nil {
 				return xerrors.WrapKV(err, xerrors.KeySubdir, dstPath)
 			}
@@ -252,7 +274,7 @@ func (gen *Generator) processDirFirstPass(dir string) (err error) {
 
 		filename := entry.Name()
 		eg.Go(func() error {
-			return gen.convertWithErrorModule(dir, filename, true, firstPass)
+			return gen.convertWithErrorModule(dir, filename, checkProtoFileConflicts, firstPass)
 		})
 	}
 	return nil
@@ -270,18 +292,6 @@ func (gen *Generator) getImporter(absPath string) importer.Importer {
 	return gen.cachedImporters[absPath]
 }
 
-func (gen *Generator) addBookParser(absPath string, parser *tableParser) {
-	gen.cacheMu.Lock()
-	defer gen.cacheMu.Unlock()
-	gen.cachedBookParsers[absPath] = parser
-}
-
-func (gen *Generator) getBookParser(absPath string) *tableParser {
-	gen.cacheMu.RLock()
-	defer gen.cacheMu.RUnlock()
-	return gen.cachedBookParsers[absPath]
-}
-
 func (gen *Generator) convertWithErrorModule(dir, filename string, checkProtoFileConflicts bool, pass parsePass) error {
 	fmt := format.GetFormat(filename)
 	if format.IsInputDocumentFormat(fmt) {
@@ -378,8 +388,8 @@ func (gen *Generator) convertDocument(dir, filename string, checkProtoFileConfli
 
 func (gen *Generator) convertTable(dir, filename string, checkProtoFileConflicts bool, pass parsePass) (err error) {
 	absPath := filepath.Join(dir, filename)
-	var imp importer.Importer
-	if pass == firstPass {
+	imp := gen.getImporter(absPath)
+	if imp == nil {
 		parser := confgen.NewSheetParser(xproto.InternalProtoPackage, gen.LocationName, gen.strcaseCtx, book.MetasheetOptions())
 		imp, err = importer.New(absPath, importer.Parser(parser), importer.Mode(importer.Protogen))
 		if err != nil {
@@ -390,8 +400,6 @@ func (gen *Generator) convertTable(dir, filename string, checkProtoFileConflicts
 		}
 		// cache this new importer
 		gen.addImporter(absPath, imp)
-	} else {
-		imp = gen.getImporter(absPath)
 	}
 
 	basename := filepath.Base(imp.Filename())
@@ -405,26 +413,18 @@ func (gen *Generator) convertTable(dir, filename string, checkProtoFileConflicts
 	if rewrittenBookName != relativePath {
 		debugBookName += " (rewrite: " + rewrittenBookName + ")"
 	}
-
+	// alias
+	bookOpts := imp.GetBookOptions()
+	bookName := imp.BookName()
+	alias := bookOpts.GetAlias()
+	if alias != "" {
+		debugBookName += " (alias: " + alias + ")"
+	}
 	if pass == secondPass {
 		log.Infof("%15s: %s, %d sheet(s) will be parsed", "analyzing book", debugBookName, len(imp.GetSheets()))
 	}
-	bookOpts := imp.GetBookOptions()
-	var bp *tableParser
-	if pass == firstPass {
-		// create a book parser
-		bookName := imp.BookName()
-		alias := bookOpts.GetAlias()
-		if alias != "" {
-			debugBookName += " (alias: " + alias + ")"
-		}
-		bp = newTableParser(bookName, alias, rewrittenBookName, gen)
-		// cache this new tableParser
-		gen.addBookParser(absPath, bp)
-	} else {
-		bp = gen.getBookParser(absPath)
-	}
-
+	// create a book parser
+	bp := newTableParser(bookName, alias, rewrittenBookName, gen)
 	for _, sheet := range imp.GetSheets() {
 		// parse sheet header
 		ws := sheet.ToWorkseet()
diff --git a/options/options.go b/options/options.go
index c7f9919..b6fc9b5 100644
--- a/options/options.go
+++ b/options/options.go
@@ -146,13 +146,18 @@ type ProtoInputOption struct {
 	// Default: "".
 	MetasheetName string `yaml:"metasheetName"`
 
-	// In the first-pass stage of protogen, whether to process all files when
-	// only generate some specified files. In the first-pass under the hood,
-	// all predefined types will be parsed and then can be recognized and used
-	// in the second-pass.
+	// Specify the first-pass mode to parse predefined types when generate
+	// specified config files. Under the hood, the parsed predefined types will
+	// be recognized and used in the second-pass.
 	//
-	// Default: false.
-	FirstPassProcessAll bool `yaml:"firstPassProcessAll"`
+	// The first-pass mode can be:
+	//
+	//	- "": default mode, parse based on specified config files.
+	//  - "normal": parse based on all config files.
+	//  - "advanced": parse based on all previous generated proto files.
+	//
+	// Default: "".
+	FirstPassMode FirstPassMode `yaml:"firstPassMode"`
 }
 
 // Output options for generating proto files.
@@ -304,12 +309,19 @@ type ConfOutputOption struct {
 	UseEnumNumbers bool `yaml:"useEnumNumbers"`
 
 	// Specify dry run mode:
-	//  - patch: if sheet options are specified: Patch (PATCH_MERGE) and Scatter
+	//  - "patch": if sheet options are specified: Patch (PATCH_MERGE) and Scatter
 	//
 	// Default: "".
 	DryRun DryRun `yaml:"dryRun"`
 }
 
+type FirstPassMode = string
+
+const (
+	FirstPassModeNormal   FirstPassMode = "normal"
+	FirstPassModeAdvanced FirstPassMode = "advanced"
+)
+
 type DryRun = string
 
 const (


Test patch:
diff --git a/internal/protogen/protogen_test.go b/internal/protogen/protogen_test.go
index 29abd6e..9e6b0a1 100644
--- a/internal/protogen/protogen_test.go
+++ b/internal/protogen/protogen_test.go
@@ -76,7 +76,7 @@ func TestGenerator_GenWorkbook(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			name: "test1",
+			name: "test1-FirstPassModeDefault",
 			gen: NewGenerator("protoconf", "./", outdir,
 				options.Proto(
 					&options.ProtoOption{
@@ -90,6 +90,7 @@ func TestGenerator_GenWorkbook(t *testing.T) {
 							Formats: []format.Format{
 								format.YAML,
 							},
+							FirstPassMode: "",
 						},
 						Output: &options.ProtoOutputOption{
 							FilenameWithSubdirPrefix: true,
@@ -100,7 +101,65 @@ func TestGenerator_GenWorkbook(t *testing.T) {
 					},
 				),
 			),
-			args:    args{relWorkbookPaths: []string{"./testdata/yaml/Test.yaml"}},
+			args:    args{relWorkbookPaths: []string{"./testdata/yaml/Test.yaml", "./testdata/csv/Unittest#*.csv"}},
+			wantErr: false,
+		},
+		{
+			name: "test2-FirstPassModeNormal",
+			gen: NewGenerator("protoconf", "./", outdir,
+				options.Proto(
+					&options.ProtoOption{
+						Input: &options.ProtoInputOption{
+							ProtoPaths: []string{outdir},
+							ProtoFiles: []string{
+								"common/base.proto",
+								"common/common.proto",
+								"common/union.proto",
+							},
+							Formats: []format.Format{
+								format.YAML,
+							},
+							FirstPassMode: options.FirstPassModeNormal,
+						},
+						Output: &options.ProtoOutputOption{
+							FilenameWithSubdirPrefix: true,
+							FileOptions: map[string]string{
+								"go_package": "github.com/tableauio/tableau/test/functest/protoconf",
+							},
+						},
+					},
+				),
+			),
+			args:    args{relWorkbookPaths: []string{"./testdata/yaml/Test.yaml", "./testdata/csv/Unittest#*.csv"}},
+			wantErr: false,
+		},
+		{
+			name: "test3-FirstPassModeAdvanced",
+			gen: NewGenerator("protoconf", "./", outdir,
+				options.Proto(
+					&options.ProtoOption{
+						Input: &options.ProtoInputOption{
+							ProtoPaths: []string{outdir},
+							ProtoFiles: []string{
+								"common/base.proto",
+								"common/common.proto",
+								"common/union.proto",
+							},
+							Formats: []format.Format{
+								format.YAML,
+							},
+							FirstPassMode: options.FirstPassModeAdvanced,
+						},
+						Output: &options.ProtoOutputOption{
+							FilenameWithSubdirPrefix: true,
+							FileOptions: map[string]string{
+								"go_package": "github.com/tableauio/tableau/test/functest/protoconf",
+							},
+						},
+					},
+				),
+			),
+			args:    args{relWorkbookPaths: []string{"./testdata/yaml/Test.yaml", "./testdata/csv/Unittest#*.csv"}},
 			wantErr: false,
 		},
 	}
diff --git a/internal/protogen/testdata/csv/Unittest#@TABLEAU.csv b/internal/protogen/testdata/csv/Unittest#@TABLEAU.csv
new file mode 100644
index 0000000..e69de29
diff --git a/internal/protogen/testdata/csv/Unittest#ItemConf.csv b/internal/protogen/testdata/csv/Unittest#ItemConf.csv
new file mode 100644
index 0000000..e286615
--- /dev/null
+++ b/internal/protogen/testdata/csv/Unittest#ItemConf.csv
@@ -0,0 +1,6 @@
+ID,Num
+"map<uint32, .Item>|{sequence:1}",int32
+Item's ID,Item's num
+1,100
+2,200
+3,300