Repository: tableauio/tableau
Pull request #132 (2024-08-23 09:25:08)

protogen(xml): improve XML importer v2.1
1. add support for scalar/enum list and predefined struct list
```xml
<?xml version="1.0" encoding="UTF-8"?>
<!--
<@TABLEAU>
    <Item Sheet="ItemConf" />
</@TABLEAU>

<ItemConf>
    <Value>[int32]</Value>
    <Fruit>[enum<.FruitType>]</Fruit>
    <Item @type="[.Item]"/>
</ItemConf>
-->

<ItemConf>
    <Value>1</Value>
    <Fruit>FRUIT_TYPE_APPLE</Fruit>
    <Fruit>FRUIT_TYPE_ORANGE</Fruit>
    <Fruit>FRUIT_TYPE_BANANA</Fruit>
    <Item ID="1" Num="10"/>
    <Item ID="2" Num="20"/>
</ItemConf>
```

2. for all kinds of lists and maps, auto add the `list` or `map` suffix on generated field name, so you don't need to manually add the  `list` or `map` suffix, or use plurals when naming a xml node or attribute

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/tableauio/tableau/pull/132?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) Report
Attention: Patch coverage is `71.87500%` with `18 lines` in your changes missing coverage. Please review.
> Project coverage is 55.11%. Comparing base [(`4aca14f`)](https://app.codecov.io/gh/tableauio/tableau/commit/4aca14f13c41ee9307ba847fadd6f39e324dc097?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) to head [(`3cdc27f`)](https://app.codecov.io/gh/tableauio/tableau/commit/3cdc27f25806ac96496ad31f4a8305f4d7203d40?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio).

| [Files](https://app.codecov.io/gh/tableauio/tableau/pull/132?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) | Patch % | Lines |
|---|---|---|
| [internal/confgen/document\_parser.go](https://app.codecov.io/gh/tableauio/tableau/pull/132?src=pr&el=tree&filepath=internal%2Fconfgen%2Fdocument_parser.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-aW50ZXJuYWwvY29uZmdlbi9kb2N1bWVudF9wYXJzZXIuZ28=) | 60.60% | [9 Missing and 4 partials :warning: ](https://app.codecov.io/gh/tableauio/tableau/pull/132?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) |
| [internal/importer/xml.go](https://app.codecov.io/gh/tableauio/tableau/pull/132?src=pr&el=tree&filepath=internal%2Fimporter%2Fxml.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-aW50ZXJuYWwvaW1wb3J0ZXIveG1sLmdv) | 86.95% | [2 Missing and 1 partial :warning: ](https://app.codecov.io/gh/tableauio/tableau/pull/132?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) |
| [internal/importer/book/node.go](https://app.codecov.io/gh/tableauio/tableau/pull/132?src=pr&el=tree&filepath=internal%2Fimporter%2Fbook%2Fnode.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio#diff-aW50ZXJuYWwvaW1wb3J0ZXIvYm9vay9ub2RlLmdv) | 75.00% | [1 Missing and 1 partial :warning: ](https://app.codecov.io/gh/tableauio/tableau/pull/132?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #132      +/-   ##
==========================================
- Coverage   55.13%   55.11%   -0.02%     
==========================================
  Files          74       74              
  Lines        9185     9231      +46     
==========================================
+ Hits         5064     5088      +24     
- Misses       3566     3581      +15     
- Partials      555      562       +7     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/tableauio/tableau/pull/132?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tableauio).


Patch:
diff --git a/internal/confgen/document_parser.go b/internal/confgen/document_parser.go
index 4c3dae9..e4e23b4 100644
--- a/internal/confgen/document_parser.go
+++ b/internal/confgen/document_parser.go
@@ -302,12 +302,15 @@ func (sp *documentParser) parseListField(field *Field, msg protoreflect.Message,
 	// Mutable returns a mutable reference to a composite type.
 	newValue := msg.Mutable(field.fd)
 	list := newValue.List()
-	if field.opts.Layout == tableaupb.Layout_LAYOUT_INCELL {
+	switch {
+	case field.opts.Layout == tableaupb.Layout_LAYOUT_INCELL,
+		// node of XML scalar list with only 1 element is just like an incell list
+		node.Kind == book.ScalarNode:
 		present, err = sp.parser.parseIncellListField(field, list, node.Value)
 		if err != nil {
 			return false, xerrors.WithMessageKV(err, node.DebugKV()...)
 		}
-	} else {
+	default:
 		for _, elemNode := range node.Children {
 			elemPresent := false
 			newListValue := list.NewElement()
diff --git a/internal/importer/xml.go b/internal/importer/xml.go
index 39f1c4d..6475728 100644
--- a/internal/importer/xml.go
+++ b/internal/importer/xml.go
@@ -212,13 +212,21 @@ func parseXMLNode(node *xmldom.Node, bnode *book.Node, mode ImporterMode) error
 		bnode.Kind = book.MapNode
 		bnode.Name = node.Name
 		if typeAttr := node.GetAttribute(atTypeDisplacement); typeAttr != nil {
-			fmt.Println(node.Name, typeAttr.Value)
 			// predefined struct
 			if len(node.Attributes) != 1 || len(node.Children) != 0 || node.Text != "" {
 				return errors.Errorf("predefined struct should not have children, text, or other attributes|name: %s", node.Name)
 			}
-			bnode.Kind = book.ScalarNode
-			bnode.Value = typeAttr.Value
+			bnode.Kind = book.MapNode
+			bnode.Children = append(bnode.Children, &book.Node{
+				Name:  book.KeywordType,
+				Value: typeAttr.Value,
+			})
+			if desc := types.MatchList(typeAttr.Value); desc != nil {
+				bnode.Children = append(bnode.Children, &book.Node{
+					Name:  book.KeywordVariable,
+					Value: fmt.Sprintf("%sList", bnode.Name),
+				})
+			}
 			return nil
 		}
 		// NOTE: curBNode may be pointed to one subnode when needed
@@ -283,21 +291,36 @@ func parseXMLNode(node *xmldom.Node, bnode *book.Node, mode ImporterMode) error
 				return errors.Wrapf(err, "parse xml node failed")
 			}
 			if subNode.Kind == book.ScalarNode {
-				bnode.Children = append(bnode.Children, subNode)
-			} else if existingBnode := bnode.FindChild(subNode.Name); existingBnode == nil {
-				bnode.Children = append(bnode.Children, &book.Node{
-					Kind: book.ListNode,
-					Name: subNode.Name,
-					Children: []*book.Node{
-						{
-							Kind:     book.MapNode,
-							Children: subNode.Children,
-						},
-					},
-				})
+				if existingBnode := bnode.FindChild(subNode.Name); existingBnode == nil {
+					bnode.Children = append(bnode.Children, subNode)
+				} else if existingBnode.Kind == book.ScalarNode {
+					existingBnode.Kind = book.ListNode
+					subNode.Name = ""
+					existingBnode.Children = append(existingBnode.Children, &book.Node{
+						Kind:  book.ScalarNode,
+						Value: existingBnode.Value,
+					}, subNode)
+					existingBnode.Value = ""
+				} else {
+					subNode.Name = ""
+					existingBnode.Children = append(existingBnode.Children, subNode)
+				}
 			} else {
-				subNode.Name = ""
-				existingBnode.Children = append(existingBnode.Children, subNode)
+				if existingBnode := bnode.FindChild(subNode.Name); existingBnode == nil {
+					bnode.Children = append(bnode.Children, &book.Node{
+						Kind: book.ListNode,
+						Name: subNode.Name,
+						Children: []*book.Node{
+							{
+								Kind:     book.MapNode,
+								Children: subNode.Children,
+							},
+						},
+					})
+				} else {
+					subNode.Name = ""
+					existingBnode.Children = append(existingBnode.Children, subNode)
+				}
 			}
 		}
 	}
@@ -324,6 +347,10 @@ func parseXMLAttribute(bnode *book.Node, attrName, attrValue string, isFirstAttr
 									Name:  book.KeywordType,
 									Value: attrValue,
 								},
+								{
+									Name:  book.KeywordVariable,
+									Value: fmt.Sprintf("%sMap", attrName),
+								},
 								{
 									Name:  book.KeywordIncell,
 									Value: "true",
@@ -346,6 +373,10 @@ func parseXMLAttribute(bnode *book.Node, attrName, attrValue string, isFirstAttr
 						Name:  book.KeywordType,
 						Value: attrValue,
 					},
+					{
+						Name:  book.KeywordVariable,
+						Value: fmt.Sprintf("%sMap", attrName),
+					},
 					{
 						Name:  book.KeywordIncell,
 						Value: "true",
@@ -382,7 +413,7 @@ func parseXMLAttribute(bnode *book.Node, attrName, attrValue string, isFirstAttr
 			return curBNode, nil
 		}
 	} else if desc := types.MatchList(attrValue); desc != nil {
-		if desc.ElemType != "" {
+		if desc.ElemType != "" && desc.ColumnType != "" {
 			// struct list
 			curBNode = &book.Node{
 				Kind: book.MapNode,
@@ -407,6 +438,41 @@ func parseXMLAttribute(bnode *book.Node, attrName, attrValue string, isFirstAttr
 				bnode.Children = append(bnode.Children, curBNode.Children[0])
 				return bnode, nil
 			}
+		} else if desc.ElemType != "" {
+			// scalar or enum list
+			curBNode = &book.Node{
+				Kind: book.MapNode,
+				Name: book.KeywordStruct,
+				Children: []*book.Node{
+					{
+						Kind: book.MapNode,
+						Name: attrName,
+						Children: []*book.Node{
+							{
+								Name:  book.KeywordType,
+								Value: attrValue,
+							},
+							{
+								Name:  book.KeywordVariable,
+								Value: fmt.Sprintf("%sList", attrName),
+							},
+						},
+					},
+				},
+			}
+			if isFirstAttr {
+				bnode.Children = append(bnode.Children, &book.Node{
+					Name:  book.KeywordType,
+					Value: fmt.Sprintf("[%s]", bnode.Name),
+				}, &book.Node{
+					Name:  book.KeywordVariable,
+					Value: fmt.Sprintf("%sList", bnode.Name),
+				}, curBNode)
+				return curBNode, nil
+			} else {
+				bnode.Children = append(bnode.Children, curBNode.Children[0])
+				return bnode, nil
+			}
 		} else {
 			// incell list
 			curBNode = &book.Node{


Test patch:
diff --git a/internal/importer/testdata/Test.xml b/internal/importer/testdata/Test.xml
index e1216b4..8ea6b56 100644
--- a/internal/importer/testdata/Test.xml
+++ b/internal/importer/testdata/Test.xml
@@ -12,5 +12,7 @@
     <CrossCell Value="[CrossCell]int32" Name="string" Cost="[]int32" />
     <Incell Value="[]int32" Cost="[]int32" />
     <EnumIncell>[]enum<.FruitType></EnumIncell>
+    <ScalarList>[int32]</ScalarList>
+    <Predefined @type="[.Item]"/>
 </Item>
 -->
diff --git a/test/functest/conf/ListConf.json b/test/functest/conf/ListConf.json
index 9756c49..986cf80 100644
--- a/test/functest/conf/ListConf.json
+++ b/test/functest/conf/ListConf.json
@@ -41,5 +41,18 @@
                 }
             ]
         }
-    }
+    },
+    "valueList": [
+        1
+    ],
+    "itemList": [
+        {
+            "id": 1,
+            "num": 10
+        },
+        {
+            "id": 2,
+            "num": 20
+        }
+    ]
 }
\ No newline at end of file
diff --git a/test/functest/conf/ListConf2.json b/test/functest/conf/ListConf2.json
index c1fa07a..3666d29 100644
--- a/test/functest/conf/ListConf2.json
+++ b/test/functest/conf/ListConf2.json
@@ -21,5 +21,10 @@
                 "FRUIT_TYPE_BANANA"
             ]
         }
-    }
+    },
+    "fruitList": [
+        "FRUIT_TYPE_APPLE",
+        "FRUIT_TYPE_ORANGE",
+        "FRUIT_TYPE_BANANA"
+    ]
 }
\ No newline at end of file
diff --git a/test/functest/conf/MapConf.json b/test/functest/conf/MapConf.json
index 749bc1e..d87d9b2 100644
--- a/test/functest/conf/MapConf.json
+++ b/test/functest/conf/MapConf.json
@@ -40,8 +40,8 @@
             }
         }
     },
-    "incell": {
-        "map": {
+    "friends": {
+        "friendMap": {
             "Jerry": 2,
             "Tom": 1
         }
diff --git a/test/functest/proto/xml__list__list.proto b/test/functest/proto/xml__list__list.proto
index 98aa98c..3283d1f 100644
--- a/test/functest/proto/xml__list__list.proto
+++ b/test/functest/proto/xml__list__list.proto
@@ -26,6 +26,8 @@ message ListConf {
       }
     }
   }
+  repeated int32 value_list = 2 [(tableau.field) = {name:"Value"}];
+  repeated protoconf.Item item_list = 3 [(tableau.field) = {name:"Item"}];
 }
 
 message ListConf2 {
@@ -43,4 +45,5 @@ message ListConf2 {
       repeated protoconf.FruitType enum_incell_list = 2 [(tableau.field) = {name:"EnumIncell" layout:LAYOUT_INCELL}];
     }
   }
+  repeated protoconf.FruitType fruit_list = 2 [(tableau.field) = {name:"Fruit"}];
 }
diff --git a/test/functest/proto/xml__map__map.proto b/test/functest/proto/xml__map__map.proto
index ab92025..d8f5a42 100644
--- a/test/functest/proto/xml__map__map.proto
+++ b/test/functest/proto/xml__map__map.proto
@@ -25,21 +25,21 @@ message MapConf {
   message Entry {
     uint32 id = 1 [(tableau.field) = {name:"Id"}];
     int32 value = 2 [(tableau.field) = {name:"Value" prop:{default:"1"}}];
-    map<int32, string> incell_map = 3 [(tableau.field) = {name:"IncellMap" layout:LAYOUT_INCELL}];
+    map<int32, string> incell_map = 3 [(tableau.field) = {name:"Incell" layout:LAYOUT_INCELL}];
   }
-  Incell incell = 4 [(tableau.field) = {name:"Incell"}];
-  message Incell {
-    map<string, int32> map = 1 [(tableau.field) = {name:"Map" layout:LAYOUT_INCELL}];
+  Friends friends = 4 [(tableau.field) = {name:"Friends"}];
+  message Friends {
+    map<string, int32> friend_map = 1 [(tableau.field) = {name:"Friend" layout:LAYOUT_INCELL}];
   }
-  map<int32, int32> incell_map = 5 [(tableau.field) = {name:"IncellMap" layout:LAYOUT_INCELL}];
-  map<int32, KeyEnumMapValue> key_enum_map = 6 [(tableau.field) = {name:"KeyEnumMap" key:"@key" layout:LAYOUT_INCELL span:SPAN_INNER_CELL}];
-  message KeyEnumMapValue {
+  map<int32, int32> incell_map = 5 [(tableau.field) = {name:"Incell" layout:LAYOUT_INCELL}];
+  map<int32, KeyEnumValue> key_enum_map = 6 [(tableau.field) = {name:"KeyEnum" key:"@key" layout:LAYOUT_INCELL span:SPAN_INNER_CELL}];
+  message KeyEnumValue {
     protoconf.FruitType key = 1 [(tableau.field) = {name:"@key"}];
     int32 value = 2 [(tableau.field) = {name:"@value"}];
   }
-  map<int32, protoconf.FruitType> value_enum_map = 7 [(tableau.field) = {name:"ValueEnumMap" layout:LAYOUT_INCELL}];
-  map<int32, KvEnumMapValue> kv_enum_map = 8 [(tableau.field) = {name:"KvEnumMap" key:"@key" layout:LAYOUT_INCELL span:SPAN_INNER_CELL}];
-  message KvEnumMapValue {
+  map<int32, protoconf.FruitType> value_enum_map = 7 [(tableau.field) = {name:"ValueEnum" layout:LAYOUT_INCELL}];
+  map<int32, KvEnumValue> kv_enum_map = 8 [(tableau.field) = {name:"KvEnum" key:"@key" layout:LAYOUT_INCELL span:SPAN_INNER_CELL}];
+  message KvEnumValue {
     protoconf.FruitType key = 1 [(tableau.field) = {name:"@key"}];
     protoconf.FruitType value = 2 [(tableau.field) = {name:"@value"}];
   }
diff --git a/test/functest/testdata/xml/list/List.xml b/test/functest/testdata/xml/list/List.xml
index 0cdcfee..141506e 100644
--- a/test/functest/testdata/xml/list/List.xml
+++ b/test/functest/testdata/xml/list/List.xml
@@ -11,6 +11,8 @@
             <CrossCell Value="[CrossCell]int32" Name="string" Cost="[]int32" />
         </Parent2>
     </Parent>
+    <Value>[int32]</Value>
+    <Item @type="[.Item]"/>
 </ListConf>
 
 <ListConf2>
@@ -20,6 +22,7 @@
             <EnumIncell>[]enum<.FruitType></EnumIncell>
         </Parent2>
     </Parent>
+    <Fruit>[enum<.FruitType>]</Fruit>
 </ListConf2>
 -->
 
@@ -32,6 +35,9 @@
             <CrossCell Value="1" Name="Grape" Cost="10,11,12,13" />
         </Parent2>
     </Parent>
+    <Value>1</Value>
+    <Item ID="1" Num="10"/>
+    <Item ID="2" Num="20"/>
 </ListConf>
 
 <ListConf2>
@@ -41,4 +47,7 @@
             <EnumIncell>Apple, Orange, Banana</EnumIncell>
         </Parent2>
     </Parent>
+    <Fruit>FRUIT_TYPE_APPLE</Fruit>
+    <Fruit>FRUIT_TYPE_ORANGE</Fruit>
+    <Fruit>FRUIT_TYPE_BANANA</Fruit>
 </ListConf2>
diff --git a/test/functest/testdata/xml/map/Map.xml b/test/functest/testdata/xml/map/Map.xml
index 8fcead4..dccf370 100644
--- a/test/functest/testdata/xml/map/Map.xml
+++ b/test/functest/testdata/xml/map/Map.xml
@@ -6,12 +6,12 @@
 
 <MapConf Open="bool">
     <PreEntry Name="map<string, PreEntry>" Cost="[]int32" />
-    <Entry Id="map<uint32, Entry>" Value="int32|{default:"1"}" IncellMap="map<int32, string>" />
-    <Incell Map="map<string, int32>" />
-    <IncellMap>map<int32, int32></IncellMap>
-    <KeyEnumMap>map<enum<.FruitType>, int32></KeyEnumMap>
-    <ValueEnumMap>map<int32, enum<.FruitType>></ValueEnumMap>
-    <KvEnumMap>map<enum<.FruitType>, enum<.FruitType>></KvEnumMap>
+    <Entry Id="map<uint32, Entry>" Value="int32|{default:"1"}" Incell="map<int32, string>" />
+    <Friends Friend="map<string, int32>" />
+    <Incell>map<int32, int32></Incell>
+    <KeyEnum>map<enum<.FruitType>, int32></KeyEnum>
+    <ValueEnum>map<int32, enum<.FruitType>></ValueEnum>
+    <KvEnum>map<enum<.FruitType>, enum<.FruitType>></KvEnum>
 
     <Meal Id="map<string, Meal>" Time="duration|{default:"30m"}">
         <Meat Name="map<string, Meat>" Weight="string" />
@@ -28,14 +28,14 @@
 <MapConf Open="true">
     <PreEntry Name="1" Cost="1,2,3" />
     <PreEntry Name="2" Cost="4,5,6" />
-    <Entry Id="1" Value="1" IncellMap="1:first,2:second" />
-    <Entry Id="2" Value="2" IncellMap=""/>
-    <Entry Id="3" IncellMap="3:third" />
-    <Incell Map="Tom:1,Jerry:2" />
-    <IncellMap>1:1,2:10,3:100</IncellMap>
-    <KeyEnumMap>Apple:1,Banana:2</KeyEnumMap>
-    <ValueEnumMap>1:Banana,2:Orange</ValueEnumMap>
-    <KvEnumMap>Apple:Banana,Banana:Orange</KvEnumMap>
+    <Entry Id="1" Value="1" Incell="1:first,2:second" />
+    <Entry Id="2" Value="2" Incell=""/>
+    <Entry Id="3" Incell="3:third" />
+    <Friends Friend="Tom:1,Jerry:2" />
+    <Incell>1:1,2:10,3:100</Incell>
+    <KeyEnum>Apple:1,Banana:2</KeyEnum>
+    <ValueEnum>1:Banana,2:Orange</ValueEnum>
+    <KvEnum>Apple:Banana,Banana:Orange</KvEnum>
 
     <Meal Id="breakfast" Time="15m">
         <Meat Name="Egg" Weight="100g" />