Repository: milvus-io/milvus-sdk-go
Pull request #604 (2023-10-24 11:43:42)

Add wildcard outputfield expansion logic back
See also #596
/kind bug

sre-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/604#" title="Author self-approved">congqixia</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=milvus-io%2Fmilvus-sdk-go).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/milvus-io/milvus-sdk-go/blob/master/OWNERS)~~ [congqixia]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/604?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) Report
> Merging [#604](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/604?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) (2d1d6e7) into [master](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/commit/647cbab091d5ee9825dcc9061acf5c2b817da27a?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) (647cbab) will **increase** coverage by `0.07%`.
> The diff coverage is `100.00%`.

[![Impacted file tree graph](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/604/graphs/tree.svg?width=650&height=150&src=pr&token=WJK43PV390&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io)](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/604?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io)

```diff
@@            Coverage Diff             @@
##           master     #604      +/-   ##
==========================================
+ Coverage   82.85%   82.93%   +0.07%     
==========================================
  Files          40       40              
  Lines        3640     3656      +16     
==========================================
+ Hits         3016     3032      +16     
  Misses        512      512              
  Partials      112      112              
```


| [Files](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/604?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) | Coverage Δ | |
|---|---|---|
| [client/data.go](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/604?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io#diff-Y2xpZW50L2RhdGEuZ28=) | `70.39% <100.00%> (+1.38%)` | :arrow_up: |


Patch:
diff --git a/client/data.go b/client/data.go
index 4d18ed4..106f53f 100644
--- a/client/data.go
+++ b/client/data.go
@@ -92,7 +92,8 @@ func (c *GrpcClient) Search(ctx context.Context, collName string, partitions []s
 	return sr, nil
 }
 
-func (c *GrpcClient) parseSearchResult(_ *entity.Schema, outputFields []string, fieldDataList []*schemapb.FieldData, _, from, to int) ([]entity.Column, error) {
+func (c *GrpcClient) parseSearchResult(sch *entity.Schema, outputFields []string, fieldDataList []*schemapb.FieldData, _, from, to int) ([]entity.Column, error) {
+	outputFields = expandWildcard(sch, outputFields)
 	// duplicated name will have only one column now
 	outputSet := make(map[string]struct{})
 	for _, output := range outputFields {
@@ -141,6 +142,37 @@ func (c *GrpcClient) parseSearchResult(_ *entity.Schema, outputFields []string,
 	return columns, nil
 }
 
+func expandWildcard(schema *entity.Schema, outputFields []string) []string {
+	wildcard := false
+	for _, outputField := range outputFields {
+		if outputField == "*" {
+			wildcard = true
+		}
+	}
+	if !wildcard {
+		return outputFields
+	}
+
+	set := make(map[string]struct{})
+	result := make([]string, 0, len(schema.Fields))
+	for _, field := range schema.Fields {
+		result = append(result, field.Name)
+		set[field.Name] = struct{}{}
+	}
+
+	// add dynamic fields output
+	for _, output := range outputFields {
+		if output == "*" {
+			continue
+		}
+		_, ok := set[output]
+		if !ok {
+			result = append(result, output)
+		}
+	}
+	return result
+}
+
 func PKs2Expr(backName string, ids entity.Column) string {
 	var expr string
 	var pkName = ids.Name()


Test patch:
diff --git a/client/data_test.go b/client/data_test.go
index 0e787b6..aff3bb9 100644
--- a/client/data_test.go
+++ b/client/data_test.go
@@ -1178,3 +1178,42 @@ func TestVector2PlaceHolder(t *testing.T) {
 		}
 	})
 }
+
+type WildcardSuite struct {
+	suite.Suite
+
+	schema *entity.Schema
+}
+
+func (s *WildcardSuite) SetupTest() {
+	s.schema = entity.NewSchema().
+		WithField(entity.NewField().WithName("pk").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
+		WithField(entity.NewField().WithName("attr").WithDataType(entity.FieldTypeInt64)).
+		WithField(entity.NewField().WithName("$meta").WithDataType(entity.FieldTypeJSON).WithIsDynamic(true)).
+		WithField(entity.NewField().WithName("vector").WithDataType(entity.FieldTypeFloatVector).WithDim(128))
+}
+
+func (s *WildcardSuite) TestExpandWildcard() {
+	type testCase struct {
+		tag    string
+		input  []string
+		expect []string
+	}
+
+	cases := []testCase{
+		{tag: "normal", input: []string{"pk", "attr"}, expect: []string{"pk", "attr"}},
+		{tag: "with_wildcard", input: []string{"*"}, expect: []string{"pk", "attr", "$meta", "vector"}},
+		{tag: "wildcard_dynamic", input: []string{"*", "a"}, expect: []string{"pk", "attr", "$meta", "vector", "a"}},
+	}
+
+	for _, tc := range cases {
+		s.Run(tc.tag, func() {
+			output := expandWildcard(s.schema, tc.input)
+			s.ElementsMatch(tc.expect, output)
+		})
+	}
+}
+
+func TestExpandWildcard(t *testing.T) {
+	suite.Run(t, new(WildcardSuite))
+}
diff --git a/test/testcases/search_test.go b/test/testcases/search_test.go
index ffaddc2..7f4ea91 100644
--- a/test/testcases/search_test.go
+++ b/test/testcases/search_test.go
@@ -1099,8 +1099,7 @@ func TestSearchScannAllMetricsWithRawData(t *testing.T) {
 				sp,
 			)
 			common.CheckErr(t, errSearch, true)
-			// TODO output_fields include * ???
-			common.CheckOutputFields(t, resSearch[0].Fields, []string{"*", common.DefaultIntFieldName, common.DefaultFloatFieldName,
+			common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName,
 				common.DefaultJSONFieldName, common.DefaultFloatVecFieldName})
 			common.CheckSearchResult(t, resSearch, 1, common.DefaultTopK)
 		}
@@ -1148,8 +1147,7 @@ func TestRangeSearchScannL2(t *testing.T) {
 	// verify error nil, output all fields, range score
 	common.CheckErr(t, errSearch, true)
 	common.CheckSearchResult(t, resSearch, 1, common.DefaultTopK)
-	// TODO output_fields include * ??? https://github.com/milvus-io/milvus-sdk-go/issues/596
-	common.CheckOutputFields(t, resSearch[0].Fields, []string{"*", common.DefaultIntFieldName, common.DefaultFloatFieldName,
+	common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName,
 		common.DefaultJSONFieldName, common.DefaultFloatVecFieldName})
 	for _, s := range resSearch[0].Scores {
 		require.GreaterOrEqual(t, s, float32(15.0))
@@ -1208,8 +1206,7 @@ func TestRangeSearchScannIPCosine(t *testing.T) {
 		// verify error nil, output all fields, range score
 		common.CheckErr(t, errSearch, true)
 		common.CheckSearchResult(t, resSearch, 1, common.DefaultTopK)
-		// TODO output_fields include * ??? https://github.com/milvus-io/milvus-sdk-go/issues/596
-		common.CheckOutputFields(t, resSearch[0].Fields, []string{"*", common.DefaultIntFieldName, common.DefaultFloatFieldName,
+		common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName,
 			common.DefaultJSONFieldName, common.DefaultFloatVecFieldName})
 		for _, s := range resSearch[0].Scores {
 			require.GreaterOrEqual(t, s, float32(0))
@@ -1270,8 +1267,7 @@ func TestRangeSearchScannBinary(t *testing.T) {
 		// verify error nil, output all fields, range score
 		common.CheckErr(t, errSearch, true)
 		common.CheckSearchResult(t, resSearch, 1, common.DefaultTopK)
-		// TODO output_fields include * ??? https://github.com/milvus-io/milvus-sdk-go/issues/596
-		common.CheckOutputFields(t, resSearch[0].Fields, []string{"*", common.DefaultIntFieldName, common.DefaultFloatFieldName, common.DefaultBinaryVecFieldName})
+		common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, common.DefaultBinaryVecFieldName})
 		for _, s := range resSearch[0].Scores {
 			require.GreaterOrEqual(t, s, float32(0))
 			require.Less(t, s, float32(100))