Repository: milvus-io/milvus-sdk-go
Pull request #581 (2023-09-13 08:43:13)

ScaNN support with_raw_data build option
https://github.com/zilliztech/knowhere/issues/73

sre-ci-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/581#" title="Author self-approved">chasingegg</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **yah01** after the PR has been reviewed.
You can assign the PR to them by writing `/assign @yah01` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=milvus-io%2Fmilvus-sdk-go).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/milvus-io/milvus-sdk-go/blob/master/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["yah01"]} -->
chasingegg: /kind improvement
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/581?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) Report
> Merging [#581](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/581?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) (23b9502) into [master](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/commit/849300d6d8593f92c3ad48d2aa255b41400d9721?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) (849300d) will **decrease** coverage by `0.02%`.
> The diff coverage is `77.77%`.

[![Impacted file tree graph](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/581/graphs/tree.svg?width=650&height=150&src=pr&token=WJK43PV390&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io)](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/581?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io)

```diff
@@            Coverage Diff             @@
##           master     #581      +/-   ##
==========================================
- Coverage   82.89%   82.87%   -0.02%     
==========================================
  Files          40       40              
  Lines        3647     3656       +9     
==========================================
+ Hits         3023     3030       +7     
- Misses        511      512       +1     
- Partials      113      114       +1     
```


| [Files Changed](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/581?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) | Coverage Δ | |
|---|---|---|
| [entity/indexes\_gen.go](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/581?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io#diff-ZW50aXR5L2luZGV4ZXNfZ2VuLmdv) | `99.25% <77.77%> (-0.75%)` | :arrow_down: |

sre-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/581#" title="Author self-approved">chasingegg</a>*, *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/581#pullrequestreview-1628432567" title="LGTM">congqixia</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=milvus-io%2Fmilvus-sdk-go).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/milvus-io/milvus-sdk-go/blob/master/OWNERS)~~ [congqixia]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/entity/genidx/genidx.go b/entity/genidx/genidx.go
index 88e3f45..2f997be 100644
--- a/entity/genidx/genidx.go
+++ b/entity/genidx/genidx.go
@@ -716,6 +716,11 @@ func main() {
 						Name:           "nlist",
 						ValidationRule: "[1, 65536]",
 					},
+					{
+						Name:           "with_raw_data",
+						Type:           "bool",
+						ValidationRule: "in (false, true)",
+					},
 				},
 				SearchParams: []idxParam{
 					{
@@ -729,6 +734,7 @@ func main() {
 				},
 				ValidExamples: []string{
 					"nlist = 100",
+					"with_raw_data = true",
 				},
 				InvalidExamples: []string{
 					"nlist = 0",
diff --git a/entity/indexes_gen.go b/entity/indexes_gen.go
index ebcd3c6..0604c5b 100755
--- a/entity/indexes_gen.go
+++ b/entity/indexes_gen.go
@@ -730,6 +730,7 @@ var _ Index = &IndexSCANN{}
 // IndexSCANN idx type for IVF_FLAT
 type IndexSCANN struct { //auto generated fields
 	nlist int
+	with_raw_data bool
 	metricType MetricType
 }
 
@@ -752,6 +753,7 @@ func(i *IndexSCANN) SupportBinary() bool {
 func(i *IndexSCANN) Params() map[string]string {
 	params := map[string]string {//auto generated mapping 
 		"nlist": fmt.Sprintf("%v",i.nlist),
+		"with_raw_data": fmt.Sprintf("%v",i.with_raw_data),
 	}
 	bs, _ := json.Marshal(params)
 	return map[string]string {
@@ -764,6 +766,8 @@ func(i *IndexSCANN) Params() map[string]string {
 // NewIndexSCANN create index with construction parameters
 func NewIndexSCANN(metricType MetricType, 
 	nlist int,
+
+	with_raw_data bool,
 ) (*IndexSCANN, error) {
 	// auto generate parameters validation code, if any
 	if nlist < 1 {
@@ -773,9 +777,23 @@ func NewIndexSCANN(metricType MetricType,
 		return nil, errors.New("nlist has to be in range [1, 65536]")
 	}
 	
+	validRange := []bool{false, true}
+	with_raw_dataOk := false
+	for _, v := range validRange {
+		if v == with_raw_data {
+			with_raw_dataOk = true
+			break
+		}
+	}
+	if !with_raw_dataOk {
+		return nil, errors.New("with_raw_data not valid")
+	}
+	
 	return &IndexSCANN{ 
 	//auto generated setting
 	nlist: nlist,
+	//auto generated setting
+	with_raw_data: with_raw_data,
 	metricType: metricType,
 	}, nil
 }


Test patch:
diff --git a/entity/indexes_gen_test.go b/entity/indexes_gen_test.go
index 97fa4f7..cec8afb 100755
--- a/entity/indexes_gen_test.go
+++ b/entity/indexes_gen_test.go
@@ -573,6 +573,7 @@ func TestIndexGPUIvfPQ(t *testing.T){
 func TestIndexSCANN(t *testing.T){
 	
 	var nlist int
+	var with_raw_data bool
 
 	mt := L2
 	
@@ -582,6 +583,7 @@ func TestIndexSCANN(t *testing.T){
 		nlist = 100
 		idx0, err := NewIndexSCANN(mt, 
 			nlist,
+			with_raw_data,
 		)
 		assert.Nil(t, err)
 		assert.NotNil(t, idx0)
@@ -590,6 +592,18 @@ func TestIndexSCANN(t *testing.T){
 		assert.NotNil(t, idx0.Params())
 		assert.False(t, idx0.SupportBinary())
 		
+		with_raw_data = true
+		idx1, err := NewIndexSCANN(mt, 
+			nlist,
+			with_raw_data,
+		)
+		assert.Nil(t, err)
+		assert.NotNil(t, idx1)
+		assert.Equal(t, "SCANN", idx1.Name())
+		assert.EqualValues(t, "IVF_FLAT", idx1.IndexType())
+		assert.NotNil(t, idx1.Params())
+		assert.False(t, idx1.SupportBinary())
+		
 	})
 
 	t.Run("invalid usage case", func(t *testing.T){
@@ -597,6 +611,7 @@ func TestIndexSCANN(t *testing.T){
 		nlist = 0
 		idx0, err := NewIndexSCANN(mt, 
 			nlist,
+			with_raw_data,
 		)
 		assert.NotNil(t, err)
 		assert.Nil(t, idx0)
@@ -604,6 +619,7 @@ func TestIndexSCANN(t *testing.T){
 		nlist = 65537
 		idx1, err := NewIndexSCANN(mt, 
 			nlist,
+			with_raw_data,
 		)
 		assert.NotNil(t, err)
 		assert.Nil(t, idx1)