Repository: milvus-io/milvus-sdk-go
Pull request #552 (2023-08-11 08:24:29)

Support SCANN index type
See also milvus-io/milvus#26099
/kind improvement

sre-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/552#" title="Author self-approved">congqixia</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=milvus-io%2Fmilvus-sdk-go).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/milvus-io/milvus-sdk-go/blob/master/OWNERS)~~ [congqixia]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/552?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) Report
> Merging [#552](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/552?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) (a1dbddf) into [master](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/commit/2607d41ac7846bc9e81ce31cfd2052b0715755a9?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) (2607d41) will **increase** coverage by `0.10%`.
> Report is 1 commits behind head on master.
> The diff coverage is `93.93%`.

[![Impacted file tree graph](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/552/graphs/tree.svg?width=650&height=150&src=pr&token=WJK43PV390&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io)](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/552?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io)

```diff
@@            Coverage Diff             @@
##           master     #552      +/-   ##
==========================================
+ Coverage   82.91%   83.01%   +0.10%     
==========================================
  Files          39       39              
  Lines        3599     3632      +33     
==========================================
+ Hits         2984     3015      +31     
- Misses        503      504       +1     
- Partials      112      113       +1     
```


| [Files Changed](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/552?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) | Coverage Δ | |
|---|---|---|
| [entity/index.go](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/552?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io#diff-ZW50aXR5L2luZGV4Lmdv) | `100.00% <ø> (ø)` | |
| [entity/indexes\_search\_param\_gen.go](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/552?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io#diff-ZW50aXR5L2luZGV4ZXNfc2VhcmNoX3BhcmFtX2dlbi5nbw==) | `98.34% <85.71%> (-1.66%)` | :arrow_down: |
| [entity/indexes\_gen.go](https://app.codecov.io/gh/milvus-io/milvus-sdk-go/pull/552?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io#diff-ZW50aXR5L2luZGV4ZXNfZ2VuLmdv) | `100.00% <100.00%> (ø)` | |

yah01: /lgtm

Patch:
diff --git a/entity/genidx/genidx.go b/entity/genidx/genidx.go
index 379979e..88e3f45 100644
--- a/entity/genidx/genidx.go
+++ b/entity/genidx/genidx.go
@@ -708,6 +708,41 @@ func main() {
 					"nprobe = 65537",
 				},
 			},
+			{
+				IdxName: "SCANN",
+				IdxType: entity.IvfFlat,
+				ConstructParams: []idxParam{
+					{
+						Name:           "nlist",
+						ValidationRule: "[1, 65536]",
+					},
+				},
+				SearchParams: []idxParam{
+					{
+						Name:           "nprobe",
+						ValidationRule: "[1, 65536]", // [1, nlist], refer to index construct param, not supported yet
+					},
+					{
+						Name:           "reorder_k",
+						ValidationRule: "[1, 9223372036854775807]", // [topk, MAX_INT], refer to index construct param, not supported yet
+					},
+				},
+				ValidExamples: []string{
+					"nlist = 100",
+				},
+				InvalidExamples: []string{
+					"nlist = 0",
+					"nlist = 65537",
+				},
+				ValidSearchParams: []string{
+					"nprobe, reorder_k = 10, 200",
+				},
+				InvalidSearchParams: []string{
+					"nprobe, reorder_k = 0, 200",
+					"nprobe, reorder_k = 65537, 200",
+					"nprobe, reorder_k = 10, -1",
+				},
+			},
 		},
 	}
 
diff --git a/entity/index.go b/entity/index.go
index bc92bff..1dae8e5 100644
--- a/entity/index.go
+++ b/entity/index.go
@@ -36,6 +36,7 @@ const (
 	IvfHNSW    IndexType = "IVF_HNSW"
 	AUTOINDEX  IndexType = "AUTOINDEX"
 	DISKANN    IndexType = "DISKANN"
+	SCANN      IndexType = "SCANN"
 
 	GPUIvfFlat IndexType = "GPU_IVF_FLAT"
 	GPUIvfPQ   IndexType = "GPU_IVF_PQ"
diff --git a/entity/indexes_gen.go b/entity/indexes_gen.go
index acbd696..ebcd3c6 100755
--- a/entity/indexes_gen.go
+++ b/entity/indexes_gen.go
@@ -724,3 +724,59 @@ func NewIndexGPUIvfPQ(metricType MetricType,
 	}, nil
 }
 
+
+var _ Index = &IndexSCANN{}
+
+// IndexSCANN idx type for IVF_FLAT
+type IndexSCANN struct { //auto generated fields
+	nlist int
+	metricType MetricType
+}
+
+// Name returns index type name, implementing Index interface
+func(i *IndexSCANN) Name() string {
+	return "SCANN"
+}
+
+// IndexType returns IndexType, implementing Index interface
+func(i *IndexSCANN) IndexType() IndexType {
+	return IndexType("IVF_FLAT")
+}
+
+// SupportBinary returns whether index type support binary vector
+func(i *IndexSCANN) SupportBinary() bool {
+	return 0 & 2 > 0
+}
+
+// Params returns index construction params, implementing Index interface
+func(i *IndexSCANN) Params() map[string]string {
+	params := map[string]string {//auto generated mapping 
+		"nlist": fmt.Sprintf("%v",i.nlist),
+	}
+	bs, _ := json.Marshal(params)
+	return map[string]string {
+		"params": string(bs),
+		"index_type": string(i.IndexType()),
+		"metric_type": string(i.metricType),
+	}
+}
+
+// NewIndexSCANN create index with construction parameters
+func NewIndexSCANN(metricType MetricType, 
+	nlist int,
+) (*IndexSCANN, error) {
+	// auto generate parameters validation code, if any
+	if nlist < 1 {
+		return nil, errors.New("nlist has to be in range [1, 65536]")
+	}
+	if nlist > 65536 {
+		return nil, errors.New("nlist has to be in range [1, 65536]")
+	}
+	
+	return &IndexSCANN{ 
+	//auto generated setting
+	nlist: nlist,
+	metricType: metricType,
+	}, nil
+}
+
diff --git a/entity/indexes_search_param_gen.go b/entity/indexes_search_param_gen.go
index bcff101..a5a8148 100755
--- a/entity/indexes_search_param_gen.go
+++ b/entity/indexes_search_param_gen.go
@@ -380,3 +380,45 @@ func NewIndexGPUIvfPQSearchParam(
 	return sp, nil
 }
 
+var _ SearchParam = &IndexSCANNSearchParam{}
+
+// IndexSCANNSearchParam search param struct for index type IVF_FLAT
+type IndexSCANNSearchParam struct { //auto generated fields
+	baseSearchParams
+	
+	nprobe int
+	reorder_k int
+}
+
+// NewIndexSCANNSearchParam create index search param
+func NewIndexSCANNSearchParam(
+	nprobe int,
+
+	reorder_k int,
+) (*IndexSCANNSearchParam, error) {
+	// auto generate parameters validation code, if any
+	if nprobe < 1 {
+		return nil, errors.New("nprobe has to be in range [1, 65536]")
+	}
+	if nprobe > 65536 {
+		return nil, errors.New("nprobe has to be in range [1, 65536]")
+	}
+	
+	if reorder_k < 1 {
+		return nil, errors.New("reorder_k has to be in range [1, 9223372036854775807]")
+	}
+	if reorder_k > 9223372036854775807 {
+		return nil, errors.New("reorder_k has to be in range [1, 9223372036854775807]")
+	}
+	
+	sp := &IndexSCANNSearchParam{
+		baseSearchParams: newBaseSearchParams(),
+	}
+	
+	//auto generated setting
+	sp.params["nprobe"] = nprobe
+	sp.params["reorder_k"] = reorder_k
+
+	return sp, nil
+}
+


Test patch:
diff --git a/entity/indexes_gen_test.go b/entity/indexes_gen_test.go
index 4de1bbc..97fa4f7 100755
--- a/entity/indexes_gen_test.go
+++ b/entity/indexes_gen_test.go
@@ -570,3 +570,44 @@ func TestIndexGPUIvfPQ(t *testing.T){
 	})
 }
 
+func TestIndexSCANN(t *testing.T){
+	
+	var nlist int
+
+	mt := L2
+	
+
+	t.Run("valid usage case", func(t *testing.T){
+		
+		nlist = 100
+		idx0, err := NewIndexSCANN(mt, 
+			nlist,
+		)
+		assert.Nil(t, err)
+		assert.NotNil(t, idx0)
+		assert.Equal(t, "SCANN", idx0.Name())
+		assert.EqualValues(t, "IVF_FLAT", idx0.IndexType())
+		assert.NotNil(t, idx0.Params())
+		assert.False(t, idx0.SupportBinary())
+		
+	})
+
+	t.Run("invalid usage case", func(t *testing.T){
+		
+		nlist = 0
+		idx0, err := NewIndexSCANN(mt, 
+			nlist,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx0)
+		
+		nlist = 65537
+		idx1, err := NewIndexSCANN(mt, 
+			nlist,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx1)
+		
+	})
+}
+
diff --git a/entity/indexes_search_param_gen_test.go b/entity/indexes_search_param_gen_test.go
index f196d33..67b9d59 100755
--- a/entity/indexes_search_param_gen_test.go
+++ b/entity/indexes_search_param_gen_test.go
@@ -449,3 +449,51 @@ func TestIndexGPUIvfPQSearchParam(t *testing.T) {
 	
 }
 
+func TestIndexSCANNSearchParam(t *testing.T) {
+	
+	var nprobe int
+	var reorder_k int
+
+	t.Run("valid usage case", func(t *testing.T){
+		
+		nprobe, reorder_k = 10, 200
+		idx0, err := NewIndexSCANNSearchParam(
+			nprobe,
+			reorder_k,
+		)
+		assert.Nil(t, err)
+		assert.NotNil(t, idx0)
+		assert.NotNil(t, idx0.Params())
+		
+	})
+	
+	t.Run("invalid usage case", func(t *testing.T){
+		
+		nprobe, reorder_k = 0, 200
+		idx0, err := NewIndexSCANNSearchParam(
+			nprobe,
+			reorder_k,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx0)
+		
+		nprobe, reorder_k = 65537, 200
+		idx1, err := NewIndexSCANNSearchParam(
+			nprobe,
+			reorder_k,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx1)
+		
+		nprobe, reorder_k = 10, -1
+		idx2, err := NewIndexSCANNSearchParam(
+			nprobe,
+			reorder_k,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx2)
+		
+	})
+	
+}
+