Repository: milvus-io/milvus-sdk-go
Pull request #547 (2023-08-04 14:06:08)

Update L3 cases and nightly qn replicas
- Update L3 cases
- update nightly qn replicas from 1 to 4

sre-ci-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/547#" title="Author self-approved">ThreadDao</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **sunby** after the PR has been reviewed.
You can assign the PR to them by writing `/assign @sunby` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=milvus-io%2Fmilvus-sdk-go).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/milvus-io/milvus-sdk-go/blob/master/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["sunby"]} -->
sre-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/547#pullrequestreview-1573251509" title="LGTM">congqixia</a>*, *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/547#" title="Author self-approved">ThreadDao</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=milvus-io%2Fmilvus-sdk-go).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/milvus-io/milvus-sdk-go/blob/master/OWNERS)~~ [congqixia]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/ci/milvus-operator/milvus-nightly.yaml b/ci/milvus-operator/milvus-nightly.yaml
index 8144d0e..0b7effd 100644
--- a/ci/milvus-operator/milvus-nightly.yaml
+++ b/ci/milvus-operator/milvus-nightly.yaml
@@ -119,7 +119,7 @@ spec:
   components:
     image: ${IMAGE_REPO}:${IMAGE_TAG}
     queryNode:
-      replicas: 2
+      replicas: 4
     dataNode:
       replicas: 1
     indexNode:
diff --git a/client/data.go b/client/data.go
index 27bbf76..49335b7 100644
--- a/client/data.go
+++ b/client/data.go
@@ -395,10 +395,10 @@ func (c *GrpcClient) CalcDistance(ctx context.Context, collName string, partitio
 			return nil, err
 		}
 	}
-	if err := c.checkCollField(ctx, collName, opLeft.Name()); err != nil {
+	if err := c.checkCollField(ctx, collName, opLeft.Name(), isVectorField); err != nil {
 		return nil, err
 	}
-	if err := c.checkCollField(ctx, collName, opRight.Name()); err != nil {
+	if err := c.checkCollField(ctx, collName, opRight.Name(), isVectorField); err != nil {
 		return nil, err
 	}
 
diff --git a/client/index.go b/client/index.go
index c7c0ac0..54a0798 100644
--- a/client/index.go
+++ b/client/index.go
@@ -21,7 +21,7 @@ import (
 	"github.com/milvus-io/milvus-sdk-go/v2/entity"
 )
 
-func (c *GrpcClient) checkCollField(ctx context.Context, collName string, fieldName string) error {
+func (c *GrpcClient) checkCollField(ctx context.Context, collName string, fieldName string, filters ...func(string, string, *entity.Field) error) error {
 	if err := c.checkCollectionExists(ctx, collName); err != nil {
 		return err
 	}
@@ -33,8 +33,10 @@ func (c *GrpcClient) checkCollField(ctx context.Context, collName string, fieldN
 	for _, field := range coll.Schema.Fields {
 		if field.Name == fieldName {
 			f = field
-			if f.DataType != entity.FieldTypeFloatVector && f.DataType != entity.FieldTypeBinaryVector {
-				return fmt.Errorf("field %s of collection %s is not vector field", fieldName, collName)
+			for _, filter := range filters {
+				if err := filter(collName, fieldName, f); err != nil {
+					return err
+				}
 			}
 			break
 		}
@@ -45,6 +47,13 @@ func (c *GrpcClient) checkCollField(ctx context.Context, collName string, fieldN
 	return nil
 }
 
+func isVectorField(collName, fieldName string, f *entity.Field) error {
+	if f.DataType != entity.FieldTypeFloatVector && f.DataType != entity.FieldTypeBinaryVector {
+		return fmt.Errorf("field %s of collection %s is not vector field", fieldName, collName)
+	}
+	return nil
+}
+
 type indexDef struct {
 	name           string
 	fieldName      string
diff --git a/entity/genidx/genidx.go b/entity/genidx/genidx.go
index 379979e..88e3f45 100644
--- a/entity/genidx/genidx.go
+++ b/entity/genidx/genidx.go
@@ -708,6 +708,41 @@ func main() {
 					"nprobe = 65537",
 				},
 			},
+			{
+				IdxName: "SCANN",
+				IdxType: entity.IvfFlat,
+				ConstructParams: []idxParam{
+					{
+						Name:           "nlist",
+						ValidationRule: "[1, 65536]",
+					},
+				},
+				SearchParams: []idxParam{
+					{
+						Name:           "nprobe",
+						ValidationRule: "[1, 65536]", // [1, nlist], refer to index construct param, not supported yet
+					},
+					{
+						Name:           "reorder_k",
+						ValidationRule: "[1, 9223372036854775807]", // [topk, MAX_INT], refer to index construct param, not supported yet
+					},
+				},
+				ValidExamples: []string{
+					"nlist = 100",
+				},
+				InvalidExamples: []string{
+					"nlist = 0",
+					"nlist = 65537",
+				},
+				ValidSearchParams: []string{
+					"nprobe, reorder_k = 10, 200",
+				},
+				InvalidSearchParams: []string{
+					"nprobe, reorder_k = 0, 200",
+					"nprobe, reorder_k = 65537, 200",
+					"nprobe, reorder_k = 10, -1",
+				},
+			},
 		},
 	}
 
diff --git a/entity/index.go b/entity/index.go
index bc92bff..310f224 100644
--- a/entity/index.go
+++ b/entity/index.go
@@ -36,9 +36,12 @@ const (
 	IvfHNSW    IndexType = "IVF_HNSW"
 	AUTOINDEX  IndexType = "AUTOINDEX"
 	DISKANN    IndexType = "DISKANN"
+	SCANN      IndexType = "SCANN"
 
 	GPUIvfFlat IndexType = "GPU_IVF_FLAT"
 	GPUIvfPQ   IndexType = "GPU_IVF_PQ"
+
+	Scalar IndexType = "SCALAR"
 )
 
 // Metric Constants
diff --git a/entity/index_scalar.go b/entity/index_scalar.go
new file mode 100644
index 0000000..44d6fb6
--- /dev/null
+++ b/entity/index_scalar.go
@@ -0,0 +1,19 @@
+package entity
+
+var _ Index = (*indexScalar)(nil)
+
+type indexScalar struct {
+	baseIndex
+}
+
+func (i *indexScalar) Params() map[string]string {
+	return map[string]string{}
+}
+
+func NewScalarIndex() Index {
+	return &indexScalar{
+		baseIndex: baseIndex{
+			it: Scalar,
+		},
+	}
+}
diff --git a/entity/indexes_gen.go b/entity/indexes_gen.go
index acbd696..ebcd3c6 100755
--- a/entity/indexes_gen.go
+++ b/entity/indexes_gen.go
@@ -724,3 +724,59 @@ func NewIndexGPUIvfPQ(metricType MetricType,
 	}, nil
 }
 
+
+var _ Index = &IndexSCANN{}
+
+// IndexSCANN idx type for IVF_FLAT
+type IndexSCANN struct { //auto generated fields
+	nlist int
+	metricType MetricType
+}
+
+// Name returns index type name, implementing Index interface
+func(i *IndexSCANN) Name() string {
+	return "SCANN"
+}
+
+// IndexType returns IndexType, implementing Index interface
+func(i *IndexSCANN) IndexType() IndexType {
+	return IndexType("IVF_FLAT")
+}
+
+// SupportBinary returns whether index type support binary vector
+func(i *IndexSCANN) SupportBinary() bool {
+	return 0 & 2 > 0
+}
+
+// Params returns index construction params, implementing Index interface
+func(i *IndexSCANN) Params() map[string]string {
+	params := map[string]string {//auto generated mapping 
+		"nlist": fmt.Sprintf("%v",i.nlist),
+	}
+	bs, _ := json.Marshal(params)
+	return map[string]string {
+		"params": string(bs),
+		"index_type": string(i.IndexType()),
+		"metric_type": string(i.metricType),
+	}
+}
+
+// NewIndexSCANN create index with construction parameters
+func NewIndexSCANN(metricType MetricType, 
+	nlist int,
+) (*IndexSCANN, error) {
+	// auto generate parameters validation code, if any
+	if nlist < 1 {
+		return nil, errors.New("nlist has to be in range [1, 65536]")
+	}
+	if nlist > 65536 {
+		return nil, errors.New("nlist has to be in range [1, 65536]")
+	}
+	
+	return &IndexSCANN{ 
+	//auto generated setting
+	nlist: nlist,
+	metricType: metricType,
+	}, nil
+}
+
diff --git a/entity/indexes_search_param_gen.go b/entity/indexes_search_param_gen.go
index bcff101..a5a8148 100755
--- a/entity/indexes_search_param_gen.go
+++ b/entity/indexes_search_param_gen.go
@@ -380,3 +380,45 @@ func NewIndexGPUIvfPQSearchParam(
 	return sp, nil
 }
 
+var _ SearchParam = &IndexSCANNSearchParam{}
+
+// IndexSCANNSearchParam search param struct for index type IVF_FLAT
+type IndexSCANNSearchParam struct { //auto generated fields
+	baseSearchParams
+	
+	nprobe int
+	reorder_k int
+}
+
+// NewIndexSCANNSearchParam create index search param
+func NewIndexSCANNSearchParam(
+	nprobe int,
+
+	reorder_k int,
+) (*IndexSCANNSearchParam, error) {
+	// auto generate parameters validation code, if any
+	if nprobe < 1 {
+		return nil, errors.New("nprobe has to be in range [1, 65536]")
+	}
+	if nprobe > 65536 {
+		return nil, errors.New("nprobe has to be in range [1, 65536]")
+	}
+	
+	if reorder_k < 1 {
+		return nil, errors.New("reorder_k has to be in range [1, 9223372036854775807]")
+	}
+	if reorder_k > 9223372036854775807 {
+		return nil, errors.New("reorder_k has to be in range [1, 9223372036854775807]")
+	}
+	
+	sp := &IndexSCANNSearchParam{
+		baseSearchParams: newBaseSearchParams(),
+	}
+	
+	//auto generated setting
+	sp.params["nprobe"] = nprobe
+	sp.params["reorder_k"] = reorder_k
+
+	return sp, nil
+}
+
diff --git a/examples/index/index.go b/examples/index/index.go
index bccceb2..3888ecd 100644
--- a/examples/index/index.go
+++ b/examples/index/index.go
@@ -18,9 +18,8 @@ func main() {
 	// Milvus instance proxy address, may verify in your env/settings
 	milvusAddr := `localhost:19530`
 
-	// setup context for client creation, use 2 seconds here
-	ctx := context.Background()
-	ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+	// setup context
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	c, err := client.NewClient(ctx, client.Config{
@@ -46,32 +45,10 @@ func main() {
 	}
 
 	// define collection schema, see film.csv
-	schema := &entity.Schema{
-		CollectionName: collectionName,
-		Description:    "this is the example collection for inser and search",
-		AutoID:         false,
-		Fields: []*entity.Field{
-			{
-				Name:       "ID",
-				DataType:   entity.FieldTypeInt64, // int64 only for now
-				PrimaryKey: true,
-				AutoID:     false,
-			},
-			{
-				Name:       "Year",
-				DataType:   entity.FieldTypeInt32,
-				PrimaryKey: false,
-				AutoID:     false,
-			},
-			{
-				Name:     "Vector",
-				DataType: entity.FieldTypeFloatVector,
-				TypeParams: map[string]string{
-					entity.TypeParamDim: "8",
-				},
-			},
-		},
-	}
+	schema := entity.NewSchema().WithName(collectionName).WithDescription("this is the example collection for indexing").
+		WithField(entity.NewField().WithName("ID").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
+		WithField(entity.NewField().WithName("Year").WithDataType(entity.FieldTypeInt32)).
+		WithField(entity.NewField().WithName("Vector").WithDataType(entity.FieldTypeFloatVector).WithDim(8))
 
 	err = c.CreateCollection(ctx, schema, entity.DefaultShardNumber)
 	if err != nil {
@@ -112,6 +89,20 @@ func main() {
 		log.Fatal("failed to flush collection:", err.Error())
 	}
 	log.Println("flush completed")
+	// Now add index
+	idx, err := entity.NewIndexIvfFlat(entity.L2, 2)
+	if err != nil {
+		log.Fatal("fail to create ivf flat index:", err.Error())
+	}
+	err = c.CreateIndex(ctx, collectionName, "Vector", idx, false)
+	if err != nil {
+		log.Fatal("fail to create index:", err.Error())
+	}
+
+	sidx := entity.NewScalarIndex()
+	if err := c.CreateIndex(ctx, collectionName, "Year", sidx, false); err != nil {
+		log.Fatal("failed to create scalar index", err.Error())
+	}
 
 	// load collection with async=false
 	err = c.LoadCollection(ctx, collectionName, false)
@@ -153,25 +144,6 @@ func main() {
 		}
 	}
 
-	// Now add index
-	idx, err := entity.NewIndexIvfFlat(entity.L2, 2)
-	if err != nil {
-		log.Fatal("fail to create ivf flat index:", err.Error())
-	}
-	err = c.CreateIndex(ctx, collectionName, "Vector", idx, false)
-	if err != nil {
-		log.Fatal("fail to create index:", err.Error())
-	}
-
-	// do search again
-	start = time.Now()
-	sr, err = c.Search(ctx, collectionName, []string{}, "Year > 1990", []string{"ID"}, []entity.Vector{vector}, "Vector",
-		entity.L2, 10, sp)
-	if err != nil {
-		log.Fatal("fail to search collection:", err.Error())
-	}
-	log.Println("search with index time elapsed:", time.Since(start))
-
 	// clean up
 	_ = c.DropCollection(ctx, collectionName)
 }


Test patch:
diff --git a/entity/indexes_gen_test.go b/entity/indexes_gen_test.go
index 4de1bbc..97fa4f7 100755
--- a/entity/indexes_gen_test.go
+++ b/entity/indexes_gen_test.go
@@ -570,3 +570,44 @@ func TestIndexGPUIvfPQ(t *testing.T){
 	})
 }
 
+func TestIndexSCANN(t *testing.T){
+	
+	var nlist int
+
+	mt := L2
+	
+
+	t.Run("valid usage case", func(t *testing.T){
+		
+		nlist = 100
+		idx0, err := NewIndexSCANN(mt, 
+			nlist,
+		)
+		assert.Nil(t, err)
+		assert.NotNil(t, idx0)
+		assert.Equal(t, "SCANN", idx0.Name())
+		assert.EqualValues(t, "IVF_FLAT", idx0.IndexType())
+		assert.NotNil(t, idx0.Params())
+		assert.False(t, idx0.SupportBinary())
+		
+	})
+
+	t.Run("invalid usage case", func(t *testing.T){
+		
+		nlist = 0
+		idx0, err := NewIndexSCANN(mt, 
+			nlist,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx0)
+		
+		nlist = 65537
+		idx1, err := NewIndexSCANN(mt, 
+			nlist,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx1)
+		
+	})
+}
+
diff --git a/entity/indexes_search_param_gen_test.go b/entity/indexes_search_param_gen_test.go
index f196d33..67b9d59 100755
--- a/entity/indexes_search_param_gen_test.go
+++ b/entity/indexes_search_param_gen_test.go
@@ -449,3 +449,51 @@ func TestIndexGPUIvfPQSearchParam(t *testing.T) {
 	
 }
 
+func TestIndexSCANNSearchParam(t *testing.T) {
+	
+	var nprobe int
+	var reorder_k int
+
+	t.Run("valid usage case", func(t *testing.T){
+		
+		nprobe, reorder_k = 10, 200
+		idx0, err := NewIndexSCANNSearchParam(
+			nprobe,
+			reorder_k,
+		)
+		assert.Nil(t, err)
+		assert.NotNil(t, idx0)
+		assert.NotNil(t, idx0.Params())
+		
+	})
+	
+	t.Run("invalid usage case", func(t *testing.T){
+		
+		nprobe, reorder_k = 0, 200
+		idx0, err := NewIndexSCANNSearchParam(
+			nprobe,
+			reorder_k,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx0)
+		
+		nprobe, reorder_k = 65537, 200
+		idx1, err := NewIndexSCANNSearchParam(
+			nprobe,
+			reorder_k,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx1)
+		
+		nprobe, reorder_k = 10, -1
+		idx2, err := NewIndexSCANNSearchParam(
+			nprobe,
+			reorder_k,
+		)
+		assert.NotNil(t, err)
+		assert.Nil(t, idx2)
+		
+	})
+	
+}
+
diff --git a/test/testcases/index_test.go b/test/testcases/index_test.go
index 5252a18..21b19cc 100644
--- a/test/testcases/index_test.go
+++ b/test/testcases/index_test.go
@@ -171,6 +171,7 @@ func TestCreateIndexNotExistField(t *testing.T) {
 
 // test create index on non-vector field
 func TestCreateIndexNotSupportedField(t *testing.T) {
+	t.Skip("scalar index shall be supported")
 	ctx := createContext(t, time.Second*common.DefaultTimeout)
 	// connect
 	mc := createMilvusClient(ctx, t)
diff --git a/test/testcases/resource_group_test.go b/test/testcases/resource_group_test.go
index 97186d5..cf3c04f 100644
--- a/test/testcases/resource_group_test.go
+++ b/test/testcases/resource_group_test.go
@@ -22,6 +22,14 @@ const configQnNodes = int32(4)
 const newRgNode = int32(2)
 
 func resetRgs(t *testing.T, ctx context.Context, mc *base.MilvusClient) {
+	// release and drop all collections
+	collections, _ := mc.ListCollections(ctx)
+	for _, coll := range collections {
+		mc.ReleaseCollection(ctx, coll.Name)
+		err := mc.DropCollection(ctx, coll.Name)
+		common.CheckErr(t, err, true)
+	}
+
 	// reset resource groups
 	rgs, errList := mc.ListResourceGroups(ctx)
 	common.CheckErr(t, errList, true)
@@ -146,7 +154,7 @@ func TestDescribeRgNotExisted(t *testing.T) {
 	mc := createMilvusClient(ctx, t)
 
 	_, errDescribe := mc.DescribeResourceGroup(ctx, common.GenRandomString(6))
-	common.CheckErr(t, errDescribe, false, "resource group doesn't exist")
+	common.CheckErr(t, errDescribe, false, "resource group not found")
 }
 
 // drop rg with not existed name -> successfully
@@ -252,9 +260,9 @@ func TestTransferInvalidNodes(t *testing.T) {
 		errMsg   string
 	}
 	invalidNodes := []invalidNodesStruct{
-		{nodesNum: 0, errMsg: "transfer node num can't be"},
-		{nodesNum: -1, errMsg: "transfer node num can't be"},
-		{nodesNum: 99, errMsg: "failed to transfer node between resource group, err=nodes not enough"},
+		{nodesNum: 0, errMsg: "expected=NumNode > 0, actual=invalid NumNode"},
+		{nodesNum: -1, errMsg: "expected=NumNode > 0, actual=invalid NumNode"},
+		{nodesNum: 99, errMsg: "nodes not enough"},
 	}
 	// transfer node
 	for _, invalidNode := range invalidNodes {
@@ -272,11 +280,11 @@ func TestTransferRgNotExisted(t *testing.T) {
 
 	// source not exist
 	errSource := mc.TransferNode(ctx, common.GenRandomString(6), common.DefaultRgName, newRgNode)
-	common.CheckErr(t, errSource, false, "resource group doesn't exist")
+	common.CheckErr(t, errSource, false, "source resource group not found")
 
 	// target not exist
 	errTarget := mc.TransferNode(ctx, common.DefaultRgName, common.GenRandomString(6), newRgNode)
-	common.CheckErr(t, errTarget, false, "resource group doesn't exist")
+	common.CheckErr(t, errTarget, false, "target resource group not found")
 
 	// transfer to self
 	errSelf := mc.TransferNode(ctx, common.DefaultRgName, common.DefaultRgName, newRgNode)
@@ -343,8 +351,27 @@ func TestTransferReplicas(t *testing.T) {
 	}
 	common.CheckResourceGroup(t, newRg, expRg)
 
+	// drop new rg that loaded collection
+	err := mc.DropResourceGroup(ctx, rgName)
+	common.CheckErr(t, err, false, "some replicas still loaded in resource group")
+
 	// search
-	// todo
+	sp, err := entity.NewIndexHNSWSearchParam(74)
+	searchRes, _ := mc.Search(
+		ctx, collName,
+		[]string{common.DefaultPartition},
+		"",
+		[]string{common.DefaultFloatFieldName},
+		common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector),
+		common.DefaultFloatVecFieldName,
+		entity.L2,
+		common.DefaultTopK,
+		sp,
+	)
+	// check search result contains search vector, which from all partitions
+	common.CheckErr(t, err, true)
+	common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultTopK)
+
 }
 
 // test transfer replica of not existed collection
@@ -385,8 +412,8 @@ func TestTransferReplicaInvalidReplicaNumber(t *testing.T) {
 		errMsg        string
 	}
 	invalidReplicas := []invalidReplicasStruct{
-		{replicaNumber: 0, errMsg: "transfer replica num can't be [0]"},
-		{replicaNumber: -1, errMsg: "transfer replica num can't be [-1]"},
+		{replicaNumber: 0, errMsg: "expected=NumReplica > 0, actual=invalid NumReplica"},
+		{replicaNumber: -1, errMsg: "expected=NumReplica > 0, actual=invalid NumReplica"},
 		{replicaNumber: 1, errMsg: "only found [0] replicas in source resource group"},
 	}
 
@@ -416,11 +443,11 @@ func TestTransferReplicaRgNotExisted(t *testing.T) {
 
 	// source not exist
 	errSource := mc.TransferReplica(ctx, common.GenRandomString(6), common.DefaultRgName, collName, 1)
-	common.CheckErr(t, errSource, false, "resource group doesn't exist")
+	common.CheckErr(t, errSource, false, "resource group not found")
 
 	// target not exist
 	errTarget := mc.TransferReplica(ctx, common.DefaultRgName, common.GenRandomString(6), collName, 1)
-	common.CheckErr(t, errTarget, false, "resource group doesn't exist")
+	common.CheckErr(t, errTarget, false, "resource group not found")
 
 	// transfer to self -> error
 	errSelf := mc.TransferReplica(ctx, rgName, rgName, collName, 1)