Repository: milvus-io/milvus-sdk-go
Pull request #443 (2023-04-24 07:44:51)

Add JSON data type
See also #432 
Related to https://github.com/milvus-io/milvus/issues/23389
/kind enhancement

sre-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/milvus-io/milvus-sdk-go/pull/443#" title="Author self-approved">congqixia</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=milvus-io%2Fmilvus-sdk-go).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/milvus-io/milvus-sdk-go/blob/master/OWNERS)~~ [congqixia]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
codecov[bot]: ## [Codecov](https://codecov.io/gh/milvus-io/milvus-sdk-go/pull/443?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) Report
> Merging [#443](https://codecov.io/gh/milvus-io/milvus-sdk-go/pull/443?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) (5665f9f) into [master](https://codecov.io/gh/milvus-io/milvus-sdk-go/commit/e6dc3802e1a7f25b43981e04757e41f1f2efa443?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) (e6dc380) will **increase** coverage by `0.13%`.
> The diff coverage is `100.00%`.

[![Impacted file tree graph](https://codecov.io/gh/milvus-io/milvus-sdk-go/pull/443/graphs/tree.svg?width=650&height=150&src=pr&token=WJK43PV390&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io)](https://codecov.io/gh/milvus-io/milvus-sdk-go/pull/443?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io)

```diff
@@            Coverage Diff             @@
##           master     #443      +/-   ##
==========================================
+ Coverage   87.35%   87.48%   +0.13%     
==========================================
  Files          31       32       +1     
  Lines        2878     2909      +31     
==========================================
+ Hits         2514     2545      +31     
  Misses        274      274              
  Partials       90       90              
```


| [Impacted Files](https://codecov.io/gh/milvus-io/milvus-sdk-go/pull/443?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io) | Coverage Δ | |
|---|---|---|
| [entity/columns\_json.go](https://codecov.io/gh/milvus-io/milvus-sdk-go/pull/443?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io#diff-ZW50aXR5L2NvbHVtbnNfanNvbi5nbw==) | `100.00% <100.00%> (ø)` | |
| [entity/schema.go](https://codecov.io/gh/milvus-io/milvus-sdk-go/pull/443?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=milvus-io#diff-ZW50aXR5L3NjaGVtYS5nbw==) | `93.44% <100.00%> (+0.33%)` | :arrow_up: |

yah01: /lgtm

Patch:
diff --git a/entity/columns_json.go b/entity/columns_json.go
new file mode 100644
index 0000000..7cdbb67
--- /dev/null
+++ b/entity/columns_json.go
@@ -0,0 +1,84 @@
+package entity
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/errors"
+	schema "github.com/milvus-io/milvus-proto/go-api/schemapb"
+)
+
+var _ (Column) = (*ColumnJSONBytes)(nil)
+
+// ColumnJSONBytes column type for JSON.
+// all items are marshaled json bytes.
+type ColumnJSONBytes struct {
+	name   string
+	values [][]byte
+}
+
+// Name returns column name.
+func (c *ColumnJSONBytes) Name() string {
+	return c.name
+}
+
+// Type returns column FieldType.
+func (c *ColumnJSONBytes) Type() FieldType {
+	return FieldTypeJSON
+}
+
+// Len returns column values length.
+func (c *ColumnJSONBytes) Len() int {
+	return len(c.values)
+}
+
+// FieldData return column data mapped to schema.FieldData.
+func (c *ColumnJSONBytes) FieldData() *schema.FieldData {
+	fd := &schema.FieldData{
+		Type:      schema.DataType_JSON,
+		FieldName: c.name,
+	}
+
+	fd.Field = &schema.FieldData_Scalars{
+		Scalars: &schema.ScalarField{
+			Data: &schema.ScalarField_JsonData{
+				JsonData: &schema.JSONArray{
+					Data: c.values,
+				},
+			},
+		},
+	}
+
+	return fd
+}
+
+// ValueByIdx returns value of the provided index.
+func (c *ColumnJSONBytes) ValueByIdx(idx int) ([]byte, error) {
+	if idx < 0 || idx >= c.Len() {
+		return nil, errors.New("index out of range")
+	}
+	return c.values[idx], nil
+}
+
+// AppendValue append value into column.
+func (c *ColumnJSONBytes) AppendValue(i interface{}) error {
+	v, ok := i.([]byte)
+	if !ok {
+		return fmt.Errorf("invalid type, expected []byte, got %T", i)
+	}
+	c.values = append(c.values, v)
+
+	return nil
+}
+
+// Data returns column data.
+func (c *ColumnJSONBytes) Data() [][]byte {
+	return c.values
+}
+
+// NewColumnJSONBytes composes a Column with json bytes.
+func NewColumnJSONBytes(name string, values [][]byte) *ColumnJSONBytes {
+	return &ColumnJSONBytes{
+		name:   name,
+		values: values,
+	}
+}
diff --git a/entity/schema.go b/entity/schema.go
index e1ca53c..ca304e5 100644
--- a/entity/schema.go
+++ b/entity/schema.go
@@ -163,6 +163,8 @@ func (t FieldType) Name() string {
 		return "String"
 	case FieldTypeVarChar:
 		return "VarChar"
+	case FieldTypeJSON:
+		return "JSON"
 	case FieldTypeBinaryVector:
 		return "BinaryVector"
 	case FieldTypeFloatVector:
@@ -193,6 +195,8 @@ func (t FieldType) String() string {
 		return "string"
 	case FieldTypeVarChar:
 		return "string"
+	case FieldTypeJSON:
+		return "JSON"
 	case FieldTypeBinaryVector:
 		return "[]byte"
 	case FieldTypeFloatVector:
@@ -223,15 +227,15 @@ func (t FieldType) PbFieldType() (string, string) {
 		return "String", "string"
 	case FieldTypeVarChar:
 		return "VarChar", "string"
+	case FieldTypeJSON:
+		return "JSON", "JSON"
 	case FieldTypeBinaryVector:
 		return "[]byte", ""
 	case FieldTypeFloatVector:
 		return "[]float32", ""
 	default:
 		return "undefined", ""
-
 	}
-
 }
 
 // Match schema definition
@@ -256,6 +260,9 @@ const (
 	FieldTypeString FieldType = 20
 	// FieldTypeVarChar field type varchar
 	FieldTypeVarChar FieldType = 21 // variable-length strings with a specified maximum length
+	// FieldTypeArray FieldType = 22
+	// FieldTypeJSON field type JSON
+	FieldTypeJSON FieldType = 23
 	// FieldTypeBinaryVector field type binary vector
 	FieldTypeBinaryVector FieldType = 100
 	// FieldTypeFloatVector field type float vector


Test patch:
diff --git a/entity/columns_json_test.go b/entity/columns_json_test.go
new file mode 100644
index 0000000..90da76e
--- /dev/null
+++ b/entity/columns_json_test.go
@@ -0,0 +1,78 @@
+package entity
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type ColumnJSONBytesSuite struct {
+	suite.Suite
+}
+
+func (s *ColumnJSONBytesSuite) SetupSuite() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func (s *ColumnJSONBytesSuite) TestAttrMethods() {
+	columnName := fmt.Sprintf("column_jsonbs_%d", rand.Int())
+	columnLen := 8 + rand.Intn(10)
+
+	v := make([][]byte, columnLen)
+	column := NewColumnJSONBytes(columnName, v)
+
+	s.Run("test_meta", func() {
+		ft := FieldTypeJSON
+		s.Equal("JSON", ft.Name())
+		s.Equal("JSON", ft.String())
+		pbName, pbType := ft.PbFieldType()
+		s.Equal("JSON", pbName)
+		s.Equal("JSON", pbType)
+	})
+
+	s.Run("test_column_attribute", func() {
+		s.Equal(columnName, column.Name())
+		s.Equal(FieldTypeJSON, column.Type())
+		s.Equal(columnLen, column.Len())
+		s.EqualValues(v, column.Data())
+	})
+
+	s.Run("test_column_field_data", func() {
+		fd := column.FieldData()
+		s.NotNil(fd)
+		s.Equal(fd.GetFieldName(), columnName)
+	})
+
+	s.Run("test_column_valuer_by_idx", func() {
+		_, err := column.ValueByIdx(-1)
+		s.Error(err)
+		_, err = column.ValueByIdx(columnLen)
+		s.Error(err)
+		for i := 0; i < columnLen; i++ {
+			v, err := column.ValueByIdx(i)
+			s.NoError(err)
+			s.Equal(column.values[i], v)
+		}
+	})
+
+	s.Run("test_append_value", func() {
+		item := make([]byte, 10)
+		err := column.AppendValue(item)
+		s.NoError(err)
+		s.Equal(columnLen+1, column.Len())
+		val, err := column.ValueByIdx(columnLen)
+		s.NoError(err)
+		s.Equal(item, val)
+
+		err = column.AppendValue(1)
+		s.Error(err)
+	})
+
+}
+
+func TestColumnJSONBytes(t *testing.T) {
+	suite.Run(t, new(ColumnJSONBytesSuite))
+}