Repository: blevesearch/bleve
Pull request #2015 (2024-04-17 22:16:52)

MB-61009: Raise max supported dimensionality of vectors to 4096
+ Allow calling application to change min/max.
+ Will require downstream changes to disallow any setting greater
  than 2048 in the event of a mixed version cluster.


Patch:
diff --git a/document/field_vector_base64.go b/document/field_vector_base64.go
new file mode 100644
index 00000000..e62dbe0a
--- /dev/null
+++ b/document/field_vector_base64.go
@@ -0,0 +1,140 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build vectors
+// +build vectors
+
+package document
+
+import (
+	"encoding/base64"
+	"encoding/binary"
+	"fmt"
+	"math"
+
+	"github.com/blevesearch/bleve/v2/size"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+type VectorBase64Field struct {
+	vectorField    *VectorField
+	base64Encoding string
+}
+
+func (n *VectorBase64Field) Size() int {
+	return n.vectorField.Size()
+}
+
+func (n *VectorBase64Field) Name() string {
+	return n.vectorField.Name()
+}
+
+func (n *VectorBase64Field) ArrayPositions() []uint64 {
+	return n.vectorField.ArrayPositions()
+}
+
+func (n *VectorBase64Field) Options() index.FieldIndexingOptions {
+	return n.vectorField.Options()
+}
+
+func (n *VectorBase64Field) NumPlainTextBytes() uint64 {
+	return n.vectorField.NumPlainTextBytes()
+}
+
+func (n *VectorBase64Field) AnalyzedLength() int {
+	return n.vectorField.AnalyzedLength()
+}
+
+func (n *VectorBase64Field) EncodedFieldType() byte {
+	return 'e'
+}
+
+func (n *VectorBase64Field) AnalyzedTokenFrequencies() index.TokenFrequencies {
+	return n.vectorField.AnalyzedTokenFrequencies()
+}
+
+func (n *VectorBase64Field) Analyze() {
+}
+
+func (n *VectorBase64Field) Value() []byte {
+	return n.vectorField.Value()
+}
+
+func (n *VectorBase64Field) GoString() string {
+	return fmt.Sprintf("&document.vectorFieldBase64Field{Name:%s, Options: %s, "+
+		"Value: %+v}", n.vectorField.Name(), n.vectorField.Options(), n.vectorField.Value())
+}
+
+// For the sake of not polluting the API, we are keeping arrayPositions as a
+// parameter, but it is not used.
+func NewVectorBase64Field(name string, arrayPositions []uint64, vectorBase64 string,
+	dims int, similarity, vectorIndexOptimizedFor string) (*VectorBase64Field, error) {
+
+	vector, err := DecodeVector(vectorBase64)
+	if err != nil {
+		return nil, err
+	}
+
+	return &VectorBase64Field{
+		vectorField: NewVectorFieldWithIndexingOptions(name, arrayPositions,
+			vector, dims, similarity,
+			vectorIndexOptimizedFor, DefaultVectorIndexingOptions),
+
+		base64Encoding: vectorBase64,
+	}, nil
+}
+
+// This function takes a base64 encoded string and decodes it into
+// a vector.
+func DecodeVector(encodedValue string) ([]float32, error) {
+
+	// We first decode the encoded string into a byte array.
+	decodedString, err := base64.StdEncoding.DecodeString(encodedValue)
+	if err != nil {
+		return nil, err
+	}
+
+	// The array is expected to be divisible by 4 because each float32
+	// should occupy 4 bytes
+	if len(decodedString)%size.SizeOfFloat32 != 0 {
+		return nil, fmt.Errorf("Decoded byte array not divisible by %d", size.SizeOfFloat32)
+	}
+	dims := int(len(decodedString) / size.SizeOfFloat32)
+	decodedVector := make([]float32, dims)
+
+	// We iterate through the array 4 bytes at a time and convert each of
+	// them to a float32 value by reading them in a little endian notation
+	for i := 0; i < dims; i++ {
+		bytes := decodedString[i*size.SizeOfFloat32 : (i+1)*size.SizeOfFloat32]
+		decodedVector[i] = math.Float32frombits(binary.LittleEndian.Uint32(bytes))
+	}
+
+	return decodedVector, nil
+}
+
+func (n *VectorBase64Field) Vector() []float32 {
+	return n.vectorField.Vector()
+}
+
+func (n *VectorBase64Field) Dims() int {
+	return n.vectorField.Dims()
+}
+
+func (n *VectorBase64Field) Similarity() string {
+	return n.vectorField.Similarity()
+}
+
+func (n *VectorBase64Field) IndexOptimizedFor() string {
+	return n.vectorField.IndexOptimizedFor()
+}
diff --git a/go.mod b/go.mod
index dbbb63af..a716a231 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/blevesearch/zapx/v13 v13.3.10
 	github.com/blevesearch/zapx/v14 v14.3.10
 	github.com/blevesearch/zapx/v15 v15.3.13
-	github.com/blevesearch/zapx/v16 v16.0.13-0.20240410221809-91a5e17d9469
+	github.com/blevesearch/zapx/v16 v16.0.13-0.20240418135927-eeb2336467b1
 	github.com/couchbase/moss v0.2.0
 	github.com/golang/protobuf v1.3.2
 	github.com/spf13/cobra v1.7.0
diff --git a/go.sum b/go.sum
index b75d6cd8..f2ad3924 100644
--- a/go.sum
+++ b/go.sum
@@ -43,8 +43,8 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7
 github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
 github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
 github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
-github.com/blevesearch/zapx/v16 v16.0.13-0.20240410221809-91a5e17d9469 h1:jLO0IyEccNxI5V143PmZ122MEChtWk0NbXxX7RkyET8=
-github.com/blevesearch/zapx/v16 v16.0.13-0.20240410221809-91a5e17d9469/go.mod h1:ZlRi/XtzWhXIB+V5TSa2Ipq6H+lFl8HHD5Kxeg/i1zE=
+github.com/blevesearch/zapx/v16 v16.0.13-0.20240418135927-eeb2336467b1 h1:uRrhSNd1p8/y91CpYccRHDPxXnOyBnXa3PXdkMsdo8g=
+github.com/blevesearch/zapx/v16 v16.0.13-0.20240418135927-eeb2336467b1/go.mod h1:ZlRi/XtzWhXIB+V5TSa2Ipq6H+lFl8HHD5Kxeg/i1zE=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=
diff --git a/mapping/document.go b/mapping/document.go
index 73bb124d..3131f33b 100644
--- a/mapping/document.go
+++ b/mapping/document.go
@@ -443,6 +443,8 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
 					fieldMapping.processGeoShape(property, pathString, path, indexes, context)
 				} else if fieldMapping.Type == "geopoint" {
 					fieldMapping.processGeoPoint(property, pathString, path, indexes, context)
+				} else if fieldMapping.Type == "vector_base64" {
+					fieldMapping.processVectorBase64(property, pathString, path, indexes, context)
 				} else {
 					fieldMapping.processString(propertyValueString, pathString, path, indexes, context)
 				}
diff --git a/mapping/mapping_no_vectors.go b/mapping/mapping_no_vectors.go
index f9f35f57..90cb1e22 100644
--- a/mapping/mapping_no_vectors.go
+++ b/mapping/mapping_no_vectors.go
@@ -21,11 +21,20 @@ func NewVectorFieldMapping() *FieldMapping {
 	return nil
 }
 
+func NewVectorBase64FieldMapping() *FieldMapping {
+	return nil
+}
+
 func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
 	pathString string, path []string, indexes []uint64, context *walkContext) bool {
 	return false
 }
 
+func (fm *FieldMapping) processVectorBase64(propertyMightBeVector interface{},
+	pathString string, path []string, indexes []uint64, context *walkContext) {
+
+}
+
 // -----------------------------------------------------------------------------
 // document validation functions
 
diff --git a/mapping/mapping_vectors.go b/mapping/mapping_vectors.go
index a0b71260..3aed8a77 100644
--- a/mapping/mapping_vectors.go
+++ b/mapping/mapping_vectors.go
@@ -26,10 +26,11 @@ import (
 	index "github.com/blevesearch/bleve_index_api"
 )
 
-// Min and Max allowed dimensions for a vector field
-const (
+// Min and Max allowed dimensions for a vector field;
+// p.s must be set/updated at process init() _only_
+var (
 	MinVectorDims = 1
-	MaxVectorDims = 2048
+	MaxVectorDims = 4096
 )
 
 func NewVectorFieldMapping() *FieldMapping {
@@ -43,6 +44,17 @@ func NewVectorFieldMapping() *FieldMapping {
 	}
 }
 
+func NewVectorBase64FieldMapping() *FieldMapping {
+	return &FieldMapping{
+		Type:         "vector_base64",
+		Store:        false,
+		Index:        true,
+		IncludeInAll: false,
+		DocValues:    false,
+		SkipFreqNorm: true,
+	}
+}
+
 // validate and process a flat vector
 func processFlatVector(vecV reflect.Value, dims int) ([]float32, bool) {
 	if vecV.Len() != dims {
@@ -140,13 +152,28 @@ func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
 	return true
 }
 
+func (fm *FieldMapping) processVectorBase64(propertyMightBeVectorBase64 interface{},
+	pathString string, path []string, indexes []uint64, context *walkContext) {
+	encodedString, ok := propertyMightBeVectorBase64.(string)
+	if !ok {
+		return
+	}
+
+	propertyMightBeVector, err := document.DecodeVector(encodedString)
+	if err != nil {
+		return
+	}
+
+	fm.processVector(propertyMightBeVector, pathString, path, indexes, context)
+}
+
 // -----------------------------------------------------------------------------
 // document validation functions
 
 func validateFieldMapping(field *FieldMapping, parentName string,
 	fieldAliasCtx map[string]*FieldMapping) error {
 	switch field.Type {
-	case "vector":
+	case "vector", "vector_base64":
 		return validateVectorFieldAlias(field, parentName, fieldAliasCtx)
 	default: // non-vector field
 		return validateFieldType(field)
diff --git a/mapping_vector.go b/mapping_vector.go
index 59431386..c73dac9e 100644
--- a/mapping_vector.go
+++ b/mapping_vector.go
@@ -22,3 +22,7 @@ import "github.com/blevesearch/bleve/v2/mapping"
 func NewVectorFieldMapping() *mapping.FieldMapping {
 	return mapping.NewVectorFieldMapping()
 }
+
+func NewVectorBase64FieldMapping() *mapping.FieldMapping {
+	return mapping.NewVectorBase64FieldMapping()
+}
diff --git a/search_knn.go b/search_knn.go
index ba6b005f..cbeb3b45 100644
--- a/search_knn.go
+++ b/search_knn.go
@@ -23,12 +23,15 @@ import (
 	"fmt"
 	"sort"
 
+	"github.com/blevesearch/bleve/v2/document"
 	"github.com/blevesearch/bleve/v2/search"
 	"github.com/blevesearch/bleve/v2/search/collector"
 	"github.com/blevesearch/bleve/v2/search/query"
 	index "github.com/blevesearch/bleve_index_api"
 )
 
+const supportForVectorSearch = true
+
 type knnOperator string
 
 // Must be updated only at init
@@ -67,11 +70,13 @@ type SearchRequest struct {
 	sortFunc func(sort.Interface)
 }
 
+// Vector takes precedence over vectorBase64 in case both fields are given
 type KNNRequest struct {
-	Field  string       `json:"field"`
-	Vector []float32    `json:"vector"`
-	K      int64        `json:"k"`
-	Boost  *query.Boost `json:"boost,omitempty"`
+	Field        string       `json:"field"`
+	Vector       []float32    `json:"vector"`
+	VectorBase64 string       `json:"vector_base64"`
+	K            int64        `json:"k"`
+	Boost        *query.Boost `json:"boost,omitempty"`
 }
 
 func (r *SearchRequest) AddKNN(field string, vector []float32, k int64, boost float64) {
@@ -231,6 +236,16 @@ func validateKNN(req *SearchRequest) error {
 		if q == nil {
 			return fmt.Errorf("knn query cannot be nil")
 		}
+		if q.VectorBase64 != "" {
+			if q.Vector == nil {
+				vec, err := document.DecodeVector(q.VectorBase64)
+				if err != nil {
+					return err
+				}
+
+				q.Vector = vec
+			}
+		}
 		if q.K <= 0 || len(q.Vector) == 0 {
 			return fmt.Errorf("k must be greater than 0 and vector must be non-empty")
 		}
diff --git a/search_no_knn.go b/search_no_knn.go
index aff82611..bb72e15a 100644
--- a/search_no_knn.go
+++ b/search_no_knn.go
@@ -28,6 +28,8 @@ import (
 	index "github.com/blevesearch/bleve_index_api"
 )
 
+const supportForVectorSearch = false
+
 // A SearchRequest describes all the parameters
 // needed to search the index.
 // Query is required.


Test patch:
diff --git a/document/field_vector_base64_test.go b/document/field_vector_base64_test.go
new file mode 100644
index 00000000..ac4bd8d4
--- /dev/null
+++ b/document/field_vector_base64_test.go
@@ -0,0 +1,113 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build vectors
+// +build vectors
+
+package document
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"fmt"
+	"math/rand"
+	"testing"
+)
+
+func TestDecodeVector(t *testing.T) {
+	vec := make([]float32, 2048)
+	for i := range vec {
+		vec[i] = rand.Float32()
+	}
+
+	vecBytes := bytifyVec(vec)
+	encodedVec := base64.StdEncoding.EncodeToString(vecBytes)
+
+	decodedVec, err := DecodeVector(encodedVec)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(decodedVec) != len(vec) {
+		t.Errorf("Decoded vector dimensions not same as original vector dimensions")
+	}
+
+	for i := range vec {
+		if vec[i] != decodedVec[i] {
+			t.Errorf("Decoded vector not the same as original vector")
+		}
+	}
+}
+
+func BenchmarkDecodeVector128(b *testing.B) {
+	vec := make([]float32, 128)
+	for i := range vec {
+		vec[i] = rand.Float32()
+	}
+
+	vecBytes := bytifyVec(vec)
+	encodedVec := base64.StdEncoding.EncodeToString(vecBytes)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = DecodeVector(encodedVec)
+	}
+}
+
+func BenchmarkDecodeVector784(b *testing.B) {
+	vec := make([]float32, 784)
+	for i := range vec {
+		vec[i] = rand.Float32()
+	}
+
+	vecBytes := bytifyVec(vec)
+	encodedVec := base64.StdEncoding.EncodeToString(vecBytes)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = DecodeVector(encodedVec)
+	}
+}
+
+func BenchmarkDecodeVector1536(b *testing.B) {
+	vec := make([]float32, 1536)
+	for i := range vec {
+		vec[i] = rand.Float32()
+	}
+
+	vecBytes := bytifyVec(vec)
+	encodedVec := base64.StdEncoding.EncodeToString(vecBytes)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = DecodeVector(encodedVec)
+	}
+}
+
+func bytifyVec(vec []float32) []byte {
+
+	buf := new(bytes.Buffer)
+
+	for _, v := range vec {
+		err := binary.Write(buf, binary.LittleEndian, v)
+		if err != nil {
+			fmt.Println(err)
+		}
+	}
+
+	return buf.Bytes()
+}
diff --git a/index_test.go b/index_test.go
index c6cc0793..e5094440 100644
--- a/index_test.go
+++ b/index_test.go
@@ -401,9 +401,15 @@ func TestBytesRead(t *testing.T) {
 	}
 	stats, _ := idx.StatsMap()["index"].(map[string]interface{})
 	prevBytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64)
-	if prevBytesRead != 21639 && res.Cost == prevBytesRead {
-		t.Fatalf("expected bytes read for query string 21639, got %v",
-			prevBytesRead)
+
+	expectedBytesRead := uint64(21639)
+	if supportForVectorSearch {
+		expectedBytesRead = 22049
+	}
+
+	if prevBytesRead != expectedBytesRead && res.Cost == prevBytesRead {
+		t.Fatalf("expected bytes read for query string %v, got %v",
+			expectedBytesRead, prevBytesRead)
 	}
 
 	// subsequent queries on the same field results in lesser amount
@@ -553,8 +559,14 @@ func TestBytesReadStored(t *testing.T) {
 
 	stats, _ := idx.StatsMap()["index"].(map[string]interface{})
 	bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64)
-	if bytesRead != 11501 && bytesRead == res.Cost {
-		t.Fatalf("expected the bytes read stat to be around 11501, got %v", bytesRead)
+
+	expectedBytesRead := uint64(11501)
+	if supportForVectorSearch {
+		expectedBytesRead = 11911
+	}
+
+	if bytesRead != expectedBytesRead && bytesRead == res.Cost {
+		t.Fatalf("expected the bytes read stat to be around %v, got %v", expectedBytesRead, bytesRead)
 	}
 	prevBytesRead := bytesRead
 
@@ -624,8 +636,14 @@ func TestBytesReadStored(t *testing.T) {
 
 	stats, _ = idx1.StatsMap()["index"].(map[string]interface{})
 	bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
-	if bytesRead != 3687 && bytesRead == res.Cost {
-		t.Fatalf("expected the bytes read stat to be around 3687, got %v", bytesRead)
+
+	expectedBytesRead = uint64(3687)
+	if supportForVectorSearch {
+		expectedBytesRead = 4097
+	}
+
+	if bytesRead != expectedBytesRead && bytesRead == res.Cost {
+		t.Fatalf("expected the bytes read stat to be around %v, got %v", expectedBytesRead, bytesRead)
 	}
 	prevBytesRead = bytesRead
 
diff --git a/mapping/mapping_vectors_test.go b/mapping/mapping_vectors_test.go
index 8c04d03a..970a7de7 100644
--- a/mapping/mapping_vectors_test.go
+++ b/mapping/mapping_vectors_test.go
@@ -17,7 +17,9 @@
 
 package mapping
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestVectorFieldAliasValidation(t *testing.T) {
 	tests := []struct {
@@ -26,8 +28,8 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 		mappingStr string // index mapping json string
 
 		// expected output
-		expValidity bool   // validity of the mapping
-		errMsg      string // error message, given expValidity is false
+		expValidity bool     // validity of the mapping
+		errMsgs     []string // error message, given expValidity is false
 	}{
 		{
 			name: "test1",
@@ -52,7 +54,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: false,
-			errMsg:      `field: 'cityVec', invalid alias (different dimensions 4 and 3)`,
+			errMsgs:     []string{`field: 'cityVec', invalid alias (different dimensions 4 and 3)`},
 		},
 		{
 			name: "test2",
@@ -79,7 +81,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: false,
-			errMsg:      `field: 'cityVec', invalid alias (different similarity values dot_product and l2_norm)`,
+			errMsgs:     []string{`field: 'cityVec', invalid alias (different similarity values dot_product and l2_norm)`},
 		},
 		{
 			name: "test3",
@@ -104,7 +106,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: true,
-			errMsg:      "",
+			errMsgs:     []string{},
 		},
 		{
 			name: "test4",
@@ -134,7 +136,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: false,
-			errMsg:      `field: 'vecData', invalid alias (different dimensions 3 and 4)`,
+			errMsgs:     []string{`field: 'vecData', invalid alias (different dimensions 3 and 4)`, `field: 'vecData', invalid alias (different dimensions 4 and 3)`},
 		},
 		{
 			name: "test5",
@@ -170,7 +172,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: false,
-			errMsg:      `field: 'vecData', invalid alias (different dimensions 4 and 3)`,
+			errMsgs:     []string{`field: 'vecData', invalid alias (different dimensions 4 and 3)`},
 		},
 	}
 
@@ -189,9 +191,18 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					test.expValidity, isValid)
 			}
 
-			if !isValid && err.Error() != test.errMsg {
-				t.Fatalf("invalid error message, expected: %v, got: %v",
-					test.errMsg, err.Error())
+			if !isValid {
+				errStringMatched := false
+				for _, possibleErrMsg := range test.errMsgs {
+					if err.Error() == possibleErrMsg {
+						errStringMatched = true
+						break
+					}
+				}
+				if !errStringMatched {
+					t.Fatalf("invalid error message, expected one of: %v, got: %v",
+						test.errMsgs, err.Error())
+				}
 			}
 		})
 	}
diff --git a/search/scorer/scorer_knn_test.go b/search/scorer/scorer_knn_test.go
index 46a1de9b..1c005506 100644
--- a/search/scorer/scorer_knn_test.go
+++ b/search/scorer/scorer_knn_test.go
@@ -89,7 +89,7 @@ func TestKNNScorerExplanation(t *testing.T) {
 					Children: []*search.Explanation{
 						{
 							Value:   maxKNNScore,
-							Message: "vector(field(desc:one) with similarity_metric(l2_norm)=1.797693e+308",
+							Message: "vector(field(desc:one) with similarity_metric(l2_norm)=3.402823e+38",
 						},
 					},
 				},
diff --git a/search_knn_test.go b/search_knn_test.go
index b54ce5a9..24c677c4 100644
--- a/search_knn_test.go
+++ b/search_knn_test.go
@@ -19,6 +19,9 @@ package bleve
 
 import (
 	"archive/zip"
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"math"
@@ -130,13 +133,15 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 	}
 
 	index := NewIndexAlias()
+	var reqSort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: "content"}}
 	for testCaseNum, testCase := range testCases {
+		originalRequest := searchRequests[testCase.queryIndex]
 		for _, operator := range knnOperators {
-			index.indexes = make([]Index, 0)
 
-			query := searchRequests[testCase.queryIndex]
+			index.indexes = make([]Index, 0)
+			query := copySearchRequest(originalRequest, nil)
 			query.AddKNNOperator(operator)
-			query.Sort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: true, Field: "content"}}
+			query.Sort = reqSort.Copy()
 			query.Explain = true
 
 			nameToIndex := createPartitionedIndex(documents, index, 1, testCase.mapping, t, false)
@@ -150,7 +155,13 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 				t.Fatalf("test case #%d failed: expected control result hits to have valid `Index`", testCaseNum)
 			}
 			cleanUp(t, nameToIndex)
+
 			index.indexes = make([]Index, 0)
+			query = copySearchRequest(originalRequest, nil)
+			query.AddKNNOperator(operator)
+			query.Sort = reqSort.Copy()
+			query.Explain = true
+
 			nameToIndex = createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, false)
 			experimentalResult, err := index.Search(query)
 			if err != nil {
@@ -165,6 +176,11 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 			cleanUp(t, nameToIndex)
 
 			index.indexes = make([]Index, 0)
+			query = copySearchRequest(originalRequest, nil)
+			query.AddKNNOperator(operator)
+			query.Sort = reqSort.Copy()
+			query.Explain = true
+
 			nameToIndex = createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, true)
 			multiLevelIndexResult, err := index.Search(query)
 			if err != nil {
@@ -177,6 +193,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 			}
 			verifyResult(t, multiLevelIndexResult, experimentalResult, testCaseNum, false)
 			cleanUp(t, nameToIndex)
+
 		}
 	}
 
@@ -187,17 +204,14 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 		},
 	}
 
-	var sort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortField{Desc: false, Field: "content"}}
-
 	index = NewIndexAlias()
 	for testCaseNum, testCase := range testCases {
 		index.indexes = make([]Index, 0)
 		nameToIndex := createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, false)
 		originalRequest := searchRequests[testCase.queryIndex]
 		for _, operator := range knnOperators {
-
 			from, size := originalRequest.From, originalRequest.Size
-			query := copySearchRequest(searchRequests[testCase.queryIndex], nil)
+			query := copySearchRequest(originalRequest, nil)
 			query.AddKNNOperator(operator)
 			query.Explain = true
 			query.From = from
@@ -211,7 +225,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 
 			// 1. Sort And Facet are there
 			query.Facets = facets
-			query.Sort = sort
+			query.Sort = reqSort.Copy()
 
 			res1, err := index.Search(query)
 			if err != nil {
@@ -232,7 +246,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 
 			// 2. Sort is there, Facet is not there
 			query.Facets = nil
-			query.Sort = sort
+			query.Sort = reqSort.Copy()
 
 			res2, err := index.Search(query)
 			if err != nil {
@@ -279,7 +293,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 
 			// Test early exit fail case -> matchNone + facetRequest
 			query.Query = NewMatchNoneQuery()
-			query.Sort = sort
+			query.Sort = reqSort.Copy()
 			// control case
 			query.Facets = nil
 			res4Ctrl, err := index.Search(query)
@@ -321,7 +335,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 
 	// Test From + Size pagination for Hybrid Search (2-Phase)
 	query := copySearchRequest(searchRequests[4], nil)
-	query.Sort = sort
+	query.Sort = reqSort.Copy()
 	query.Facets = facets
 	query.Explain = true
 
@@ -330,7 +344,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 	// Test From + Size pagination for Early Exit Hybrid Search (1-Phase)
 	query = copySearchRequest(searchRequests[4], nil)
 	query.Query = NewMatchNoneQuery()
-	query.Sort = sort
+	query.Sort = reqSort.Copy()
 	query.Facets = nil
 	query.Explain = true
 
@@ -397,10 +411,181 @@ func min(a, b int) int {
 	return b
 }
 
+func TestVectorBase64Index(t *testing.T) {
+	dataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	documents := makeDatasetIntoDocuments(dataset)
+
+	_, searchRequestsCopy, err := readDatasetAndQueries(testInputCompressedFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, doc := range documents {
+		vec, ok := doc["vector"].([]float32)
+		if !ok {
+			t.Fatal("Typecasting vector to float array failed")
+		}
+
+		buf := new(bytes.Buffer)
+		for _, v := range vec {
+			err := binary.Write(buf, binary.LittleEndian, v)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		doc["vectorEncoded"] = base64.StdEncoding.EncodeToString(buf.Bytes())
+	}
+
+	for _, sr := range searchRequestsCopy {
+		for _, kr := range sr.KNN {
+			kr.Field = "vectorEncoded"
+		}
+	}
+
+	contentFM := NewTextFieldMapping()
+	contentFM.Analyzer = en.AnalyzerName
+
+	vecFML2 := mapping.NewVectorFieldMapping()
+	vecFML2.Dims = testDatasetDims
+	vecFML2.Similarity = index.EuclideanDistance
+
+	vecBFML2 := mapping.NewVectorBase64FieldMapping()
+	vecBFML2.Dims = testDatasetDims
+	vecBFML2.Similarity = index.EuclideanDistance
+
+	vecFMDot := mapping.NewVectorFieldMapping()
+	vecFMDot.Dims = testDatasetDims
+	vecFMDot.Similarity = index.CosineSimilarity
+
+	vecBFMDot := mapping.NewVectorBase64FieldMapping()
+	vecBFMDot.Dims = testDatasetDims
+	vecBFMDot.Similarity = index.CosineSimilarity
+
+	indexMappingL2 := NewIndexMapping()
+	indexMappingL2.DefaultMapping.AddFieldMappingsAt("content", contentFM)
+	indexMappingL2.DefaultMapping.AddFieldMappingsAt("vector", vecFML2)
+	indexMappingL2.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFML2)
+
+	indexMappingDot := NewIndexMapping()
+	indexMappingDot.DefaultMapping.AddFieldMappingsAt("content", contentFM)
+	indexMappingDot.DefaultMapping.AddFieldMappingsAt("vector", vecFMDot)
+	indexMappingDot.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFMDot)
+
+	tmpIndexPathL2 := createTmpIndexPath(t)
+	defer cleanupTmpIndexPath(t, tmpIndexPathL2)
+
+	tmpIndexPathDot := createTmpIndexPath(t)
+	defer cleanupTmpIndexPath(t, tmpIndexPathDot)
+
+	indexL2, err := New(tmpIndexPathL2, indexMappingL2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := indexL2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	indexDot, err := New(tmpIndexPathDot, indexMappingDot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := indexDot.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	batchL2 := indexL2.NewBatch()
+	batchDot := indexDot.NewBatch()
+
+	for _, doc := range documents {
+		err = batchL2.Index(doc["id"].(string), doc)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = batchDot.Index(doc["id"].(string), doc)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	err = indexL2.Batch(batchL2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = indexDot.Batch(batchDot)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i, _ := range searchRequests {
+		for _, operator := range knnOperators {
+			controlQuery := searchRequests[i]
+			testQuery := searchRequestsCopy[i]
+
+			controlQuery.AddKNNOperator(operator)
+			testQuery.AddKNNOperator(operator)
+
+			controlResultL2, err := indexL2.Search(controlQuery)
+			if err != nil {
+				t.Fatal(err)
+			}
+			testResultL2, err := indexL2.Search(testQuery)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if controlResultL2 != nil && testResultL2 != nil {
+				if len(controlResultL2.Hits) == len(testResultL2.Hits) {
+					for j, _ := range controlResultL2.Hits {
+						if controlResultL2.Hits[j].ID != testResultL2.Hits[j].ID {
+							t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, controlResultL2.Hits[j].ID, testResultL2.Hits[j].ID)
+						}
+					}
+				}
+			} else if (controlResultL2 == nil && testResultL2 != nil) ||
+				(controlResultL2 != nil && testResultL2 == nil) {
+				t.Fatalf("testcase %d failed: expected result %s, got result %s", i, controlResultL2, testResultL2)
+			}
+
+			controlResultDot, err := indexDot.Search(controlQuery)
+			if err != nil {
+				t.Fatal(err)
+			}
+			testResultDot, err := indexDot.Search(testQuery)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if controlResultDot != nil && testResultDot != nil {
+				if len(controlResultDot.Hits) == len(testResultDot.Hits) {
+					for j, _ := range controlResultDot.Hits {
+						if controlResultDot.Hits[j].ID != testResultDot.Hits[j].ID {
+							t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, controlResultDot.Hits[j].ID, testResultDot.Hits[j].ID)
+						}
+					}
+				}
+			} else if (controlResultDot == nil && testResultDot != nil) ||
+				(controlResultDot != nil && testResultDot == nil) {
+				t.Fatalf("testcase %d failed: expected result %s, got result %s", i, controlResultDot, testResultDot)
+			}
+		}
+	}
+}
+
 type testDocument struct {
 	ID      string    `json:"id"`
 	Content string    `json:"content"`
-	Vector  []float64 `json:"vector"`
+	Vector  []float32 `json:"vector"`
 }
 
 func readDatasetAndQueries(fileName string) ([]testDocument, []*SearchRequest, error) {
@@ -745,6 +930,8 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 	indexMappingDotProduct.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping)
 	indexMappingDotProduct.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMappingDot)
 
+	var reqSort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: "content"}}
+
 	testCases := []struct {
 		numSegments int
 		queryIndex  int
@@ -852,6 +1039,7 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 		},
 	}
 	for testCaseNum, testCase := range testCases {
+		originalRequest := searchRequests[testCase.queryIndex]
 		for _, operator := range knnOperators {
 			// run single segment test first
 			tmpIndexPath := createTmpIndexPath(t)
@@ -859,8 +1047,8 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			query := searchRequests[testCase.queryIndex]
-			query.Sort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: "content"}}
+			query := copySearchRequest(originalRequest, nil)
+			query.Sort = reqSort.Copy()
 			query.AddKNNOperator(operator)
 			query.Explain = true
 
@@ -882,7 +1070,13 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 				t.Fatalf("test case #%d failed: expected control result hits to have valid `Index`", testCaseNum)
 			}
 			if testCase.scoreValue == "none" {
+
+				query := copySearchRequest(originalRequest, nil)
+				query.Sort = reqSort.Copy()
+				query.AddKNNOperator(operator)
+				query.Explain = true
 				query.Score = testCase.scoreValue
+
 				expectedResultScoreNone, err := index.Search(query)
 				if err != nil {
 					cleanUp(t, nameToIndex)
@@ -893,7 +1087,6 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 					t.Fatalf("test case #%d failed: expected score none hits to have valid `Index`", testCaseNum)
 				}
 				verifyResult(t, controlResult, expectedResultScoreNone, testCaseNum, true)
-				query.Score = ""
 			}
 			cleanUp(t, nameToIndex)
 
@@ -910,6 +1103,12 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 				cleanUp(t, nameToIndex)
 				t.Fatal(err)
 			}
+
+			query = copySearchRequest(originalRequest, nil)
+			query.Sort = reqSort.Copy()
+			query.AddKNNOperator(operator)
+			query.Explain = true
+
 			experimentalResult, err := index.Search(query)
 			if err != nil {
 				cleanUp(t, nameToIndex)