Repository: blevesearch/bleve
Pull request #1971 (2024-01-25 15:21:07)

MB-60945 - Add scorch event for pre merge check
This PR adds an event to check whether a merge can proceed based on criteria set by the caller(eg. cbft). If not, it's aborted and the merger loop continues its next iteration. 
Also includes minor refactor to the function signature of the event callback registry to accommodate the new event type.


metonymic-smokey: TODO Will investigate other metrics for tracking memory used during a merge.

metonymic-smokey: Potential downsides:
1. Since there is no estimate for the memory used by a merge, it's possible that merges which wouldn't take up very high memory too would be unnecessarily blocked.



Patch:
diff --git a/analysis/datetime/timestamp/microseconds/microseconds.go b/analysis/datetime/timestamp/microseconds/microseconds.go
index 3efed0ed..a0e2c949 100644
--- a/analysis/datetime/timestamp/microseconds/microseconds.go
+++ b/analysis/datetime/timestamp/microseconds/microseconds.go
@@ -40,7 +40,7 @@ func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error)
 	if timestamp < minBound || timestamp > maxBound {
 		return time.Time{}, "", analysis.ErrInvalidTimestampRange
 	}
-	return time.UnixMicro(timestamp), "", nil
+	return time.UnixMicro(timestamp), Name, nil
 }
 
 func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {
diff --git a/analysis/datetime/timestamp/milliseconds/milliseconds.go b/analysis/datetime/timestamp/milliseconds/milliseconds.go
index 790153d2..63826b45 100644
--- a/analysis/datetime/timestamp/milliseconds/milliseconds.go
+++ b/analysis/datetime/timestamp/milliseconds/milliseconds.go
@@ -40,7 +40,7 @@ func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error)
 	if timestamp < minBound || timestamp > maxBound {
 		return time.Time{}, "", analysis.ErrInvalidTimestampRange
 	}
-	return time.UnixMilli(timestamp), "", nil
+	return time.UnixMilli(timestamp), Name, nil
 }
 
 func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {
diff --git a/analysis/datetime/timestamp/nanoseconds/nanoseconds.go b/analysis/datetime/timestamp/nanoseconds/nanoseconds.go
index de318f3f..8bb1ab1b 100644
--- a/analysis/datetime/timestamp/nanoseconds/nanoseconds.go
+++ b/analysis/datetime/timestamp/nanoseconds/nanoseconds.go
@@ -40,7 +40,7 @@ func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error)
 	if timestamp < minBound || timestamp > maxBound {
 		return time.Time{}, "", analysis.ErrInvalidTimestampRange
 	}
-	return time.Unix(0, timestamp), "", nil
+	return time.Unix(0, timestamp), Name, nil
 }
 
 func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {
diff --git a/analysis/datetime/timestamp/seconds/seconds.go b/analysis/datetime/timestamp/seconds/seconds.go
index 8d3ee3b4..58e947c8 100644
--- a/analysis/datetime/timestamp/seconds/seconds.go
+++ b/analysis/datetime/timestamp/seconds/seconds.go
@@ -40,7 +40,7 @@ func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error)
 	if timestamp < minBound || timestamp > maxBound {
 		return time.Time{}, "", analysis.ErrInvalidTimestampRange
 	}
-	return time.Unix(timestamp, 0), "", nil
+	return time.Unix(timestamp, 0), Name, nil
 }
 
 func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {
diff --git a/docs/vectors.md b/docs/vectors.md
index b4264d29..bddf239e 100644
--- a/docs/vectors.md
+++ b/docs/vectors.md
@@ -15,9 +15,9 @@
 
 ```go
 doc := struct{
-    Id   string
-    Text string
-    Vec  []float32
+    Id   string    `json:"id"`
+    Text string    `json:"text"`
+    Vec  []float32 `json:"vec"`
 }{
     Id:   "example",
     Text: "hello from united states",
diff --git a/document/field_vector_base64.go b/document/field_vector_base64.go
new file mode 100644
index 00000000..67825960
--- /dev/null
+++ b/document/field_vector_base64.go
@@ -0,0 +1,149 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build vectors
+// +build vectors
+
+package document
+
+import (
+	"encoding/base64"
+	"encoding/binary"
+	"fmt"
+	"math"
+
+	"github.com/blevesearch/bleve/v2/size"
+	"github.com/blevesearch/bleve/v2/util"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+type VectorBase64Field struct {
+	vectorField    *VectorField
+	base64Encoding string
+}
+
+func (n *VectorBase64Field) Size() int {
+	return n.vectorField.Size()
+}
+
+func (n *VectorBase64Field) Name() string {
+	return n.vectorField.Name()
+}
+
+func (n *VectorBase64Field) ArrayPositions() []uint64 {
+	return n.vectorField.ArrayPositions()
+}
+
+func (n *VectorBase64Field) Options() index.FieldIndexingOptions {
+	return n.vectorField.Options()
+}
+
+func (n *VectorBase64Field) NumPlainTextBytes() uint64 {
+	return n.vectorField.NumPlainTextBytes()
+}
+
+func (n *VectorBase64Field) AnalyzedLength() int {
+	return n.vectorField.AnalyzedLength()
+}
+
+func (n *VectorBase64Field) EncodedFieldType() byte {
+	return 'e'
+}
+
+func (n *VectorBase64Field) AnalyzedTokenFrequencies() index.TokenFrequencies {
+	return n.vectorField.AnalyzedTokenFrequencies()
+}
+
+func (n *VectorBase64Field) Analyze() {
+}
+
+func (n *VectorBase64Field) Value() []byte {
+	return n.vectorField.Value()
+}
+
+func (n *VectorBase64Field) GoString() string {
+	return fmt.Sprintf("&document.vectorFieldBase64Field{Name:%s, Options: %s, "+
+		"Value: %+v}", n.vectorField.Name(), n.vectorField.Options(), n.vectorField.Value())
+}
+
+// For the sake of not polluting the API, we are keeping arrayPositions as a
+// parameter, but it is not used.
+func NewVectorBase64Field(name string, arrayPositions []uint64, vectorBase64 string,
+	dims int, similarity, vectorIndexOptimizedFor string) (*VectorBase64Field, error) {
+
+	decodedVector, err := DecodeVector(vectorBase64)
+	if err != nil {
+		return nil, err
+	}
+
+	return &VectorBase64Field{
+		vectorField: NewVectorFieldWithIndexingOptions(name, arrayPositions,
+			decodedVector, dims, similarity,
+			vectorIndexOptimizedFor, DefaultVectorIndexingOptions),
+
+		base64Encoding: vectorBase64,
+	}, nil
+}
+
+// This function takes a base64 encoded string and decodes it into
+// a vector.
+func DecodeVector(encodedValue string) ([]float32, error) {
+	// We first decode the encoded string into a byte array.
+	decodedString, err := base64.StdEncoding.DecodeString(encodedValue)
+	if err != nil {
+		return nil, err
+	}
+
+	// The array is expected to be divisible by 4 because each float32
+	// should occupy 4 bytes
+	if len(decodedString)%size.SizeOfFloat32 != 0 {
+		return nil, fmt.Errorf("decoded byte array not divisible by %d", size.SizeOfFloat32)
+	}
+	dims := int(len(decodedString) / size.SizeOfFloat32)
+
+	if dims <= 0 {
+		return nil, fmt.Errorf("unable to decode encoded vector")
+	}
+
+	decodedVector := make([]float32, dims)
+
+	// We iterate through the array 4 bytes at a time and convert each of
+	// them to a float32 value by reading them in a little endian notation
+	for i := 0; i < dims; i++ {
+		bytes := decodedString[i*size.SizeOfFloat32 : (i+1)*size.SizeOfFloat32]
+		entry := math.Float32frombits(binary.LittleEndian.Uint32(bytes))
+		if !util.IsValidFloat32(float64(entry)) {
+			return nil, fmt.Errorf("invalid float32 value: %f", entry)
+		}
+		decodedVector[i] = entry
+	}
+
+	return decodedVector, nil
+}
+
+func (n *VectorBase64Field) Vector() []float32 {
+	return n.vectorField.Vector()
+}
+
+func (n *VectorBase64Field) Dims() int {
+	return n.vectorField.Dims()
+}
+
+func (n *VectorBase64Field) Similarity() string {
+	return n.vectorField.Similarity()
+}
+
+func (n *VectorBase64Field) IndexOptimizedFor() string {
+	return n.vectorField.IndexOptimizedFor()
+}
diff --git a/go.mod b/go.mod
index dbbb63af..68f0f1a7 100644
--- a/go.mod
+++ b/go.mod
@@ -3,15 +3,15 @@ module github.com/blevesearch/bleve/v2
 go 1.21
 
 require (
-	github.com/RoaringBitmap/roaring v1.9.1
+	github.com/RoaringBitmap/roaring v1.9.3
 	github.com/bits-and-blooms/bitset v1.12.0
-	github.com/blevesearch/bleve_index_api v1.1.6
+	github.com/blevesearch/bleve_index_api v1.1.7
 	github.com/blevesearch/geo v0.1.20
 	github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/blevesearch/go-porterstemmer v1.0.3
 	github.com/blevesearch/goleveldb v1.0.1
 	github.com/blevesearch/gtreap v0.1.1
-	github.com/blevesearch/scorch_segment_api/v2 v2.2.11
+	github.com/blevesearch/scorch_segment_api/v2 v2.2.12
 	github.com/blevesearch/segment v0.9.1
 	github.com/blevesearch/snowball v0.6.1
 	github.com/blevesearch/snowballstem v0.9.0
@@ -23,7 +23,7 @@ require (
 	github.com/blevesearch/zapx/v13 v13.3.10
 	github.com/blevesearch/zapx/v14 v14.3.10
 	github.com/blevesearch/zapx/v15 v15.3.13
-	github.com/blevesearch/zapx/v16 v16.0.13-0.20240410221809-91a5e17d9469
+	github.com/blevesearch/zapx/v16 v16.0.13-0.20240514143026-a9fced173441
 	github.com/couchbase/moss v0.2.0
 	github.com/golang/protobuf v1.3.2
 	github.com/spf13/cobra v1.7.0
@@ -32,7 +32,7 @@ require (
 )
 
 require (
-	github.com/blevesearch/go-faiss v1.0.13 // indirect
+	github.com/blevesearch/go-faiss v1.0.15 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
 	github.com/couchbase/ghistogram v0.1.0 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
diff --git a/go.sum b/go.sum
index b75d6cd8..d0ab2a63 100644
--- a/go.sum
+++ b/go.sum
@@ -1,13 +1,13 @@
-github.com/RoaringBitmap/roaring v1.9.1 h1:LXcSqGGGMKm+KAzUyWn7ZeREqoOkoMX+KwLOK1thc4I=
-github.com/RoaringBitmap/roaring v1.9.1/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
+github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM=
+github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
 github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
 github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
-github.com/blevesearch/bleve_index_api v1.1.6 h1:orkqDFCBuNU2oHW9hN2YEJmet+TE9orml3FCGbl1cKk=
-github.com/blevesearch/bleve_index_api v1.1.6/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
+github.com/blevesearch/bleve_index_api v1.1.7 h1:KvvKVn/9+dKCEJHL6flDH3NCiXeKnK3P85e4EOgVjdM=
+github.com/blevesearch/bleve_index_api v1.1.7/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
 github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
 github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
-github.com/blevesearch/go-faiss v1.0.13 h1:zfFs7ZYD0NqXVSY37j0JZjZT1BhE9AE4peJfcx/NB4A=
-github.com/blevesearch/go-faiss v1.0.13/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
+github.com/blevesearch/go-faiss v1.0.15 h1:aBrj6fwLuY8CkhECFbvkc4qhLTkrYI84QoEaw9z1jMI=
+github.com/blevesearch/go-faiss v1.0.15/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
@@ -19,8 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
 github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
 github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.2.11 h1:nb5KTeIhDUu+Ka6He7xXvOXcJOt9Db7c3Vy2ptqJQRs=
-github.com/blevesearch/scorch_segment_api/v2 v2.2.11/go.mod h1:QVakeAECt+Fxe+zu0A4V1bgPdqNeC93wQvzaXDF3NPo=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.12 h1:LuBJ4AfKlMsyi/8xFhxIArEnrlAcI3qGTvzwCTdfKgA=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.12/go.mod h1:80vK+flQwZlyj4UralqpicXVQWFgkKTzdqbSvOGHhWs=
 github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
@@ -43,8 +43,8 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7
 github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
 github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
 github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
-github.com/blevesearch/zapx/v16 v16.0.13-0.20240410221809-91a5e17d9469 h1:jLO0IyEccNxI5V143PmZ122MEChtWk0NbXxX7RkyET8=
-github.com/blevesearch/zapx/v16 v16.0.13-0.20240410221809-91a5e17d9469/go.mod h1:ZlRi/XtzWhXIB+V5TSa2Ipq6H+lFl8HHD5Kxeg/i1zE=
+github.com/blevesearch/zapx/v16 v16.0.13-0.20240514143026-a9fced173441 h1:SJhTyViAfEQBT6icz7omIHgnVVNqscpUAlfINzRmNn4=
+github.com/blevesearch/zapx/v16 v16.0.13-0.20240514143026-a9fced173441/go.mod h1:Viwste/2ooJPzsLNuncnvyTkbAL2iAgM4ge5agqkdPg=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=
diff --git a/http/doc_get.go b/http/doc_get.go
index 328955eb..ac783a16 100644
--- a/http/doc_get.go
+++ b/http/doc_get.go
@@ -18,7 +18,12 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"time"
 
+	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds"
+	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds"
+	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds"
+	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds"
 	index "github.com/blevesearch/bleve_index_api"
 )
 
@@ -91,10 +96,24 @@ func (h *DocGetHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			d, layout, err := field.DateTime()
 			if err == nil {
 				if layout == "" {
-					// layout not set probably means it was indexed as a timestamp
-					newval = strconv.FormatInt(d.UnixNano(), 10)
+					// missing layout means we fallback to
+					// the default layout which is RFC3339
+					newval = d.Format(time.RFC3339)
 				} else {
-					newval = d.Format(layout)
+					// the layout here can now either be representative
+					// of an actual layout or a timestamp
+					switch layout {
+					case seconds.Name:
+						newval = strconv.FormatInt(d.Unix(), 10)
+					case milliseconds.Name:
+						newval = strconv.FormatInt(d.UnixMilli(), 10)
+					case microseconds.Name:
+						newval = strconv.FormatInt(d.UnixMicro(), 10)
+					case nanoseconds.Name:
+						newval = strconv.FormatInt(d.UnixNano(), 10)
+					default:
+						newval = d.Format(layout)
+					}
 				}
 			}
 		}
diff --git a/index/scorch/README.md b/index/scorch/README.md
index 9794aed7..fe2abde5 100644
--- a/index/scorch/README.md
+++ b/index/scorch/README.md
@@ -218,7 +218,7 @@ A term search for term T in field F will look something like this:
   }
 ```
 
-The searchResultPostings will be a new implementation of the TermFieldReader inteface.
+The searchResultPostings will be a new implementation of the TermFieldReader interface.
 
 As a reminder this interface is:
 
diff --git a/index/scorch/event.go b/index/scorch/event.go
index 31c9e80c..0f653ccf 100644
--- a/index/scorch/event.go
+++ b/index/scorch/event.go
@@ -22,7 +22,8 @@ var RegistryAsyncErrorCallbacks = map[string]func(error, string){}
 
 // RegistryEventCallbacks should be treated as read-only after
 // process init()'ialization.
-var RegistryEventCallbacks = map[string]func(Event){}
+// In the event of not having a callback, these return true.
+var RegistryEventCallbacks = map[string]func(Event) bool{}
 
 // Event represents the information provided in an OnEvent() callback.
 type Event struct {
@@ -62,3 +63,7 @@ var EventKindMergeTaskIntroductionStart = EventKind(7)
 // EventKindMergeTaskIntroduction is fired when the merger has completed
 // the introduction of merged segment from a single merge task.
 var EventKindMergeTaskIntroduction = EventKind(8)
+
+// EventKindPreMergeCheck is fired before the merge begins to check if
+// the caller should proceed with the merge.
+var EventKindPreMergeCheck = EventKind(9)
diff --git a/index/scorch/merge.go b/index/scorch/merge.go
index 339ec596..b74504ca 100644
--- a/index/scorch/merge.go
+++ b/index/scorch/merge.go
@@ -72,6 +72,17 @@ OUTER:
 				ctrlMsg = ctrlMsgDflt
 			}
 			if ctrlMsg != nil {
+				continueMerge := s.fireEvent(EventKindPreMergeCheck, 0)
+				// The default, if there's no handler, is to continue the merge.
+				if !continueMerge {
+					// If it's decided that this merge can't take place now,
+					// begin the merge process all over again.
+					// Retry instead of blocking/waiting here since a long wait
+					// can result in more segments introduced i.e. s.root will
+					// be updated.
+					continue OUTER
+				}
+
 				startTime := time.Now()
 
 				// lets get started
diff --git a/index/scorch/mergeplan/merge_plan.go b/index/scorch/mergeplan/merge_plan.go
index 75235066..b4e7a372 100644
--- a/index/scorch/mergeplan/merge_plan.go
+++ b/index/scorch/mergeplan/merge_plan.go
@@ -37,6 +37,11 @@ type Segment interface {
 	// Size of the live data of the segment; i.e., FullSize() minus
 	// any logical deletions.
 	LiveSize() int64
+
+	HasVector() bool
+
+	// Size of the persisted segment file.
+	FileSize() int64
 }
 
 // Plan() will functionally compute a merge plan.  A segment will be
@@ -76,6 +81,11 @@ type MergePlanOptions struct {
 	// planner’s predicted sizes.
 	MaxSegmentSize int64
 
+	// Max size (in bytes) of the persisted segment file that contains the
+	// vectors.  This is used to prevent merging of segments that
+	// contain vectors that are too large.
+	MaxSegmentFileSize int64
+
 	// The growth factor for each tier in a staircase of idealized
 	// segments computed by CalcBudget().
 	TierGrowth float64
@@ -128,6 +138,7 @@ var ErrMaxSegmentSizeTooLarge = errors.New("MaxSegmentSize exceeds the size limi
 var DefaultMergePlanOptions = MergePlanOptions{
 	MaxSegmentsPerTier:   10,
 	MaxSegmentSize:       5000000,
+	MaxSegmentFileSize:   4000000000, // 4GB
 	TierGrowth:           10.0,
 	SegmentsPerMergeTask: 10,
 	FloorSegmentSize:     2000,
@@ -170,8 +181,17 @@ func plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) {
 			minLiveSize = segment.LiveSize()
 		}
 
+		isEligible := segment.LiveSize() < o.MaxSegmentSize/2
+		// An eligible segment (based on #documents) may be too large
+		// and thus need a stricter check based on the file size.
+		// This is particularly important for segments that contain
+		// vectors.
+		if isEligible && segment.HasVector() && o.MaxSegmentFileSize > 0 {
+			isEligible = segment.FileSize() < o.MaxSegmentFileSize/2
+		}
+
 		// Only small-enough segments are eligible.
-		if segment.LiveSize() < o.MaxSegmentSize/2 {
+		if isEligible {
 			eligibles = append(eligibles, segment)
 			eligiblesLiveSize += segment.LiveSize()
 		}
@@ -215,14 +235,25 @@ func plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) {
 		for startIdx := 0; startIdx < len(eligibles); startIdx++ {
 			var roster []Segment
 			var rosterLiveSize int64
+			var rosterFileSize int64 // useful for segments with vectors
 
 			for idx := startIdx; idx < len(eligibles) && len(roster) < o.SegmentsPerMergeTask; idx++ {
 				eligible := eligibles[idx]
 
-				if rosterLiveSize+eligible.LiveSize() < o.MaxSegmentSize {
-					roster = append(roster, eligible)
-					rosterLiveSize += eligible.LiveSize()
+				if rosterLiveSize+eligible.LiveSize() >= o.MaxSegmentSize {
+					continue
 				}
+
+				if eligible.HasVector() {
+					efs := eligible.FileSize()
+					if rosterFileSize+efs >= o.MaxSegmentFileSize {
+						continue
+					}
+					rosterFileSize += efs
+				}
+
+				roster = append(roster, eligible)
+				rosterLiveSize += eligible.LiveSize()
 			}
 
 			if len(roster) > 0 {
diff --git a/index/scorch/persister.go b/index/scorch/persister.go
index afd518dd..ed0fc40e 100644
--- a/index/scorch/persister.go
+++ b/index/scorch/persister.go
@@ -549,11 +549,14 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string,
 		val := make([]byte, 8)
 		bytesWritten := atomic.LoadUint64(&snapshot.parent.stats.TotBytesWrittenAtIndexTime)
 		binary.LittleEndian.PutUint64(val, bytesWritten)
-		internalBucket.Put(TotBytesWrittenKey, val)
+		err = internalBucket.Put(TotBytesWrittenKey, val)
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 
-	var filenames []string
-	newSegmentPaths := make(map[uint64]string)
+	filenames := make([]string, 0, len(snapshot.segment))
+	newSegmentPaths := make(map[uint64]string, len(snapshot.segment))
 
 	// first ensure that each segment in this snapshot has been persisted
 	for _, segmentSnapshot := range snapshot.segment {
@@ -982,7 +985,7 @@ func getTimeSeriesSnapshots(maxDataPoints int, interval time.Duration,
 	return ptr, rv
 }
 
-// getProtectedEpochs aims to fetch the epochs keep based on a timestamp basis.
+// getProtectedSnapshots aims to fetch the epochs keep based on a timestamp basis.
 // It tries to get NumSnapshotsToKeep snapshots, each of which are separated
 // by a time duration of RollbackSamplingInterval.
 func getProtectedSnapshots(rollbackSamplingInterval time.Duration,
@@ -1133,7 +1136,7 @@ func (s *Scorch) removeOldZapFiles() error {
 	for _, f := range files {
 		fname := f.Name()
 		if filepath.Ext(fname) == ".zap" {
-			if _, exists := liveFileNames[fname]; !exists && !s.ineligibleForRemoval[fname] {
+			if _, exists := liveFileNames[fname]; !exists && !s.ineligibleForRemoval[fname] && (s.copyScheduled[fname] <= 0) {
 				err := os.Remove(s.path + string(os.PathSeparator) + fname)
 				if err != nil {
 					log.Printf("got err removing file: %s, err: %v", fname, err)
diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go
index 2e6435ee..7966d844 100644
--- a/index/scorch/scorch.go
+++ b/index/scorch/scorch.go
@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -49,6 +50,7 @@ type Scorch struct {
 	unsafeBatch bool
 
 	rootLock             sync.RWMutex
+
 	root                 *IndexSnapshot // holds 1 ref-count on the root
 	rootPersisted        []chan error   // closed when root is persisted
 	persistedCallbacks   []index.BatchCallback
@@ -56,6 +58,12 @@ type Scorch struct {
 	eligibleForRemoval   []uint64        // Index snapshot epochs that are safe to GC.
 	ineligibleForRemoval map[string]bool // Filenames that should not be GC'ed yet.
 
+	// keeps track of segments scheduled for online copy/backup operation. Each segment's filename maps to
+	// the count of copy schedules. Segments with non-zero counts are protected from removal by the cleanup
+	// operation. Counts decrement upon successful copy, allowing removal of segments with zero or absent counts.
+	// must be accessed within the rootLock as it is accessed by the asynchronous cleanup routine.
+	copyScheduled map[string]int
+
 	numSnapshotsToKeep       int
 	rollbackRetentionFactor  float64
 	checkPoints              []*snapshotMetaData
@@ -69,7 +77,7 @@ type Scorch struct {
 	rootBolt                 *bolt.DB
 	asyncTasks               sync.WaitGroup
 
-	onEvent      func(event Event)
+	onEvent      func(event Event) bool
 	onAsyncError func(err error, path string)
 
 	forceMergeRequestCh chan *mergerCtrl
@@ -112,6 +120,7 @@ func NewScorch(storeName string,
 		ineligibleForRemoval: map[string]bool{},
 		forceMergeRequestCh:  make(chan *mergerCtrl, 1),
 		segPlugin:            defaultSegmentPlugin,
+		copyScheduled:        map[string]int{},
 	}
 
 	forcedSegmentType, forcedSegmentVersion, err := configForceSegmentTypeVersion(config)
@@ -175,12 +184,14 @@ func (s *Scorch) NumEventsBlocking() uint64 {
 	return eventsStarted - eventsCompleted
 }
 
-func (s *Scorch) fireEvent(kind EventKind, dur time.Duration) {
+func (s *Scorch) fireEvent(kind EventKind, dur time.Duration) bool {
+	res := true
 	if s.onEvent != nil {
 		atomic.AddUint64(&s.stats.TotEventTriggerStarted, 1)
-		s.onEvent(Event{Kind: kind, Scorch: s, Duration: dur})
+		res = s.onEvent(Event{Kind: kind, Scorch: s, Duration: dur})
 		atomic.AddUint64(&s.stats.TotEventTriggerCompleted, 1)
 	}
+	return res
 }
 
 func (s *Scorch) fireAsyncError(err error) {
@@ -822,6 +833,10 @@ func (fs *fieldStats) Aggregate(stats segment.FieldStats) {
 
 // Returns the stats map
 func (fs *fieldStats) Fetch() map[string]map[string]uint64 {
+	if fs == nil {
+		return nil
+	}
+
 	return fs.statMap
 }
 
@@ -832,3 +847,34 @@ func newFieldStats() *fieldStats {
 	}
 	return rv
 }
+
+// CopyReader returns a low-level accessor for index data, ensuring persisted segments
+// remain on disk for backup, preventing race conditions with the persister/merger cleanup.
+// Close the reader after backup to allow segment removal by the persister/merger.
+func (s *Scorch) CopyReader() index.CopyReader {
+	s.rootLock.Lock()
+	rv := s.root
+	if rv != nil {
+		rv.AddRef()
+		var fileName string
+		// schedule a backup for all the segments from the root. Note that the
+		// both the unpersisted and persisted segments are scheduled for backup.
+		// because during the backup, the unpersisted segments may get persisted and
+		// hence we need to protect both the unpersisted and persisted segments from removal
+		// by the cleanup routine during the online backup
+		for _, seg := range rv.segment {
+			if perSeg, ok := seg.segment.(segment.PersistedSegment); ok {
+				// segment is persisted
+				fileName = filepath.Base(perSeg.Path())
+			} else {
+				// segment is not persisted
+				// the name of the segment file that is generated if the
+				// the segment is persisted in the future.
+				fileName = zapFileName(seg.id)
+			}
+			rv.parent.copyScheduled[fileName]++
+		}
+	}
+	s.rootLock.Unlock()
+	return rv
+}
diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go
index 59828e87..f0e7ae1c 100644
--- a/index/scorch/snapshot_index.go
+++ b/index/scorch/snapshot_index.go
@@ -905,3 +905,26 @@ func (is *IndexSnapshot) GetSpatialAnalyzerPlugin(typ string) (
 	}
 	return rv, nil
 }
+
+func (is *IndexSnapshot) CloseCopyReader() error {
+	// first unmark the segments that were marked for backup by this index snapshot
+	is.parent.rootLock.Lock()
+	for _, seg := range is.segment {
+		var fileName string
+		if perSeg, ok := seg.segment.(segment.PersistedSegment); ok {
+			// segment is persisted
+			fileName = filepath.Base(perSeg.Path())
+		} else {
+			// segment is not persisted
+			// the name of the segment file that is generated if the
+			// the segment is persisted in the future.
+			fileName = zapFileName(seg.id)
+		}
+		if is.parent.copyScheduled[fileName]--; is.parent.copyScheduled[fileName] <= 0 {
+			delete(is.parent.copyScheduled, fileName)
+		}
+	}
+	is.parent.rootLock.Unlock()
+	// close the index snapshot normally
+	return is.Close()
+}
diff --git a/index/scorch/snapshot_segment.go b/index/scorch/snapshot_segment.go
index 1c14af72..96e59a31 100644
--- a/index/scorch/snapshot_segment.go
+++ b/index/scorch/snapshot_segment.go
@@ -16,6 +16,7 @@ package scorch
 
 import (
 	"bytes"
+	"os"
 	"sync"
 	"sync/atomic"
 
@@ -66,6 +67,31 @@ func (s *SegmentSnapshot) LiveSize() int64 {
 	return int64(s.Count())
 }
 
+func (s *SegmentSnapshot) HasVector() bool {
+	// number of vectors, for each vector field in the segment
+	numVecs := s.stats.Fetch()["num_vectors"]
+	return len(numVecs) > 0
+}
+
+func (s *SegmentSnapshot) FileSize() int64 {
+	ps, ok := s.segment.(segment.PersistedSegment)
+	if !ok {
+		return 0
+	}
+
+	path := ps.Path()
+	if path == "" {
+		return 0
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0
+	}
+
+	return fi.Size()
+}
+
 func (s *SegmentSnapshot) Close() error {
 	return s.segment.Close()
 }
diff --git a/index_impl.go b/index_impl.go
index a5254735..55212e3e 100644
--- a/index_impl.go
+++ b/index_impl.go
@@ -25,6 +25,10 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds"
+	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds"
+	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds"
+	"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds"
 	"github.com/blevesearch/bleve/v2/document"
 	"github.com/blevesearch/bleve/v2/index/scorch"
 	"github.com/blevesearch/bleve/v2/index/upsidedown"
@@ -738,10 +742,28 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,
 								datetime, layout, err := docF.DateTime()
 								if err == nil {
 									if layout == "" {
-										// layout not set probably means it was indexed as a timestamp
-										value = strconv.FormatInt(datetime.UnixNano(), 10)
+										// missing layout means we fallback to
+										// the default layout which is RFC3339
+										value = datetime.Format(time.RFC3339)
 									} else {
-										value = datetime.Format(layout)
+										// the layout here can now either be representative
+										// of an actual datetime layout or a timestamp
+										switch layout {
+										case seconds.Name:
+											value = strconv.FormatInt(datetime.Unix(), 10)
+										case milliseconds.Name:
+											value = strconv.FormatInt(datetime.UnixMilli(), 10)
+										case microseconds.Name:
+											value = strconv.FormatInt(datetime.UnixMicro(), 10)
+										case nanoseconds.Name:
+											value = strconv.FormatInt(datetime.UnixNano(), 10)
+										default:
+											// the layout for formatting the date to a string
+											// is provided by a datetime parser which is not
+											// handling the timestamp case, hence the layout
+											// can be directly used to format the date
+											value = datetime.Format(layout)
+										}
 									}
 								}
 							case index.BooleanField:
@@ -1052,22 +1074,23 @@ func (i *indexImpl) CopyTo(d index.Directory) (err error) {
 		return ErrorIndexClosed
 	}
 
-	indexReader, err := i.i.Reader()
-	if err != nil {
-		return err
+	copyIndex, ok := i.i.(index.CopyIndex)
+	if !ok {
+		return fmt.Errorf("index implementation does not support copy reader")
+	}
+
+	copyReader := copyIndex.CopyReader()
+	if copyReader == nil {
+		return fmt.Errorf("index's copyReader is nil")
 	}
+
 	defer func() {
-		if cerr := indexReader.Close(); err == nil && cerr != nil {
+		if cerr := copyReader.CloseCopyReader(); err == nil && cerr != nil {
 			err = cerr
 		}
 	}()
 
-	irc, ok := indexReader.(IndexCopyable)
-	if !ok {
-		return fmt.Errorf("index implementation does not support copy")
-	}
-
-	err = irc.CopyTo(d)
+	err = copyReader.CopyTo(d)
 	if err != nil {
 		return fmt.Errorf("error copying index metadata: %v", err)
 	}
diff --git a/mapping/document.go b/mapping/document.go
index 73bb124d..3131f33b 100644
--- a/mapping/document.go
+++ b/mapping/document.go
@@ -443,6 +443,8 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
 					fieldMapping.processGeoShape(property, pathString, path, indexes, context)
 				} else if fieldMapping.Type == "geopoint" {
 					fieldMapping.processGeoPoint(property, pathString, path, indexes, context)
+				} else if fieldMapping.Type == "vector_base64" {
+					fieldMapping.processVectorBase64(property, pathString, path, indexes, context)
 				} else {
 					fieldMapping.processString(propertyValueString, pathString, path, indexes, context)
 				}
diff --git a/mapping/field.go b/mapping/field.go
index f4339b38..5c064fdd 100644
--- a/mapping/field.go
+++ b/mapping/field.go
@@ -102,7 +102,7 @@ func newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {
 	return rv
 }
 
-// NewKeyworFieldMapping returns a default field mapping for text with analyzer "keyword".
+// NewKeywordFieldMapping returns a default field mapping for text with analyzer "keyword".
 func NewKeywordFieldMapping() *FieldMapping {
 	return &FieldMapping{
 		Type:               "text",
diff --git a/mapping/index.go b/mapping/index.go
index 171ee1a7..fe8c9671 100644
--- a/mapping/index.go
+++ b/mapping/index.go
@@ -437,24 +437,16 @@ func (im *IndexMappingImpl) FieldAnalyzer(field string) string {
 func (im *IndexMappingImpl) FieldMappingForPath(path string) FieldMapping {
 	if im.TypeMapping != nil {
 		for _, v := range im.TypeMapping {
-			for field, property := range v.Properties {
-				for _, v1 := range property.Fields {
-					if field == path {
-						// Return field mapping if the name matches the path param.
-						return *v1
-					}
-				}
+			fm := v.fieldDescribedByPath(path)
+			if fm != nil {
+				return *fm
 			}
 		}
 	}
 
-	for field, property := range im.DefaultMapping.Properties {
-		for _, v1 := range property.Fields {
-			if field == path {
-				// Return field mapping if the name matches the path param.
-				return *v1
-			}
-		}
+	fm := im.DefaultMapping.fieldDescribedByPath(path)
+	if fm != nil {
+		return *fm
 	}
 
 	return FieldMapping{}
diff --git a/mapping/mapping_no_vectors.go b/mapping/mapping_no_vectors.go
index f9f35f57..90cb1e22 100644
--- a/mapping/mapping_no_vectors.go
+++ b/mapping/mapping_no_vectors.go
@@ -21,11 +21,20 @@ func NewVectorFieldMapping() *FieldMapping {
 	return nil
 }
 
+func NewVectorBase64FieldMapping() *FieldMapping {
+	return nil
+}
+
 func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
 	pathString string, path []string, indexes []uint64, context *walkContext) bool {
 	return false
 }
 
+func (fm *FieldMapping) processVectorBase64(propertyMightBeVector interface{},
+	pathString string, path []string, indexes []uint64, context *walkContext) {
+
+}
+
 // -----------------------------------------------------------------------------
 // document validation functions
 
diff --git a/mapping/mapping_vectors.go b/mapping/mapping_vectors.go
index a0b71260..a3879c4b 100644
--- a/mapping/mapping_vectors.go
+++ b/mapping/mapping_vectors.go
@@ -26,10 +26,11 @@ import (
 	index "github.com/blevesearch/bleve_index_api"
 )
 
-// Min and Max allowed dimensions for a vector field
-const (
+// Min and Max allowed dimensions for a vector field;
+// p.s must be set/updated at process init() _only_
+var (
 	MinVectorDims = 1
-	MaxVectorDims = 2048
+	MaxVectorDims = 4096
 )
 
 func NewVectorFieldMapping() *FieldMapping {
@@ -43,6 +44,17 @@ func NewVectorFieldMapping() *FieldMapping {
 	}
 }
 
+func NewVectorBase64FieldMapping() *FieldMapping {
+	return &FieldMapping{
+		Type:         "vector_base64",
+		Store:        false,
+		Index:        true,
+		IncludeInAll: false,
+		DocValues:    false,
+		SkipFreqNorm: true,
+	}
+}
+
 // validate and process a flat vector
 func processFlatVector(vecV reflect.Value, dims int) ([]float32, bool) {
 	if vecV.Len() != dims {
@@ -140,13 +152,35 @@ func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
 	return true
 }
 
+func (fm *FieldMapping) processVectorBase64(propertyMightBeVectorBase64 interface{},
+	pathString string, path []string, indexes []uint64, context *walkContext) {
+	encodedString, ok := propertyMightBeVectorBase64.(string)
+	if !ok {
+		return
+	}
+
+	decodedVector, err := document.DecodeVector(encodedString)
+	if err != nil || len(decodedVector) != fm.Dims {
+		return
+	}
+
+	fieldName := getFieldName(pathString, path, fm)
+	options := fm.Options()
+	field := document.NewVectorFieldWithIndexingOptions(fieldName, indexes, decodedVector,
+		fm.Dims, fm.Similarity, fm.VectorIndexOptimizedFor, options)
+	context.doc.AddField(field)
+
+	// "_all" composite field is not applicable for vector_base64 field
+	context.excludedFromAll = append(context.excludedFromAll, fieldName)
+}
+
 // -----------------------------------------------------------------------------
 // document validation functions
 
 func validateFieldMapping(field *FieldMapping, parentName string,
 	fieldAliasCtx map[string]*FieldMapping) error {
 	switch field.Type {
-	case "vector":
+	case "vector", "vector_base64":
 		return validateVectorFieldAlias(field, parentName, fieldAliasCtx)
 	default: // non-vector field
 		return validateFieldType(field)
diff --git a/mapping_vector.go b/mapping_vector.go
index 59431386..c73dac9e 100644
--- a/mapping_vector.go
+++ b/mapping_vector.go
@@ -22,3 +22,7 @@ import "github.com/blevesearch/bleve/v2/mapping"
 func NewVectorFieldMapping() *mapping.FieldMapping {
 	return mapping.NewVectorFieldMapping()
 }
+
+func NewVectorBase64FieldMapping() *mapping.FieldMapping {
+	return mapping.NewVectorBase64FieldMapping()
+}
diff --git a/search/query/query.go b/search/query/query.go
index 26ab656e..d263a0e5 100644
--- a/search/query/query.go
+++ b/search/query/query.go
@@ -65,7 +65,7 @@ type ValidatableQuery interface {
 	Validate() error
 }
 
-// ParseQuery deserializes a JSON representation of
+// ParsePreSearchData deserializes a JSON representation of
 // a PreSearchData object.
 func ParsePreSearchData(input []byte) (map[string]interface{}, error) {
 	var rv map[string]interface{}
diff --git a/search/search.go b/search/search.go
index 515a320f..8cc5115d 100644
--- a/search/search.go
+++ b/search/search.go
@@ -147,7 +147,7 @@ type DocumentMatch struct {
 	Index           string                `json:"index,omitempty"`
 	ID              string                `json:"id"`
 	IndexInternalID index.IndexInternalID `json:"-"`
-	Score           float64               `json:"score,omitempty"`
+	Score           float64               `json:"score"`
 	Expl            *Explanation          `json:"explanation,omitempty"`
 	Locations       FieldTermLocationMap  `json:"locations,omitempty"`
 	Fragments       FieldFragmentMap      `json:"fragments,omitempty"`
diff --git a/search_knn.go b/search_knn.go
index ba6b005f..7ea21d37 100644
--- a/search_knn.go
+++ b/search_knn.go
@@ -23,12 +23,15 @@ import (
 	"fmt"
 	"sort"
 
+	"github.com/blevesearch/bleve/v2/document"
 	"github.com/blevesearch/bleve/v2/search"
 	"github.com/blevesearch/bleve/v2/search/collector"
 	"github.com/blevesearch/bleve/v2/search/query"
 	index "github.com/blevesearch/bleve_index_api"
 )
 
+const supportForVectorSearch = true
+
 type knnOperator string
 
 // Must be updated only at init
@@ -67,11 +70,13 @@ type SearchRequest struct {
 	sortFunc func(sort.Interface)
 }
 
+// Vector takes precedence over vectorBase64 in case both fields are given
 type KNNRequest struct {
-	Field  string       `json:"field"`
-	Vector []float32    `json:"vector"`
-	K      int64        `json:"k"`
-	Boost  *query.Boost `json:"boost,omitempty"`
+	Field        string       `json:"field"`
+	Vector       []float32    `json:"vector"`
+	VectorBase64 string       `json:"vector_base64"`
+	K            int64        `json:"k"`
+	Boost        *query.Boost `json:"boost,omitempty"`
 }
 
 func (r *SearchRequest) AddKNN(field string, vector []float32, k int64, boost float64) {
@@ -231,6 +236,15 @@ func validateKNN(req *SearchRequest) error {
 		if q == nil {
 			return fmt.Errorf("knn query cannot be nil")
 		}
+		if len(q.Vector) == 0 && q.VectorBase64 != "" {
+			// consider vector_base64 only if vector is not provided
+			decodedVector, err := document.DecodeVector(q.VectorBase64)
+			if err != nil {
+				return err
+			}
+
+			q.Vector = decodedVector
+		}
 		if q.K <= 0 || len(q.Vector) == 0 {
 			return fmt.Errorf("k must be greater than 0 and vector must be non-empty")
 		}
diff --git a/search_no_knn.go b/search_no_knn.go
index aff82611..bb72e15a 100644
--- a/search_no_knn.go
+++ b/search_no_knn.go
@@ -28,6 +28,8 @@ import (
 	index "github.com/blevesearch/bleve_index_api"
 )
 
+const supportForVectorSearch = false
+
 // A SearchRequest describes all the parameters
 // needed to search the index.
 // Query is required.
diff --git a/util/extract.go b/util/extract.go
index e963d0c3..0d3decfa 100644
--- a/util/extract.go
+++ b/util/extract.go
@@ -48,7 +48,7 @@ func ExtractNumericValFloat32(v interface{}) (float32, bool) {
 	switch {
 	case val.CanFloat():
 		floatVal := val.Float()
-		if floatVal > math.MaxFloat32 {
+		if !IsValidFloat32(floatVal) {
 			return 0, false
 		}
 		return float32(floatVal), true
@@ -60,3 +60,7 @@ func ExtractNumericValFloat32(v interface{}) (float32, bool) {
 
 	return 0, false
 }
+
+func IsValidFloat32(val float64) bool {
+	return !math.IsNaN(val) && !math.IsInf(val, 0) && val <= math.MaxFloat32
+}


Test patch:
diff --git a/document/field_vector_base64_test.go b/document/field_vector_base64_test.go
new file mode 100644
index 00000000..91212b07
--- /dev/null
+++ b/document/field_vector_base64_test.go
@@ -0,0 +1,112 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build vectors
+// +build vectors
+
+package document
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"fmt"
+	"math/rand"
+	"testing"
+)
+
+func TestDecodeVector(t *testing.T) {
+	vec := make([]float32, 2048)
+	for i := range vec {
+		vec[i] = rand.Float32()
+	}
+
+	vecBytes := bytifyVec(vec)
+	encodedVec := base64.StdEncoding.EncodeToString(vecBytes)
+
+	decodedVector, err := DecodeVector(encodedVec)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(decodedVector) != len(vec) {
+		t.Errorf("Decoded vector dimensions not same as original vector dimensions")
+	}
+
+	for i := range vec {
+		if vec[i] != decodedVector[i] {
+			t.Fatalf("Decoded vector not the same as original vector %v != %v", vec[i], decodedVector[i])
+		}
+	}
+}
+
+func BenchmarkDecodeVector128(b *testing.B) {
+	vec := make([]float32, 128)
+	for i := range vec {
+		vec[i] = rand.Float32()
+	}
+
+	vecBytes := bytifyVec(vec)
+	encodedVec := base64.StdEncoding.EncodeToString(vecBytes)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = DecodeVector(encodedVec)
+	}
+}
+
+func BenchmarkDecodeVector784(b *testing.B) {
+	vec := make([]float32, 784)
+	for i := range vec {
+		vec[i] = rand.Float32()
+	}
+
+	vecBytes := bytifyVec(vec)
+	encodedVec := base64.StdEncoding.EncodeToString(vecBytes)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = DecodeVector(encodedVec)
+	}
+}
+
+func BenchmarkDecodeVector1536(b *testing.B) {
+	vec := make([]float32, 1536)
+	for i := range vec {
+		vec[i] = rand.Float32()
+	}
+
+	vecBytes := bytifyVec(vec)
+	encodedVec := base64.StdEncoding.EncodeToString(vecBytes)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = DecodeVector(encodedVec)
+	}
+}
+
+func bytifyVec(vec []float32) []byte {
+	buf := new(bytes.Buffer)
+
+	for _, v := range vec {
+		err := binary.Write(buf, binary.LittleEndian, v)
+		if err != nil {
+			fmt.Println(err)
+		}
+	}
+
+	return buf.Bytes()
+}
diff --git a/index/scorch/event_test.go b/index/scorch/event_test.go
index ed1dd4d9..be163a9e 100644
--- a/index/scorch/event_test.go
+++ b/index/scorch/event_test.go
@@ -35,10 +35,11 @@ func TestEventBatchIntroductionStart(t *testing.T) {
 	}()
 
 	var count int
-	RegistryEventCallbacks["test"] = func(e Event) {
+	RegistryEventCallbacks["test"] = func(e Event) bool {
 		if e.Kind == EventKindBatchIntroductionStart {
 			count++
 		}
+		return true
 	}
 
 	ourConfig := make(map[string]interface{}, len(testConfig))
diff --git a/index/scorch/merge_test.go b/index/scorch/merge_test.go
index b3783c6c..ec7e68ea 100644
--- a/index/scorch/merge_test.go
+++ b/index/scorch/merge_test.go
@@ -41,7 +41,7 @@ func TestObsoleteSegmentMergeIntroduction(t *testing.T) {
 	mergeIntroStart.Add(1)
 	mergeIntroComplete.Add(1)
 	var segIntroCompleted int
-	RegistryEventCallbacks["test"] = func(e Event) {
+	RegistryEventCallbacks["test"] = func(e Event) bool {
 		if e.Kind == EventKindBatchIntroduction {
 			segIntroCompleted++
 			if segIntroCompleted == 3 {
@@ -61,6 +61,7 @@ func TestObsoleteSegmentMergeIntroduction(t *testing.T) {
 			mergeIntroComplete.Done()
 
 		}
+		return true
 	}
 
 	ourConfig := make(map[string]interface{}, len(testConfig))
diff --git a/index/scorch/mergeplan/merge_plan_test.go b/index/scorch/mergeplan/merge_plan_test.go
index 2791d09c..4a577109 100644
--- a/index/scorch/mergeplan/merge_plan_test.go
+++ b/index/scorch/mergeplan/merge_plan_test.go
@@ -30,11 +30,16 @@ type segment struct {
 	MyId       uint64
 	MyFullSize int64
 	MyLiveSize int64
+
+	MyHasVector bool
+	MyFileSize  int64
 }
 
 func (s *segment) Id() uint64      { return s.MyId }
 func (s *segment) FullSize() int64 { return s.MyFullSize }
 func (s *segment) LiveSize() int64 { return s.MyLiveSize }
+func (s *segment) HasVector() bool { return s.MyHasVector }
+func (s *segment) FileSize() int64 { return s.MyFileSize }
 
 func makeLinearSegments(n int) (rv []Segment) {
 	for i := 0; i < n; i++ {
@@ -589,3 +594,102 @@ func emit(descrip string, cycle int, step int, segments []Segment, plan *MergePl
 	fmt.Printf("%s %d.%d ---------- %s\n", descrip, cycle, step, suffix)
 	fmt.Printf("%s\n", ToBarChart(descrip, 100, segments, plan))
 }
+
+// -----------------------------------------------------------------------------
+// Test Vector Segment Merging
+
+func TestPlanMaxSegmentFileSize(t *testing.T) {
+	tests := []struct {
+		segments []Segment
+		o        *MergePlanOptions
+
+		expectedTasks [][]uint64
+	}{
+		{
+			[]Segment{
+				&segment{ // ineligible
+					MyId:       1,
+					MyFullSize: 4000,
+					MyLiveSize: 3900,
+
+					MyHasVector: true,
+					MyFileSize:  3900 * 1000 * 4, // > 2MB
+				},
+				&segment{ // ineligible
+					MyId:       2,
+					MyFullSize: 6000,
+					MyLiveSize: 5500, // > 5000
+
+					MyHasVector: true,
+					MyFileSize:  5500 * 1000 * 4, // > 2MB
+				},
+				&segment{ // eligible
+					MyId:       3,
+					MyFullSize: 500,
+					MyLiveSize: 490,
+
+					MyHasVector: true,
+					MyFileSize:  490 * 1000 * 4,
+				},
+				&segment{ // eligible
+					MyId:       4,
+					MyFullSize: 500,
+					MyLiveSize: 480,
+
+					MyHasVector: true,
+					MyFileSize:  480 * 1000 * 4,
+				},
+				&segment{ // eligible
+					MyId:       5,
+					MyFullSize: 500,
+					MyLiveSize: 300,
+
+					MyHasVector: true,
+					MyFileSize:  300 * 1000 * 4,
+				},
+				&segment{ // eligible
+					MyId:       6,
+					MyFullSize: 500,
+					MyLiveSize: 400,
+
+					MyHasVector: true,
+					MyFileSize:  400 * 1000 * 4,
+				},
+			},
+			&MergePlanOptions{
+				MaxSegmentSize: 5000, // number of documents
+				// considering vector dimension as 1000
+				// vectorBytes = 5000 * 1000 * 4 = 20MB, which is too large
+				// So, let's set the fileSize limit to 4MB
+				MaxSegmentFileSize:   4000000, // 4MB
+				MaxSegmentsPerTier:   1,
+				SegmentsPerMergeTask: 2,
+				TierGrowth:           2.0,
+				FloorSegmentSize:     1,
+			},
+			[][]uint64{
+				{3, 4},
+			},
+		},
+	}
+
+	for testi, test := range tests {
+		t.Run(fmt.Sprintf("Test-%d", testi), func(t *testing.T) {
+			plans, err := Plan(test.segments, test.o)
+			if err != nil {
+				t.Fatalf("Plan failed, err: %v", err)
+			}
+
+			for i, task := range plans.Tasks {
+				var segIDs []uint64
+				for _, seg := range task.Segments {
+					segIDs = append(segIDs, seg.Id())
+				}
+
+				if !reflect.DeepEqual(segIDs, test.expectedTasks[0]) {
+					t.Errorf("expected task segments: %v, got: %v", test.expectedTasks[i], segIDs)
+				}
+			}
+		})
+	}
+}
diff --git a/index/scorch/reader_test.go b/index/scorch/reader_test.go
index 0eebd92b..cc33e4e7 100644
--- a/index/scorch/reader_test.go
+++ b/index/scorch/reader_test.go
@@ -103,7 +103,7 @@ func TestIndexReader(t *testing.T) {
 	expectedCount = 2
 	count = reader.Count()
 	if count != expectedCount {
-		t.Errorf("Exptected doc count to be: %d got: %d", expectedCount, count)
+		t.Errorf("Expected doc count to be: %d got: %d", expectedCount, count)
 	}
 
 	var match *index.TermFieldDoc
diff --git a/index/scorch/scorch_test.go b/index/scorch/scorch_test.go
index b8f493fe..8165774e 100644
--- a/index/scorch/scorch_test.go
+++ b/index/scorch/scorch_test.go
@@ -1210,7 +1210,7 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
 				t.Error(err)
 			} else {
 				if numFieldNumer != 35.99 {
-					t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer)
+					t.Errorf("expected numeric value 35.99, got %f", numFieldNumer)
 				}
 			}
 		} else if field.Name() == "unixEpoch" {
@@ -1291,7 +1291,7 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
 				t.Error(err)
 			} else {
 				if numFieldNumer != 36.99 {
-					t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer)
+					t.Errorf("expected numeric value 36.99, got %f", numFieldNumer)
 				}
 			}
 		} else if field.Name() == "_id" {
diff --git a/index/upsidedown/reader_test.go b/index/upsidedown/reader_test.go
index 933723c9..314e1a1c 100644
--- a/index/upsidedown/reader_test.go
+++ b/index/upsidedown/reader_test.go
@@ -98,7 +98,7 @@ func TestIndexReader(t *testing.T) {
 	expectedCount = 2
 	count = reader.Count()
 	if count != expectedCount {
-		t.Errorf("Exptected doc count to be: %d got: %d", expectedCount, count)
+		t.Errorf("Expected doc count to be: %d got: %d", expectedCount, count)
 	}
 
 	var match *index.TermFieldDoc
diff --git a/index/upsidedown/store/boltdb/store_test.go b/index/upsidedown/store/boltdb/store_test.go
index 6f2b0e78..6011701e 100644
--- a/index/upsidedown/store/boltdb/store_test.go
+++ b/index/upsidedown/store/boltdb/store_test.go
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !darwin || !arm64
+
 package boltdb
 
 import (
diff --git a/index/upsidedown/upsidedown_test.go b/index/upsidedown/upsidedown_test.go
index 3e60175b..beeea657 100644
--- a/index/upsidedown/upsidedown_test.go
+++ b/index/upsidedown/upsidedown_test.go
@@ -894,7 +894,7 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
 		t.Error(err)
 	} else {
 		if numFieldNumer != 35.99 {
-			t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer)
+			t.Errorf("expected numeric value 35.99, got %f", numFieldNumer)
 		}
 	}
 	dateField, ok := storedDoc.Fields[2].(*document.DateTimeField)
@@ -964,7 +964,7 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
 		t.Error(err)
 	} else {
 		if numFieldNumer != 36.99 {
-			t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer)
+			t.Errorf("expected numeric value 36.99, got %f", numFieldNumer)
 		}
 	}
 
diff --git a/index_test.go b/index_test.go
index c6cc0793..e5094440 100644
--- a/index_test.go
+++ b/index_test.go
@@ -401,9 +401,15 @@ func TestBytesRead(t *testing.T) {
 	}
 	stats, _ := idx.StatsMap()["index"].(map[string]interface{})
 	prevBytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64)
-	if prevBytesRead != 21639 && res.Cost == prevBytesRead {
-		t.Fatalf("expected bytes read for query string 21639, got %v",
-			prevBytesRead)
+
+	expectedBytesRead := uint64(21639)
+	if supportForVectorSearch {
+		expectedBytesRead = 22049
+	}
+
+	if prevBytesRead != expectedBytesRead && res.Cost == prevBytesRead {
+		t.Fatalf("expected bytes read for query string %v, got %v",
+			expectedBytesRead, prevBytesRead)
 	}
 
 	// subsequent queries on the same field results in lesser amount
@@ -553,8 +559,14 @@ func TestBytesReadStored(t *testing.T) {
 
 	stats, _ := idx.StatsMap()["index"].(map[string]interface{})
 	bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64)
-	if bytesRead != 11501 && bytesRead == res.Cost {
-		t.Fatalf("expected the bytes read stat to be around 11501, got %v", bytesRead)
+
+	expectedBytesRead := uint64(11501)
+	if supportForVectorSearch {
+		expectedBytesRead = 11911
+	}
+
+	if bytesRead != expectedBytesRead && bytesRead == res.Cost {
+		t.Fatalf("expected the bytes read stat to be around %v, got %v", expectedBytesRead, bytesRead)
 	}
 	prevBytesRead := bytesRead
 
@@ -624,8 +636,14 @@ func TestBytesReadStored(t *testing.T) {
 
 	stats, _ = idx1.StatsMap()["index"].(map[string]interface{})
 	bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64)
-	if bytesRead != 3687 && bytesRead == res.Cost {
-		t.Fatalf("expected the bytes read stat to be around 3687, got %v", bytesRead)
+
+	expectedBytesRead = uint64(3687)
+	if supportForVectorSearch {
+		expectedBytesRead = 4097
+	}
+
+	if bytesRead != expectedBytesRead && bytesRead == res.Cost {
+		t.Fatalf("expected the bytes read stat to be around %v, got %v", expectedBytesRead, bytesRead)
 	}
 	prevBytesRead = bytesRead
 
diff --git a/mapping/mapping_vectors_test.go b/mapping/mapping_vectors_test.go
index 8c04d03a..970a7de7 100644
--- a/mapping/mapping_vectors_test.go
+++ b/mapping/mapping_vectors_test.go
@@ -17,7 +17,9 @@
 
 package mapping
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestVectorFieldAliasValidation(t *testing.T) {
 	tests := []struct {
@@ -26,8 +28,8 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 		mappingStr string // index mapping json string
 
 		// expected output
-		expValidity bool   // validity of the mapping
-		errMsg      string // error message, given expValidity is false
+		expValidity bool     // validity of the mapping
+		errMsgs     []string // error message, given expValidity is false
 	}{
 		{
 			name: "test1",
@@ -52,7 +54,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: false,
-			errMsg:      `field: 'cityVec', invalid alias (different dimensions 4 and 3)`,
+			errMsgs:     []string{`field: 'cityVec', invalid alias (different dimensions 4 and 3)`},
 		},
 		{
 			name: "test2",
@@ -79,7 +81,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: false,
-			errMsg:      `field: 'cityVec', invalid alias (different similarity values dot_product and l2_norm)`,
+			errMsgs:     []string{`field: 'cityVec', invalid alias (different similarity values dot_product and l2_norm)`},
 		},
 		{
 			name: "test3",
@@ -104,7 +106,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: true,
-			errMsg:      "",
+			errMsgs:     []string{},
 		},
 		{
 			name: "test4",
@@ -134,7 +136,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: false,
-			errMsg:      `field: 'vecData', invalid alias (different dimensions 3 and 4)`,
+			errMsgs:     []string{`field: 'vecData', invalid alias (different dimensions 3 and 4)`, `field: 'vecData', invalid alias (different dimensions 4 and 3)`},
 		},
 		{
 			name: "test5",
@@ -170,7 +172,7 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					}
 				}`,
 			expValidity: false,
-			errMsg:      `field: 'vecData', invalid alias (different dimensions 4 and 3)`,
+			errMsgs:     []string{`field: 'vecData', invalid alias (different dimensions 4 and 3)`},
 		},
 	}
 
@@ -189,9 +191,18 @@ func TestVectorFieldAliasValidation(t *testing.T) {
 					test.expValidity, isValid)
 			}
 
-			if !isValid && err.Error() != test.errMsg {
-				t.Fatalf("invalid error message, expected: %v, got: %v",
-					test.errMsg, err.Error())
+			if !isValid {
+				errStringMatched := false
+				for _, possibleErrMsg := range test.errMsgs {
+					if err.Error() == possibleErrMsg {
+						errStringMatched = true
+						break
+					}
+				}
+				if !errStringMatched {
+					t.Fatalf("invalid error message, expected one of: %v, got: %v",
+						test.errMsgs, err.Error())
+				}
 			}
 		})
 	}
diff --git a/search/scorer/scorer_knn_test.go b/search/scorer/scorer_knn_test.go
index 46a1de9b..1c005506 100644
--- a/search/scorer/scorer_knn_test.go
+++ b/search/scorer/scorer_knn_test.go
@@ -89,7 +89,7 @@ func TestKNNScorerExplanation(t *testing.T) {
 					Children: []*search.Explanation{
 						{
 							Value:   maxKNNScore,
-							Message: "vector(field(desc:one) with similarity_metric(l2_norm)=1.797693e+308",
+							Message: "vector(field(desc:one) with similarity_metric(l2_norm)=3.402823e+38",
 						},
 					},
 				},
diff --git a/search_knn_test.go b/search_knn_test.go
index b54ce5a9..24c677c4 100644
--- a/search_knn_test.go
+++ b/search_knn_test.go
@@ -19,6 +19,9 @@ package bleve
 
 import (
 	"archive/zip"
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"math"
@@ -130,13 +133,15 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 	}
 
 	index := NewIndexAlias()
+	var reqSort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: "content"}}
 	for testCaseNum, testCase := range testCases {
+		originalRequest := searchRequests[testCase.queryIndex]
 		for _, operator := range knnOperators {
-			index.indexes = make([]Index, 0)
 
-			query := searchRequests[testCase.queryIndex]
+			index.indexes = make([]Index, 0)
+			query := copySearchRequest(originalRequest, nil)
 			query.AddKNNOperator(operator)
-			query.Sort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: true, Field: "content"}}
+			query.Sort = reqSort.Copy()
 			query.Explain = true
 
 			nameToIndex := createPartitionedIndex(documents, index, 1, testCase.mapping, t, false)
@@ -150,7 +155,13 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 				t.Fatalf("test case #%d failed: expected control result hits to have valid `Index`", testCaseNum)
 			}
 			cleanUp(t, nameToIndex)
+
 			index.indexes = make([]Index, 0)
+			query = copySearchRequest(originalRequest, nil)
+			query.AddKNNOperator(operator)
+			query.Sort = reqSort.Copy()
+			query.Explain = true
+
 			nameToIndex = createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, false)
 			experimentalResult, err := index.Search(query)
 			if err != nil {
@@ -165,6 +176,11 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 			cleanUp(t, nameToIndex)
 
 			index.indexes = make([]Index, 0)
+			query = copySearchRequest(originalRequest, nil)
+			query.AddKNNOperator(operator)
+			query.Sort = reqSort.Copy()
+			query.Explain = true
+
 			nameToIndex = createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, true)
 			multiLevelIndexResult, err := index.Search(query)
 			if err != nil {
@@ -177,6 +193,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 			}
 			verifyResult(t, multiLevelIndexResult, experimentalResult, testCaseNum, false)
 			cleanUp(t, nameToIndex)
+
 		}
 	}
 
@@ -187,17 +204,14 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 		},
 	}
 
-	var sort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortField{Desc: false, Field: "content"}}
-
 	index = NewIndexAlias()
 	for testCaseNum, testCase := range testCases {
 		index.indexes = make([]Index, 0)
 		nameToIndex := createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, false)
 		originalRequest := searchRequests[testCase.queryIndex]
 		for _, operator := range knnOperators {
-
 			from, size := originalRequest.From, originalRequest.Size
-			query := copySearchRequest(searchRequests[testCase.queryIndex], nil)
+			query := copySearchRequest(originalRequest, nil)
 			query.AddKNNOperator(operator)
 			query.Explain = true
 			query.From = from
@@ -211,7 +225,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 
 			// 1. Sort And Facet are there
 			query.Facets = facets
-			query.Sort = sort
+			query.Sort = reqSort.Copy()
 
 			res1, err := index.Search(query)
 			if err != nil {
@@ -232,7 +246,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 
 			// 2. Sort is there, Facet is not there
 			query.Facets = nil
-			query.Sort = sort
+			query.Sort = reqSort.Copy()
 
 			res2, err := index.Search(query)
 			if err != nil {
@@ -279,7 +293,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 
 			// Test early exit fail case -> matchNone + facetRequest
 			query.Query = NewMatchNoneQuery()
-			query.Sort = sort
+			query.Sort = reqSort.Copy()
 			// control case
 			query.Facets = nil
 			res4Ctrl, err := index.Search(query)
@@ -321,7 +335,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 
 	// Test From + Size pagination for Hybrid Search (2-Phase)
 	query := copySearchRequest(searchRequests[4], nil)
-	query.Sort = sort
+	query.Sort = reqSort.Copy()
 	query.Facets = facets
 	query.Explain = true
 
@@ -330,7 +344,7 @@ func TestSimilaritySearchPartitionedIndex(t *testing.T) {
 	// Test From + Size pagination for Early Exit Hybrid Search (1-Phase)
 	query = copySearchRequest(searchRequests[4], nil)
 	query.Query = NewMatchNoneQuery()
-	query.Sort = sort
+	query.Sort = reqSort.Copy()
 	query.Facets = nil
 	query.Explain = true
 
@@ -397,10 +411,181 @@ func min(a, b int) int {
 	return b
 }
 
+func TestVectorBase64Index(t *testing.T) {
+	dataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	documents := makeDatasetIntoDocuments(dataset)
+
+	_, searchRequestsCopy, err := readDatasetAndQueries(testInputCompressedFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, doc := range documents {
+		vec, ok := doc["vector"].([]float32)
+		if !ok {
+			t.Fatal("Typecasting vector to float array failed")
+		}
+
+		buf := new(bytes.Buffer)
+		for _, v := range vec {
+			err := binary.Write(buf, binary.LittleEndian, v)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		doc["vectorEncoded"] = base64.StdEncoding.EncodeToString(buf.Bytes())
+	}
+
+	for _, sr := range searchRequestsCopy {
+		for _, kr := range sr.KNN {
+			kr.Field = "vectorEncoded"
+		}
+	}
+
+	contentFM := NewTextFieldMapping()
+	contentFM.Analyzer = en.AnalyzerName
+
+	vecFML2 := mapping.NewVectorFieldMapping()
+	vecFML2.Dims = testDatasetDims
+	vecFML2.Similarity = index.EuclideanDistance
+
+	vecBFML2 := mapping.NewVectorBase64FieldMapping()
+	vecBFML2.Dims = testDatasetDims
+	vecBFML2.Similarity = index.EuclideanDistance
+
+	vecFMDot := mapping.NewVectorFieldMapping()
+	vecFMDot.Dims = testDatasetDims
+	vecFMDot.Similarity = index.CosineSimilarity
+
+	vecBFMDot := mapping.NewVectorBase64FieldMapping()
+	vecBFMDot.Dims = testDatasetDims
+	vecBFMDot.Similarity = index.CosineSimilarity
+
+	indexMappingL2 := NewIndexMapping()
+	indexMappingL2.DefaultMapping.AddFieldMappingsAt("content", contentFM)
+	indexMappingL2.DefaultMapping.AddFieldMappingsAt("vector", vecFML2)
+	indexMappingL2.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFML2)
+
+	indexMappingDot := NewIndexMapping()
+	indexMappingDot.DefaultMapping.AddFieldMappingsAt("content", contentFM)
+	indexMappingDot.DefaultMapping.AddFieldMappingsAt("vector", vecFMDot)
+	indexMappingDot.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFMDot)
+
+	tmpIndexPathL2 := createTmpIndexPath(t)
+	defer cleanupTmpIndexPath(t, tmpIndexPathL2)
+
+	tmpIndexPathDot := createTmpIndexPath(t)
+	defer cleanupTmpIndexPath(t, tmpIndexPathDot)
+
+	indexL2, err := New(tmpIndexPathL2, indexMappingL2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := indexL2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	indexDot, err := New(tmpIndexPathDot, indexMappingDot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := indexDot.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	batchL2 := indexL2.NewBatch()
+	batchDot := indexDot.NewBatch()
+
+	for _, doc := range documents {
+		err = batchL2.Index(doc["id"].(string), doc)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = batchDot.Index(doc["id"].(string), doc)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	err = indexL2.Batch(batchL2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = indexDot.Batch(batchDot)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i, _ := range searchRequests {
+		for _, operator := range knnOperators {
+			controlQuery := searchRequests[i]
+			testQuery := searchRequestsCopy[i]
+
+			controlQuery.AddKNNOperator(operator)
+			testQuery.AddKNNOperator(operator)
+
+			controlResultL2, err := indexL2.Search(controlQuery)
+			if err != nil {
+				t.Fatal(err)
+			}
+			testResultL2, err := indexL2.Search(testQuery)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if controlResultL2 != nil && testResultL2 != nil {
+				if len(controlResultL2.Hits) == len(testResultL2.Hits) {
+					for j, _ := range controlResultL2.Hits {
+						if controlResultL2.Hits[j].ID != testResultL2.Hits[j].ID {
+							t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, controlResultL2.Hits[j].ID, testResultL2.Hits[j].ID)
+						}
+					}
+				}
+			} else if (controlResultL2 == nil && testResultL2 != nil) ||
+				(controlResultL2 != nil && testResultL2 == nil) {
+				t.Fatalf("testcase %d failed: expected result %s, got result %s", i, controlResultL2, testResultL2)
+			}
+
+			controlResultDot, err := indexDot.Search(controlQuery)
+			if err != nil {
+				t.Fatal(err)
+			}
+			testResultDot, err := indexDot.Search(testQuery)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if controlResultDot != nil && testResultDot != nil {
+				if len(controlResultDot.Hits) == len(testResultDot.Hits) {
+					for j, _ := range controlResultDot.Hits {
+						if controlResultDot.Hits[j].ID != testResultDot.Hits[j].ID {
+							t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, controlResultDot.Hits[j].ID, testResultDot.Hits[j].ID)
+						}
+					}
+				}
+			} else if (controlResultDot == nil && testResultDot != nil) ||
+				(controlResultDot != nil && testResultDot == nil) {
+				t.Fatalf("testcase %d failed: expected result %s, got result %s", i, controlResultDot, testResultDot)
+			}
+		}
+	}
+}
+
 type testDocument struct {
 	ID      string    `json:"id"`
 	Content string    `json:"content"`
-	Vector  []float64 `json:"vector"`
+	Vector  []float32 `json:"vector"`
 }
 
 func readDatasetAndQueries(fileName string) ([]testDocument, []*SearchRequest, error) {
@@ -745,6 +930,8 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 	indexMappingDotProduct.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping)
 	indexMappingDotProduct.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMappingDot)
 
+	var reqSort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: "content"}}
+
 	testCases := []struct {
 		numSegments int
 		queryIndex  int
@@ -852,6 +1039,7 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 		},
 	}
 	for testCaseNum, testCase := range testCases {
+		originalRequest := searchRequests[testCase.queryIndex]
 		for _, operator := range knnOperators {
 			// run single segment test first
 			tmpIndexPath := createTmpIndexPath(t)
@@ -859,8 +1047,8 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			query := searchRequests[testCase.queryIndex]
-			query.Sort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: "content"}}
+			query := copySearchRequest(originalRequest, nil)
+			query.Sort = reqSort.Copy()
 			query.AddKNNOperator(operator)
 			query.Explain = true
 
@@ -882,7 +1070,13 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 				t.Fatalf("test case #%d failed: expected control result hits to have valid `Index`", testCaseNum)
 			}
 			if testCase.scoreValue == "none" {
+
+				query := copySearchRequest(originalRequest, nil)
+				query.Sort = reqSort.Copy()
+				query.AddKNNOperator(operator)
+				query.Explain = true
 				query.Score = testCase.scoreValue
+
 				expectedResultScoreNone, err := index.Search(query)
 				if err != nil {
 					cleanUp(t, nameToIndex)
@@ -893,7 +1087,6 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 					t.Fatalf("test case #%d failed: expected score none hits to have valid `Index`", testCaseNum)
 				}
 				verifyResult(t, controlResult, expectedResultScoreNone, testCaseNum, true)
-				query.Score = ""
 			}
 			cleanUp(t, nameToIndex)
 
@@ -910,6 +1103,12 @@ func TestSimilaritySearchMultipleSegments(t *testing.T) {
 				cleanUp(t, nameToIndex)
 				t.Fatal(err)
 			}
+
+			query = copySearchRequest(originalRequest, nil)
+			query.Sort = reqSort.Copy()
+			query.AddKNNOperator(operator)
+			query.Explain = true
+
 			experimentalResult, err := index.Search(query)
 			if err != nil {
 				cleanUp(t, nameToIndex)
diff --git a/search_test.go b/search_test.go
index 7f76978b..3d14c925 100644
--- a/search_test.go
+++ b/search_test.go
@@ -3116,7 +3116,7 @@ func TestDateRangeTimestampQueries(t *testing.T) {
 		}
 	}()
 
-	documents := map[string]map[string]interface{}{
+	documents := map[string]map[string]string{
 		"doc1": {
 			"date":         "2001/08/20 03:00:10",
 			"seconds":      "998276410",
@@ -3166,15 +3166,11 @@ func TestDateRangeTimestampQueries(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	type testResult struct {
-		docID    string // doc ID of the hit
-		hitField string // fields returned as part of the hit
-	}
 	type testStruct struct {
 		start        string
 		end          string
 		field        string
-		expectedHits []testResult
+		expectedHits []string
 	}
 
 	testQueries := []testStruct{
@@ -3182,83 +3178,47 @@ func TestDateRangeTimestampQueries(t *testing.T) {
 			start: "2001-08-20T03:00:05",
 			end:   "2001-08-20T03:00:25",
 			field: "date",
-			expectedHits: []testResult{
-				{
-					docID:    "doc1",
-					hitField: "2001/08/20 03:00:10",
-				},
-				{
-					docID:    "doc2",
-					hitField: "2001/08/20 03:00:20",
-				},
+			expectedHits: []string{
+				"doc1",
+				"doc2",
 			},
 		},
 		{
 			start: "2001-08-20T03:00:15",
 			end:   "2001-08-20T03:00:35",
 			field: "seconds",
-			expectedHits: []testResult{
-				{
-					docID:    "doc2",
-					hitField: "998276420000000000",
-				},
-				{
-					docID:    "doc3",
-					hitField: "998276430000000000",
-				},
+			expectedHits: []string{
+				"doc2",
+				"doc3",
 			},
 		},
 		{
 			start: "2001-08-20T03:00:10.150",
 			end:   "2001-08-20T03:00:10.450",
 			field: "milliseconds",
-			expectedHits: []testResult{
-				{
-					docID:    "doc2",
-					hitField: "998276410200000000",
-				},
-				{
-					docID:    "doc3",
-					hitField: "998276410300000000",
-				},
-				{
-					docID:    "doc4",
-					hitField: "998276410400000000",
-				},
+			expectedHits: []string{
+				"doc2",
+				"doc3",
+				"doc4",
 			},
 		},
 		{
 			start: "2001-08-20T03:00:10.100450",
 			end:   "2001-08-20T03:00:10.100650",
 			field: "microseconds",
-			expectedHits: []testResult{
-				{
-					docID:    "doc3",
-					hitField: "998276410100500000",
-				},
-				{
-					docID:    "doc4",
-					hitField: "998276410100600000",
-				},
+			expectedHits: []string{
+				"doc3",
+				"doc4",
 			},
 		},
 		{
 			start: "2001-08-20T03:00:10.100300550",
 			end:   "2001-08-20T03:00:10.100300850",
 			field: "nanoseconds",
-			expectedHits: []testResult{
-				{
-					docID:    "doc3",
-					hitField: "998276410100300600",
-				},
-				{
-					docID:    "doc4",
-					hitField: "998276410100300700",
-				},
-				{
-					docID:    "doc5",
-					hitField: "998276410100300800",
-				},
+			expectedHits: []string{
+				"doc3",
+				"doc4",
+				"doc5",
 			},
 		},
 	}
@@ -3277,7 +3237,7 @@ func TestDateRangeTimestampQueries(t *testing.T) {
 
 		sr := NewSearchRequest(drq)
 		sr.SortBy([]string{dtq.field})
-		sr.Fields = []string{dtq.field}
+		sr.Fields = []string{"*"}
 
 		res, err := idx.Search(sr)
 		if err != nil {
@@ -3287,11 +3247,16 @@ func TestDateRangeTimestampQueries(t *testing.T) {
 			t.Fatalf("expected %d hits, got %d", len(dtq.expectedHits), len(res.Hits))
 		}
 		for i, hit := range res.Hits {
-			if hit.ID != dtq.expectedHits[i].docID {
-				t.Fatalf("expected docID %s, got %s", dtq.expectedHits[i].docID, hit.ID)
+			if hit.ID != dtq.expectedHits[i] {
+				t.Fatalf("expected docID %s, got %s", dtq.expectedHits[i], hit.ID)
 			}
-			if hit.Fields[dtq.field].(string) != dtq.expectedHits[i].hitField {
-				t.Fatalf("expected hit field %s, got %s", dtq.expectedHits[i].hitField, hit.Fields[dtq.field])
+			if len(hit.Fields) != len(documents[hit.ID]) {
+				t.Fatalf("expected hit %s to have %d fields, got %d", hit.ID, len(documents[hit.ID]), len(hit.Fields))
+			}
+			for k, v := range documents[hit.ID] {
+				if hit.Fields[k] != v {
+					t.Fatalf("expected field %s to be %s, got %s", k, v, hit.Fields[k])
+				}
 			}
 		}
 	}