Repository: blevesearch/bleve
Pull request #2034 (2024-05-15 18:41:50)

MB-59575: Upgrade zapx/v16, go-faiss
Requires:
- blevesearch/faiss@17c3992 (branch:bleve)

Includes:
* 8c1937d Thejas-bhat | bug fix: de-duplicating fieldsInv value
* 08059f8 Thejas-bhat | MB-59575: Handling memory mapped content more responsibly


Patch:
diff --git a/go.mod b/go.mod
index 68f0f1a7..d025f67e 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/blevesearch/zapx/v13 v13.3.10
 	github.com/blevesearch/zapx/v14 v14.3.10
 	github.com/blevesearch/zapx/v15 v15.3.13
-	github.com/blevesearch/zapx/v16 v16.0.13-0.20240514143026-a9fced173441
+	github.com/blevesearch/zapx/v16 v16.0.13-0.20240515183332-8c1937d99b96
 	github.com/couchbase/moss v0.2.0
 	github.com/golang/protobuf v1.3.2
 	github.com/spf13/cobra v1.7.0
@@ -32,7 +32,7 @@ require (
 )
 
 require (
-	github.com/blevesearch/go-faiss v1.0.15 // indirect
+	github.com/blevesearch/go-faiss v1.0.16 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
 	github.com/couchbase/ghistogram v0.1.0 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
diff --git a/go.sum b/go.sum
index d0ab2a63..29041c00 100644
--- a/go.sum
+++ b/go.sum
@@ -6,8 +6,8 @@ github.com/blevesearch/bleve_index_api v1.1.7 h1:KvvKVn/9+dKCEJHL6flDH3NCiXeKnK3
 github.com/blevesearch/bleve_index_api v1.1.7/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
 github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
 github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
-github.com/blevesearch/go-faiss v1.0.15 h1:aBrj6fwLuY8CkhECFbvkc4qhLTkrYI84QoEaw9z1jMI=
-github.com/blevesearch/go-faiss v1.0.15/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
+github.com/blevesearch/go-faiss v1.0.16 h1:lfzXzzjO1mAf15MRiRY5yz6KVGr02CyRrr7m0z70Ih8=
+github.com/blevesearch/go-faiss v1.0.16/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
@@ -43,8 +43,8 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7
 github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
 github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
 github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
-github.com/blevesearch/zapx/v16 v16.0.13-0.20240514143026-a9fced173441 h1:SJhTyViAfEQBT6icz7omIHgnVVNqscpUAlfINzRmNn4=
-github.com/blevesearch/zapx/v16 v16.0.13-0.20240514143026-a9fced173441/go.mod h1:Viwste/2ooJPzsLNuncnvyTkbAL2iAgM4ge5agqkdPg=
+github.com/blevesearch/zapx/v16 v16.0.13-0.20240515183332-8c1937d99b96 h1:9Ugk3pXJ1Y3hYaee2PMjMyYevX6LKAvTC0o165sy4uY=
+github.com/blevesearch/zapx/v16 v16.0.13-0.20240515183332-8c1937d99b96/go.mod h1:OtQLZCpBsNwNLJRgeKLamFEN9dA1T4CgnmwYXPcU7BI=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=
diff --git a/index/scorch/event.go b/index/scorch/event.go
index 31c9e80c..0f653ccf 100644
--- a/index/scorch/event.go
+++ b/index/scorch/event.go
@@ -22,7 +22,8 @@ var RegistryAsyncErrorCallbacks = map[string]func(error, string){}
 
 // RegistryEventCallbacks should be treated as read-only after
 // process init()'ialization.
-var RegistryEventCallbacks = map[string]func(Event){}
+// In the event of not having a callback, these return true.
+var RegistryEventCallbacks = map[string]func(Event) bool{}
 
 // Event represents the information provided in an OnEvent() callback.
 type Event struct {
@@ -62,3 +63,7 @@ var EventKindMergeTaskIntroductionStart = EventKind(7)
 // EventKindMergeTaskIntroduction is fired when the merger has completed
 // the introduction of merged segment from a single merge task.
 var EventKindMergeTaskIntroduction = EventKind(8)
+
+// EventKindPreMergeCheck is fired before the merge begins to check if
+// the caller should proceed with the merge.
+var EventKindPreMergeCheck = EventKind(9)
diff --git a/index/scorch/merge.go b/index/scorch/merge.go
index 339ec596..b74504ca 100644
--- a/index/scorch/merge.go
+++ b/index/scorch/merge.go
@@ -72,6 +72,17 @@ OUTER:
 				ctrlMsg = ctrlMsgDflt
 			}
 			if ctrlMsg != nil {
+				continueMerge := s.fireEvent(EventKindPreMergeCheck, 0)
+				// The default, if there's no handler, is to continue the merge.
+				if !continueMerge {
+					// If it's decided that this merge can't take place now,
+					// begin the merge process all over again.
+					// Retry instead of blocking/waiting here since a long wait
+					// can result in more segments introduced i.e. s.root will
+					// be updated.
+					continue OUTER
+				}
+
 				startTime := time.Now()
 
 				// lets get started
diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go
index 7d691ff4..7966d844 100644
--- a/index/scorch/scorch.go
+++ b/index/scorch/scorch.go
@@ -77,7 +77,7 @@ type Scorch struct {
 	rootBolt                 *bolt.DB
 	asyncTasks               sync.WaitGroup
 
-	onEvent      func(event Event)
+	onEvent      func(event Event) bool
 	onAsyncError func(err error, path string)
 
 	forceMergeRequestCh chan *mergerCtrl
@@ -184,12 +184,14 @@ func (s *Scorch) NumEventsBlocking() uint64 {
 	return eventsStarted - eventsCompleted
 }
 
-func (s *Scorch) fireEvent(kind EventKind, dur time.Duration) {
+func (s *Scorch) fireEvent(kind EventKind, dur time.Duration) bool {
+	res := true
 	if s.onEvent != nil {
 		atomic.AddUint64(&s.stats.TotEventTriggerStarted, 1)
-		s.onEvent(Event{Kind: kind, Scorch: s, Duration: dur})
+		res = s.onEvent(Event{Kind: kind, Scorch: s, Duration: dur})
 		atomic.AddUint64(&s.stats.TotEventTriggerCompleted, 1)
 	}
+	return res
 }
 
 func (s *Scorch) fireAsyncError(err error) {


Test patch:
diff --git a/index/scorch/event_test.go b/index/scorch/event_test.go
index ed1dd4d9..be163a9e 100644
--- a/index/scorch/event_test.go
+++ b/index/scorch/event_test.go
@@ -35,10 +35,11 @@ func TestEventBatchIntroductionStart(t *testing.T) {
 	}()
 
 	var count int
-	RegistryEventCallbacks["test"] = func(e Event) {
+	RegistryEventCallbacks["test"] = func(e Event) bool {
 		if e.Kind == EventKindBatchIntroductionStart {
 			count++
 		}
+		return true
 	}
 
 	ourConfig := make(map[string]interface{}, len(testConfig))
diff --git a/index/scorch/merge_test.go b/index/scorch/merge_test.go
index b3783c6c..ec7e68ea 100644
--- a/index/scorch/merge_test.go
+++ b/index/scorch/merge_test.go
@@ -41,7 +41,7 @@ func TestObsoleteSegmentMergeIntroduction(t *testing.T) {
 	mergeIntroStart.Add(1)
 	mergeIntroComplete.Add(1)
 	var segIntroCompleted int
-	RegistryEventCallbacks["test"] = func(e Event) {
+	RegistryEventCallbacks["test"] = func(e Event) bool {
 		if e.Kind == EventKindBatchIntroduction {
 			segIntroCompleted++
 			if segIntroCompleted == 3 {
@@ -61,6 +61,7 @@ func TestObsoleteSegmentMergeIntroduction(t *testing.T) {
 			mergeIntroComplete.Done()
 
 		}
+		return true
 	}
 
 	ourConfig := make(map[string]interface{}, len(testConfig))