Repository: blevesearch/bleve
Pull request #2141 (2025-02-20 10:31:01)

MB-65284: Add synonym query tracking and update stats map
- Add a Scorch counter stat `TotSynonymSearches` to track the number of synonym-enabled queries received by the index. This stat will be incremented by 1 each time the FieldTermSynonymMap is set, indicating that the query will use synonyms.


Patch:
diff --git a/document/field_boolean.go b/document/field_boolean.go
index 8c2987a7..fdd14ce9 100644
--- a/document/field_boolean.go
+++ b/document/field_boolean.go
@@ -116,13 +116,13 @@ func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte
 		name:              name,
 		arrayPositions:    arrayPositions,
 		value:             value,
-		options:           DefaultNumericIndexingOptions,
+		options:           DefaultBooleanIndexingOptions,
 		numPlainTextBytes: uint64(len(value)),
 	}
 }
 
 func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField {
-	return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultNumericIndexingOptions)
+	return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultBooleanIndexingOptions)
 }
 
 func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options index.FieldIndexingOptions) *BooleanField {
diff --git a/document/field_ip.go b/document/field_ip.go
index 80a353a0..3a5ab379 100644
--- a/document/field_ip.go
+++ b/document/field_ip.go
@@ -31,7 +31,7 @@ func init() {
 	reflectStaticSizeIPField = int(reflect.TypeOf(f).Size())
 }
 
-const DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues | index.IncludeTermVectors
+const DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues
 
 type IPField struct {
 	name              string
@@ -115,7 +115,7 @@ func NewIPFieldFromBytes(name string, arrayPositions []uint64, value []byte) *IP
 		name:              name,
 		arrayPositions:    arrayPositions,
 		value:             value,
-		options:           DefaultNumericIndexingOptions,
+		options:           DefaultIPIndexingOptions,
 		numPlainTextBytes: uint64(len(value)),
 	}
 }
diff --git a/go.mod b/go.mod
index db61f7ad..fa9f80bc 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/RoaringBitmap/roaring/v2 v2.4.5
 	github.com/bits-and-blooms/bitset v1.12.0
-	github.com/blevesearch/bleve_index_api v1.2.1
+	github.com/blevesearch/bleve_index_api v1.2.2-0.20250220160913-17c09f7a4a40
 	github.com/blevesearch/geo v0.1.20
 	github.com/blevesearch/go-faiss v1.0.24
 	github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
diff --git a/go.sum b/go.sum
index 4fb0af8c..e7448b78 100644
--- a/go.sum
+++ b/go.sum
@@ -2,8 +2,8 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2
 github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0=
 github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
 github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
-github.com/blevesearch/bleve_index_api v1.2.1 h1:IuXwLvmyp7I7+e0FOA68gcHHLfzSQ4AqQ8wVab5uxk0=
-github.com/blevesearch/bleve_index_api v1.2.1/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
+github.com/blevesearch/bleve_index_api v1.2.2-0.20250220160913-17c09f7a4a40 h1:HmfirZlJf9gdQE8TCAF10cZ/2ubydgK7vCcLojCqmt0=
+github.com/blevesearch/bleve_index_api v1.2.2-0.20250220160913-17c09f7a4a40/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
 github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
 github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
 github.com/blevesearch/go-faiss v1.0.24 h1:K79IvKjoKHdi7FdiXEsAhxpMuns0x4fM0BO93bW5jLI=
diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go
index f8d532e0..2b540851 100644
--- a/index/scorch/scorch.go
+++ b/index/scorch/scorch.go
@@ -635,6 +635,7 @@ func (s *Scorch) StatsMap() map[string]interface{} {
 	m["term_searchers_started"] = m["TotTermSearchersStarted"]
 	m["term_searchers_finished"] = m["TotTermSearchersFinished"]
 	m["knn_searches"] = m["TotKNNSearches"]
+	m["synonym_searches"] = m["TotSynonymSearches"]
 
 	m["num_bytes_read_at_query_time"] = m["TotBytesReadAtQueryTime"]
 	m["num_plain_text_bytes_indexed"] = m["TotIndexedPlainTextBytes"]
diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go
index fe639d09..790a8008 100644
--- a/index/scorch/segment_plugin.go
+++ b/index/scorch/segment_plugin.go
@@ -18,8 +18,8 @@ import (
 	"fmt"
 
 	"github.com/RoaringBitmap/roaring/v2"
-	index "github.com/blevesearch/bleve_index_api"
 	"github.com/blevesearch/bleve/v2/geo"
+	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 
 	zapv11 "github.com/blevesearch/zapx/v11"
diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go
index ddca76ec..dc96cac4 100644
--- a/index/scorch/snapshot_index.go
+++ b/index/scorch/snapshot_index.go
@@ -1134,3 +1134,7 @@ func (is *IndexSnapshot) ThesaurusKeysRegexp(name string,
 		return is.AutomatonIterator(a, prefixBeg, prefixEnd)
 	})
 }
+
+func (is *IndexSnapshot) UpdateSynonymSearchCount(delta uint64) {
+	atomic.AddUint64(&is.parent.stats.TotSynonymSearches, delta)
+}
diff --git a/index/scorch/stats.go b/index/scorch/stats.go
index 269ae2f6..397b9894 100644
--- a/index/scorch/stats.go
+++ b/index/scorch/stats.go
@@ -51,7 +51,8 @@ type Stats struct {
 	TotTermSearchersStarted  uint64
 	TotTermSearchersFinished uint64
 
-	TotKNNSearches uint64
+	TotKNNSearches     uint64
+	TotSynonymSearches uint64
 
 	TotEventTriggerStarted   uint64
 	TotEventTriggerCompleted uint64
diff --git a/index_impl.go b/index_impl.go
index d59dfb9a..c04f20e5 100644
--- a/index_impl.go
+++ b/index_impl.go
@@ -636,6 +636,9 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 	setKnnHitsInCollector(knnHits, req, coll)
 
 	if fts != nil {
+		if is, ok := indexReader.(*scorch.IndexSnapshot); ok {
+			is.UpdateSynonymSearchCount(1)
+		}
 		ctx = context.WithValue(ctx, search.FieldTermSynonymMapKey, fts)
 	}
 
@@ -910,6 +913,11 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,
 								if err == nil {
 									value = v
 								}
+							case index.IPField:
+								ip, err := docF.IP()
+								if err == nil {
+									value = ip.String()
+								}
 							}
 
 							if value != nil {
diff --git a/search/query/ip_range.go b/search/query/ip_range.go
index ba46f0b2..6c447c2c 100644
--- a/search/query/ip_range.go
+++ b/search/query/ip_range.go
@@ -26,7 +26,7 @@ import (
 )
 
 type IPRangeQuery struct {
-	CIDR     string `json:"cidr, omitempty"`
+	CIDR     string `json:"cidr,omitempty"`
 	FieldVal string `json:"field,omitempty"`
 	BoostVal *Boost `json:"boost,omitempty"`
 }


Test patch:
diff --git a/mapping/mapping_vectors_test.go b/mapping/mapping_vectors_test.go
index 6d409760..6b6f92e8 100644
--- a/mapping/mapping_vectors_test.go
+++ b/mapping/mapping_vectors_test.go
@@ -310,18 +310,18 @@ func TestProcessVector(t *testing.T) {
 
 func TestNormalizeVector(t *testing.T) {
 	vectors := [][]float32{
-		[]float32{1,2,3,4,5},
-		[]float32{1,0,0,0,0},
-		[]float32{0.182574183,0.365148365,0.547722578,0.730296731},
-		[]float32{1,1,1,1,1,1,1,1},
+		[]float32{1, 2, 3, 4, 5},
+		[]float32{1, 0, 0, 0, 0},
+		[]float32{0.182574183, 0.365148365, 0.547722578, 0.730296731},
+		[]float32{1, 1, 1, 1, 1, 1, 1, 1},
 		[]float32{0},
 	}
 
 	expectedNormalizedVectors := [][]float32{
-		[]float32{0.13483998,0.26967996,0.40451995,0.5393599,0.67419994},
-		[]float32{1,0,0,0,0},
-		[]float32{0.18257418,0.36514837,0.5477226,0.73029673},
-		[]float32{0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338,0.35355338},
+		[]float32{0.13483998, 0.26967996, 0.40451995, 0.5393599, 0.67419994},
+		[]float32{1, 0, 0, 0, 0},
+		[]float32{0.18257418, 0.36514837, 0.5477226, 0.73029673},
+		[]float32{0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338},
 		[]float32{0},
 	}
 
diff --git a/search_test.go b/search_test.go
index e00d5ad0..e21d1998 100644
--- a/search_test.go
+++ b/search_test.go
@@ -1387,7 +1387,6 @@ func TestMatchQueryPartialMatch(t *testing.T) {
 		t.Errorf("Expected 1 result, but got: %v", res.Total)
 	}
 	hit := res.Hits[0]
-	fmt.Println(hit.Expl, hit.ID)
 	if hit.ID != "doc1" || hit.Expl.PartialMatch {
 		t.Errorf("Expected doc1 to be a full match")
 	}
@@ -4319,6 +4318,15 @@ func TestSynonymSearchQueries(t *testing.T) {
 		},
 	}
 
+	getTotalSynonymSearchStat := func(idx Index) int {
+		ir, err := idx.Advanced()
+		if err != nil {
+			t.Fatal(err)
+		}
+		stat := ir.StatsMap()["synonym_searches"].(uint64)
+		return int(stat)
+	}
+
 	runTestQueries := func(idx Index) error {
 		for _, dtq := range testQueries {
 			q, err := query.ParseQuery([]byte(dtq.query))
@@ -4352,6 +4360,12 @@ func TestSynonymSearchQueries(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	// now verify that the stat for number of synonym enabled queries is correct
+	totalSynonymSearchStat := getTotalSynonymSearchStat(idx)
+	if totalSynonymSearchStat != len(testQueries) {
+		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
+	}
+
 	// test with index alias - with 1 batch per index
 	numIndexes := len(batches)
 	indexes := make([]Index, numIndexes)
@@ -4384,6 +4398,20 @@ func TestSynonymSearchQueries(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	// verify the synonym search stat for the alias
+	totalSynonymSearchStat = getTotalSynonymSearchStat(indexes[0])
+	if totalSynonymSearchStat != len(testQueries) {
+		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
+	}
+	for i := 1; i < numIndexes; i++ {
+		idxStat := getTotalSynonymSearchStat(indexes[i])
+		if idxStat != totalSynonymSearchStat {
+			t.Fatalf("expected %d synonym searches, got %d", totalSynonymSearchStat, idxStat)
+		}
+	}
+	if totalSynonymSearchStat != len(testQueries) {
+		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
+	}
 	// test with multi-level alias now with two index per alias
 	// and having any extra index being in the final alias
 	numAliases := numIndexes / 2
@@ -4405,6 +4433,17 @@ func TestSynonymSearchQueries(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	// verify the synonym searches stat for the alias
+	totalSynonymSearchStat = getTotalSynonymSearchStat(indexes[0])
+	if totalSynonymSearchStat != 2*len(testQueries) {
+		t.Fatalf("expected %d synonym searches, got %d", len(testQueries), totalSynonymSearchStat)
+	}
+	for i := 1; i < numIndexes; i++ {
+		idxStat := getTotalSynonymSearchStat(indexes[i])
+		if idxStat != totalSynonymSearchStat {
+			t.Fatalf("expected %d synonym searches, got %d", totalSynonymSearchStat, idxStat)
+		}
+	}
 }
 
 func TestGeoDistanceInSort(t *testing.T) {