Repository: blevesearch/bleve
Pull request #1371 (2020-04-15 17:30:20)

Fix slice out-of-bounds panic within highlighter
Addresses https://github.com/blevesearch/bleve/issues/1370


Patch:
diff --git a/search/highlight/fragmenter/simple/simple.go b/search/highlight/fragmenter/simple/simple.go
index 6f6ecedf..9c63f7fb 100644
--- a/search/highlight/fragmenter/simple/simple.go
+++ b/search/highlight/fragmenter/simple/simple.go
@@ -58,6 +58,11 @@ OUTER:
 		// push back towards beginning
 		// without cross maxbegin
 		for start > 0 && used < s.fragmentSize {
+			if start > len(orig) {
+				// bail if out of bounds, possibly due to token replacement
+				// e.g with a regexp replacement
+				continue OUTER
+			}
 			r, size := utf8.DecodeLastRune(orig[0:start])
 			if r == utf8.RuneError {
 				continue OUTER // bail


Test patch:
diff --git a/search_test.go b/search_test.go
index 7cfd4473..50faaedd 100644
--- a/search_test.go
+++ b/search_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/blevesearch/bleve/analysis/analyzer/custom"
 	"github.com/blevesearch/bleve/analysis/analyzer/keyword"
 	"github.com/blevesearch/bleve/analysis/analyzer/standard"
+	regexp_char_filter "github.com/blevesearch/bleve/analysis/char/regexp"
 	"github.com/blevesearch/bleve/analysis/token/length"
 	"github.com/blevesearch/bleve/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/analysis/token/shingle"
@@ -38,6 +39,7 @@ import (
 	"github.com/blevesearch/bleve/index/upsidedown"
 	"github.com/blevesearch/bleve/mapping"
 	"github.com/blevesearch/bleve/search"
+	"github.com/blevesearch/bleve/search/highlight/highlighter/ansi"
 	"github.com/blevesearch/bleve/search/highlight/highlighter/html"
 	"github.com/blevesearch/bleve/search/query"
 )
@@ -1639,3 +1641,69 @@ func TestGeoDistanceIssue1301(t *testing.T) {
 		t.Fatalf("Size expected: 3, actual %d\n", sr.Total)
 	}
 }
+
+func TestSearchHighlightingWithRegexpReplacement(t *testing.T) {
+	idxMapping := NewIndexMapping()
+	if err := idxMapping.AddCustomCharFilter(regexp_char_filter.Name, map[string]interface{}{
+		"regexp":  `([a-z])\s+(\d)`,
+		"replace": "ooooo$1-$2",
+		"type":    regexp_char_filter.Name,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := idxMapping.AddCustomAnalyzer("regexp_replace", map[string]interface{}{
+		"type":      custom.Name,
+		"tokenizer": "unicode",
+		"char_filters": []string{
+			regexp_char_filter.Name,
+		},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	idxMapping.DefaultAnalyzer = "regexp_replace"
+	idxMapping.StoreDynamic = true
+	idx, err := NewUsing("testidx", idxMapping, scorch.Name, Config.DefaultKVStore, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = os.RemoveAll("testidx")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	doc := map[string]interface{}{
+		"status": "fool 10",
+	}
+
+	batch := idx.NewBatch()
+	if err = batch.Index("doc", doc); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = idx.Batch(batch); err != nil {
+		t.Fatal(err)
+	}
+
+	query := NewMatchQuery("fool 10")
+	sreq := NewSearchRequest(query)
+	sreq.Fields = []string{"*"}
+	sreq.Highlight = NewHighlightWithStyle(ansi.Name)
+
+	sres, err := idx.Search(sreq)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if sres.Total != 1 {
+		t.Fatalf("Expected 1 hit, got: %v", sres.Total)
+	}
+}