Repository: blevesearch/bleve
Pull request #1641 (2021-12-08 22:41:54)

Use of HTML character filter breaks highlighting
+ I've tracked this down as a regression that was introduced
   when I changed the regexp character filter's replacement
   behavior here - https://github.com/blevesearch/bleve/pull/1351
+ The above change is still necessary for highlighting to work
    correctly when regexp character filter is used and for
    replacements that involve '$' variables.

+ The HTML character filter which uses the regexp character
    filter will need to replace every character of the matched
    sequence with whitespace so that the number of whitespaces
    equals the length of the HTML tag.
+ Tracking ticket: https://issues.couchbase.com/browse/MB-50002


Patch:
diff --git a/analysis/char/html/html.go b/analysis/char/html/html.go
index 922edef3..4cf2c630 100644
--- a/analysis/char/html/html.go
+++ b/analysis/char/html/html.go
@@ -15,10 +15,10 @@
 package html
 
 import (
+	"bytes"
 	"regexp"
 
 	"github.com/blevesearch/bleve/v2/analysis"
-	regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
 	"github.com/blevesearch/bleve/v2/registry"
 )
 
@@ -26,9 +26,27 @@ const Name = "html"
 
 var htmlCharFilterRegexp = regexp.MustCompile(`</?[!\w]+((\s+\w+(\s*=\s*(?:".*?"|'.*?'|[^'">\s]+))?)+\s*|\s*)/?>`)
 
+type CharFilter struct {
+	r           *regexp.Regexp
+	replacement []byte
+}
+
+func New() *CharFilter {
+	return &CharFilter{
+		r:           htmlCharFilterRegexp,
+		replacement: []byte(" "),
+	}
+}
+
+func (s *CharFilter) Filter(input []byte) []byte {
+	return s.r.ReplaceAllFunc(
+		input, func(in []byte) []byte {
+			return bytes.Repeat(s.replacement, len(in))
+		})
+}
+
 func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) {
-	replaceBytes := []byte(" ")
-	return regexpCharFilter.New(htmlCharFilterRegexp, replaceBytes), nil
+	return New(), nil
 }
 
 func init() {


Test patch:
diff --git a/search_test.go b/search_test.go
index bfdbd878..ee8846a0 100644
--- a/search_test.go
+++ b/search_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
+	html_char_filter "github.com/blevesearch/bleve/v2/analysis/char/html"
 	regexp_char_filter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
 	"github.com/blevesearch/bleve/v2/analysis/token/length"
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
@@ -1810,3 +1811,81 @@ func TestAnalyzerInheritance(t *testing.T) {
 		})
 	}
 }
+
+func TestHightlightingWithHTMLCharacterFilter(t *testing.T) {
+	idxMapping := NewIndexMapping()
+	if err := idxMapping.AddCustomAnalyzer("custom-html", map[string]interface{}{
+		"type":         custom.Name,
+		"tokenizer":    "unicode",
+		"char_filters": []interface{}{html_char_filter.Name},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	fm := mapping.NewTextFieldMapping()
+	fm.Analyzer = "custom-html"
+
+	dmap := mapping.NewDocumentMapping()
+	dmap.AddFieldMappingsAt("content", fm)
+
+	idxMapping.DefaultMapping = dmap
+
+	tmpIndexPath := createTmpIndexPath(t)
+	defer cleanupTmpIndexPath(t, tmpIndexPath)
+
+	idx, err := New(tmpIndexPath, idxMapping)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		err = idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	content := "<div> Welcome to blevesearch. </div>"
+	if err = idx.Index("doc", map[string]string{
+		"content": content,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	searchStr := "blevesearch"
+	q := query.NewMatchQuery(searchStr)
+	q.SetField("content")
+	sr := NewSearchRequest(q)
+	sr.IncludeLocations = true
+	sr.Fields = []string{"*"}
+	sr.Highlight = NewHighlightWithStyle(html.Name)
+	searchResults, err := idx.Search(sr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(searchResults.Hits) != 1 ||
+		len(searchResults.Hits[0].Locations["content"][searchStr]) != 1 {
+		t.Fatalf("Expected 1 hit with 1 location")
+	}
+
+	expectedLocation := &search.Location{
+		Pos:   3,
+		Start: uint64(strings.Index(content, searchStr)),
+		End:   uint64(strings.Index(content, searchStr) + len(searchStr)),
+	}
+	expectedFragment := "&lt;div&gt; Welcome to <mark>blevesearch</mark>. &lt;/div&gt;"
+
+	gotLocation := searchResults.Hits[0].Locations["content"]["blevesearch"][0]
+	gotFragment := searchResults.Hits[0].Fragments["content"][0]
+
+	if !reflect.DeepEqual(expectedLocation, gotLocation) {
+		t.Fatalf("Mismatch in locations, got: %v, expected: %v",
+			gotLocation, expectedLocation)
+	}
+
+	if expectedFragment != gotFragment {
+		t.Fatalf("Mismatch in fragment, got: %v, expected: %v",
+			gotFragment, expectedFragment)
+	}
+}