Repository: blevesearch/bleve (go)

Issue:
HTML highlighter should escape its input
Hi!

I'm trying out HTML highlighter. I believe the highlighter takes plain text as input and outputs HTML text. Thus, it should treat its input as plain text and properly escape it when adding `<mark>` tags.

Let's say I have three documents:

    id: "doc1", text: "Lorem ipsum dolor sit amet"
    id: "doc2", text: "Thelma & Louise"
    id: "doc3", text: "I <3 Blancmange"
    id: "doc4", text: "The <script> tag is potentially dangerous"

If I search for `Lorem`, I get a nice HTML fragment `<mark>Lorem</mark> ipsum dolor sit amet`.

However, if I search for `Thelma`, I get `<mark>Thelma & Louise</mark>`. I would expect to get `<mark>Thelma &amp; Louise</mark>`.

Searching for `tag` would result in e.g. `The <script> <mark>tag</mark> is potentially dangerous`, which wound hide the rest of the HTML page after "The".

Am I possibly missing something, or should the highlighter do the escaping?

Pull request description:
escape the input text in HTML highlighter when adding <mark> tags, fixes #1464

Patch:
diff --git a/analysis/char/asciifolding/asciifolding.go b/analysis/char/asciifolding/asciifolding.go
index 469102e2..facce07e 100644
--- a/analysis/char/asciifolding/asciifolding.go
+++ b/analysis/char/asciifolding/asciifolding.go
@@ -874,7 +874,7 @@ func foldToASCII(input []rune, inputPos int, output []rune, outputPos int, lengt
 				outputPos++
 
 			case '\u24A2': // ⒢ [PARENTHESIZED LATIN SMALL LETTER G]
-				output = output[:(len(output) + 1)]
+				output = output[:(len(output) + 2)]
 				output[outputPos] = '('
 				outputPos++
 				output[outputPos] = 'g'
diff --git a/config.go b/config.go
index 482efb40..99f2e081 100644
--- a/config.go
+++ b/config.go
@@ -43,9 +43,16 @@ type configuration struct {
 }
 
 func (c *configuration) SetAnalysisQueueSize(n int) {
+	if c.analysisQueue != nil {
+		c.analysisQueue.Close()
+	}
 	c.analysisQueue = index.NewAnalysisQueue(n)
 }
 
+func (c *configuration) Shutdown() {
+	c.SetAnalysisQueueSize(0)
+}
+
 func newConfiguration() *configuration {
 	return &configuration{
 		Cache:         registry.NewCache(),
diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go
index ba98a460..fccff67a 100644
--- a/index/scorch/scorch.go
+++ b/index/scorch/scorch.go
@@ -515,21 +515,17 @@ func (s *Scorch) diskFileStats(rootSegmentPaths map[string]struct{}) (uint64,
 	return numFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot
 }
 
-func (s *Scorch) rootDiskSegmentsPaths() map[string]struct{} {
-	rv := make(map[string]struct{}, len(s.root.segment))
-	for _, segmentSnapshot := range s.root.segment {
-		if seg, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok {
-			rv[seg.Path()] = struct{}{}
-		}
-	}
-	return rv
-}
-
 func (s *Scorch) StatsMap() map[string]interface{} {
 	m := s.stats.ToMap()
 
+	indexSnapshot := s.currentSnapshot()
+	defer func() {
+		_ = indexSnapshot.Close()
+	}()
+
+	rootSegPaths := indexSnapshot.diskSegmentsPaths()
+
 	s.rootLock.RLock()
-	rootSegPaths := s.rootDiskSegmentsPaths()
 	m["CurFilesIneligibleForRemoval"] = uint64(len(s.ineligibleForRemoval))
 	s.rootLock.RUnlock()
 
@@ -556,6 +552,10 @@ func (s *Scorch) StatsMap() map[string]interface{} {
 	m["num_bytes_used_disk"] = numBytesUsedDisk
 	// total disk bytes by the latest root index, exclusive of older snapshots
 	m["num_bytes_used_disk_by_root"] = numBytesOnDiskByRoot
+	// num_bytes_used_disk_by_root_reclaimable is an approximation about the
+	// reclaimable disk space in an index. (eg: from a full compaction)
+	m["num_bytes_used_disk_by_root_reclaimable"] = uint64(float64(numBytesOnDiskByRoot) *
+		indexSnapshot.reClaimableDocsRatio())
 	m["num_files_on_disk"] = numFilesOnDisk
 	m["num_root_memorysegments"] = m["TotMemorySegmentsAtRoot"]
 	m["num_root_filesegments"] = m["TotFileSegmentsAtRoot"]
diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go
index 9d17bcb2..61204ebb 100644
--- a/index/scorch/snapshot_index.go
+++ b/index/scorch/snapshot_index.go
@@ -708,6 +708,33 @@ func (i *IndexSnapshot) DumpFields() chan interface{} {
 	return rv
 }
 
+func (i *IndexSnapshot) diskSegmentsPaths() map[string]struct{} {
+	rv := make(map[string]struct{}, len(i.segment))
+	for _, segmentSnapshot := range i.segment {
+		if seg, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok {
+			rv[seg.Path()] = struct{}{}
+		}
+	}
+	return rv
+}
+
+// reClaimableDocsRatio gives a ratio about the obsoleted or
+// reclaimable documents present in a given index snapshot.
+func (i *IndexSnapshot) reClaimableDocsRatio() float64 {
+	var totalCount, liveCount uint64
+	for _, segmentSnapshot := range i.segment {
+		if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok {
+			totalCount += uint64(segmentSnapshot.FullSize())
+			liveCount += uint64(segmentSnapshot.Count())
+		}
+	}
+
+	if totalCount > 0 {
+		return float64(totalCount-liveCount) / float64(totalCount)
+	}
+	return 0
+}
+
 // subtractStrings returns set a minus elements of set b.
 func subtractStrings(a, b []string) []string {
 	if len(b) == 0 {
diff --git a/index_impl.go b/index_impl.go
index 629cc9b2..0520fe43 100644
--- a/index_impl.go
+++ b/index_impl.go
@@ -112,6 +112,11 @@ func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string,
 		}
 		return nil, err
 	}
+	defer func(rv *indexImpl) {
+		if !rv.open {
+			rv.i.Close()
+		}
+	}(&rv)
 
 	// now persist the mapping
 	mappingBytes, err := json.Marshal(mapping)
@@ -177,6 +182,11 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde
 		}
 		return nil, err
 	}
+	defer func(rv *indexImpl) {
+		if !rv.open {
+			rv.i.Close()
+		}
+	}(rv)
 
 	// now load the mapping
 	indexReader, err := rv.i.Reader()
diff --git a/search/highlight/format/html/html.go b/search/highlight/format/html/html.go
index 8154e790..259a0379 100644
--- a/search/highlight/format/html/html.go
+++ b/search/highlight/format/html/html.go
@@ -15,6 +15,8 @@
 package html
 
 import (
+	"html"
+
 	"github.com/blevesearch/bleve/registry"
 	"github.com/blevesearch/bleve/search/highlight"
 )
@@ -54,18 +56,18 @@ func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations h
 			break
 		}
 		// add the stuff before this location
-		rv += string(f.Orig[curr:termLocation.Start])
-		// add the color
+		rv += html.EscapeString(string(f.Orig[curr:termLocation.Start]))
+		// start the <mark> tag
 		rv += a.before
 		// add the term itself
 		rv += string(f.Orig[termLocation.Start:termLocation.End])
-		// reset the color
+		// end the <mark> tag
 		rv += a.after
 		// update current
 		curr = termLocation.End
 	}
 	// add any remaining text after the last token
-	rv += string(f.Orig[curr:f.End])
+	rv += html.EscapeString(string(f.Orig[curr:f.End]))
 
 	return rv
 }


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index bcafb812..f57aa35e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -8,7 +8,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.13.x, 1.14.x]
+        go-version: [1.13.x, 1.14.x, 1.15.x]
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:
diff --git a/analysis/char/asciifolding/asciifolding_test.go b/analysis/char/asciifolding/asciifolding_test.go
index d79542a8..3eafd8cd 100644
--- a/analysis/char/asciifolding/asciifolding_test.go
+++ b/analysis/char/asciifolding/asciifolding_test.go
@@ -48,6 +48,10 @@ func TestAsciiFoldingFilter(t *testing.T) {
 			// apples from https://issues.couchbase.com/browse/MB-33486
 			input:  []byte(`Ápple Àpple Äpple Âpple Ãpple Åpple`),
 			output: []byte(`Apple Apple Apple Apple Apple Apple`),
+		}, {
+			// Fix ASCII folding of \u24A2
+			input:  []byte(`⒢`),
+			output: []byte(`(g)`),
 		},
 	}
 
diff --git a/index_test.go b/index_test.go
index c535456b..50684420 100644
--- a/index_test.go
+++ b/index_test.go
@@ -2135,6 +2135,11 @@ func testBatchRaceBug1149(t *testing.T, i Index) {
 }
 
 func TestOptimisedConjunctionSearchHits(t *testing.T) {
+	scorch.OptimizeDisjunctionUnadorned = false
+	defer func() {
+		scorch.OptimizeDisjunctionUnadorned = true
+	}()
+
 	defer func() {
 		err := os.RemoveAll("testidx")
 		if err != nil {
diff --git a/mapping/mapping_test.go b/mapping/mapping_test.go
index 2b687096..1c9ced58 100644
--- a/mapping/mapping_test.go
+++ b/mapping/mapping_test.go
@@ -959,7 +959,7 @@ func TestMappingForGeo(t *testing.T) {
 	expect = append(expect, []float64{-71.34, 41.12})
 
 	for i, geopoint := range geopoints {
-		doc := document.NewDocument(string(i))
+		doc := document.NewDocument(fmt.Sprint(i))
 		err := mapping.MapDocument(doc, geopoint)
 		if err != nil {
 			t.Fatal(err)
diff --git a/test/tests/basic/data/b.json b/test/tests/basic/data/b.json
index 58118a55..06972727 100644
--- a/test/tests/basic/data/b.json
+++ b/test/tests/basic/data/b.json
@@ -1,6 +1,6 @@
 {
 	"id": "b",
-	"name": "steve has a long name",
+	"name": "steve has <a> long & complicated name",
 	"age": 27,
 	"birthday": "2001-09-09T01:46:40Z",
 	"title": "missess"
diff --git a/test/tests/basic/searches.json b/test/tests/basic/searches.json
index 8b6206c3..7ddfce37 100644
--- a/test/tests/basic/searches.json
+++ b/test/tests/basic/searches.json
@@ -42,7 +42,7 @@
 			"size": 10,
 			"sort": ["-_score", "_id"],
 			"query": {
-				"match_phrase": "long name"
+				"match_phrase": "steve has"
 			}
 		},
 		"result": {
@@ -385,7 +385,7 @@
 				{
 					"id": "b",
 					"fragments": {
-						"name": ["steve has a <mark>long</mark> name"]
+						"name": ["steve has &lt;a&gt; <mark>long</mark> &amp; complicated name"]
 					}
 				}
 			]
@@ -409,7 +409,7 @@
 				{
 					"id": "b",
 					"fragments": {
-						"name": ["steve has a <mark>long</mark> name"]
+						"name": ["steve has &lt;a&gt; <mark>long</mark> &amp; complicated name"]
 					}
 				}
 			]
@@ -485,7 +485,7 @@
 				{
 					"id": "b",
 					"fragments": {
-						"name": ["steve has a <mark>long</mark> name"],
+						"name": ["steve has &lt;a&gt; <mark>long</mark> &amp; complicated name"],
 						"title": ["missess"]
 					}
 				}