Repository: wagtail/wagtail

Issue:
#6099 can generate wrong search index content
### Issue Summary

#6099 strips out HTML tags without considering that some create line breaks. This can generate searchable content that doesn't exist in the field.


### Steps to Reproduce

Type two lines in a paragraph in a RichTextBlock:
line one
line two

The paragraph's value will be: 
```
<p>line one</p><p>line two</p>
```

get_searchable_content() will return 3 words: 'line oneline two'

* I have confirmed that this issue can be reproduced as described on a fresh Wagtail project: (no)




Discussion:
This behaviour comes from Django's `strip_tags()`. As an alternative, we could use something like

`re.sub('<[^<]+?>', ' ', str)`
We probably want slightly different behaviour for block tags and inline tags. For example, someone might've made part of a word bold `<b>Te</b>st` but this would still need to be interpreted as one word.

This case is quite rare though so probably better to add spaces than not to so paragraphs are indexed properly. I've been pushing for switching to a JSON representation in rich text which should lead to a better solution for this (RFC coming this week hopefulyl!)
Looking at [the ticket that led to Django's current parser-based `strip_tags` implementation](https://code.djangoproject.com/ticket/19237), it seems like a regexp version is easy to get wrong. While Wagtail rich text is hopefully controlled enough that it avoids the worst edge cases, I'm not sure I'd want to risk a stray bracket obliterating an entire field and making it unindexable.

https://stackoverflow.com/questions/40139455/django-strip-tags-template-filter-add-space has a neat alternative (and a shoutout to Wagtail too :-) ) - run a search-and-replace on `</p>` tags to insert a space before running `strip_tags`. As a bonus, that also avoids the `<b>Te</b>st` issue. There are a few other tags we'd want to match besides `</p>`, but I think it's a small manageable list: `</p>`, `</h1>`-`</h6>`, `</li>`, `</blockquote>`. (Any others I've forgotten...?)

Resolving patch:
diff --git a/wagtail/core/blocks/field_block.py b/wagtail/core/blocks/field_block.py
--- a/wagtail/core/blocks/field_block.py
+++ b/wagtail/core/blocks/field_block.py
@@ -1,5 +1,4 @@
 import datetime
-from html import unescape
 
 from django import forms
 from django.db.models.fields import BLANK_CHOICE_DASH
@@ -8,10 +7,10 @@
 from django.utils.dateparse import parse_date, parse_datetime, parse_time
 from django.utils.encoding import force_str
 from django.utils.functional import cached_property
-from django.utils.html import format_html, strip_tags
+from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
-from wagtail.core.rich_text import RichText
+from wagtail.core.rich_text import RichText, get_text_for_indexing
 from wagtail.core.utils import resolve_model_string
 
 from .base import Block
@@ -573,9 +572,9 @@ def value_from_form(self, value):
         return RichText(value)
 
     def get_searchable_content(self, value):
-        # Strip HTML tags to prevent search backend earch backend from indexing them
+        # Strip HTML tags to prevent search backend from indexing them
         source = force_str(value.source)
-        return [unescape(strip_tags(source))]
+        return [get_text_for_indexing(source)]
 
     class Meta:
         icon = "doc-full"
diff --git a/wagtail/core/fields.py b/wagtail/core/fields.py
--- a/wagtail/core/fields.py
+++ b/wagtail/core/fields.py
@@ -1,12 +1,11 @@
 import json
-from html import unescape
 
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils.encoding import force_str
-from django.utils.html import strip_tags
 
 from wagtail.core.blocks import Block, BlockField, StreamBlock, StreamValue
+from wagtail.core.rich_text import get_text_for_indexing
 
 
 class RichTextField(models.TextField):
@@ -23,9 +22,9 @@ def formfield(self, **kwargs):
         return super().formfield(**defaults)
 
     def get_searchable_content(self, value):
-        # Strip HTML tags to prevent search backend earch backend from indexing them
+        # Strip HTML tags to prevent search backend from indexing them
         source = force_str(value)
-        return [unescape(strip_tags(source))]
+        return [get_text_for_indexing(source)]
 
 
 # https://github.com/django/django/blob/64200c14e0072ba0ffef86da46b2ea82fd1e019a/django/db/models/fields/subclassing.py#L31-L44
diff --git a/wagtail/core/rich_text/__init__.py b/wagtail/core/rich_text/__init__.py
--- a/wagtail/core/rich_text/__init__.py
+++ b/wagtail/core/rich_text/__init__.py
@@ -1,5 +1,9 @@
+import re
+from html import unescape
+
 from django.db.models import Model
 from django.template.loader import render_to_string
+from django.utils.html import strip_tags
 from django.utils.safestring import mark_safe
 
 from wagtail.core.rich_text.feature_registry import FeatureRegistry
@@ -33,6 +37,19 @@ def expand_db_html(html):
     return FRONTEND_REWRITER(html)
 
 
+def get_text_for_indexing(richtext):
+    """
+    Return a plain text version of a rich text string, suitable for search indexing;
+    like Django's strip_tags, but ensures that whitespace is left between block elements
+    so that <p>hello</p><p>world</p> gives "hello world", not "helloworld".
+    """
+    # insert space after </p>, </h1> - </h6>, </li> and </blockquote> tags
+    richtext = re.sub(r'(</(p|h\d|li|blockquote)>)', r'\1 ', richtext, flags=re.IGNORECASE)
+    # also insert space after <br /> and <hr />
+    richtext = re.sub(r'(<(br|hr)\s*/>)', r'\1 ', richtext, flags=re.IGNORECASE)
+    return unescape(strip_tags(richtext).strip())
+
+
 class RichText:
     """
     A custom object used to represent a renderable rich text value.


Test patch:
diff --git a/wagtail/core/tests/test_blocks.py b/wagtail/core/tests/test_blocks.py
--- a/wagtail/core/tests/test_blocks.py
+++ b/wagtail/core/tests/test_blocks.py
@@ -558,11 +558,17 @@ def test_get_searchable_content(self):
         result = block.get_searchable_content(value)
         self.assertEqual(
             result, [
-                'Merry Christmas! & a happy new year\n'
+                'Merry Christmas! & a happy new year \n'
                 'Our Santa pet Wagtail has some cool stuff in store for you all!'
             ]
         )
 
+    def test_get_searchable_content_whitespace(self):
+        block = blocks.RichTextBlock()
+        value = RichText('<p>mashed</p><p>po<i>ta</i>toes</p>')
+        result = block.get_searchable_content(value)
+        self.assertEqual(result, ['mashed potatoes'])
+
 
 class TestChoiceBlock(WagtailTestUtils, SimpleTestCase):
     def setUp(self):
diff --git a/wagtail/core/tests/test_rich_text.py b/wagtail/core/tests/test_rich_text.py
--- a/wagtail/core/tests/test_rich_text.py
+++ b/wagtail/core/tests/test_rich_text.py
@@ -173,3 +173,13 @@ def test_get_searchable_content(self):
         value = body_field.value_from_object(christmas_page)
         result = body_field.get_searchable_content(value)
         self.assertEqual(result, ['Merry Christmas from Wagtail! & co.'])
+
+    def test_get_searchable_content_whitespace(self):
+        christmas_page = EventPage.objects.get(url_path='/home/events/christmas/')
+        christmas_page.body = '<p>buttery<br />mashed</p><p>po<i>ta</i>toes</p>'
+        christmas_page.save_revision(submitted_for_moderation=False)
+
+        body_field = christmas_page._meta.get_field('body')
+        value = body_field.value_from_object(christmas_page)
+        result = body_field.get_searchable_content(value)
+        self.assertEqual(result, ['buttery mashed potatoes'])