Repository: wagtail/wagtail

Issue:
Facet tag search fails on Elasticsearch
`test_facet_tags` in the search backend tests fails (returning no results) on Elasticsearch. Have marked it as an expected failure for now (in 1d517dd0d8f3aef67403bacf8ff3724c747a3aca) but need to resolve this for the 2.2 release.


Discussion:

Resolving patch:
diff --git a/wagtail/search/backends/elasticsearch2.py b/wagtail/search/backends/elasticsearch2.py
--- a/wagtail/search/backends/elasticsearch2.py
+++ b/wagtail/search/backends/elasticsearch2.py
@@ -68,7 +68,6 @@ class Elasticsearch2Mapping:
         'IPAddressField': 'string',
         'GenericIPAddressField': 'string',
         'NullBooleanField': 'boolean',
-        'OneToOneField': 'integer',
         'PositiveIntegerField': 'integer',
         'PositiveSmallIntegerField': 'integer',
         'SlugField': 'string',
@@ -264,6 +263,8 @@ def get_document(self, obj):
                     value = list(value.values_list('pk', flat=True))
                 elif isinstance(value, models.Model):
                     value = value.pk
+                elif isinstance(value, (list, tuple)):
+                    value = [item.pk if isinstance(item, models.Model) else item for item in value]
 
             doc[self.get_field_column_name(field)] = value
 
@@ -639,6 +640,7 @@ def facet(self, field_name):
             field_name: {
                 'terms': {
                     'field': column_name,
+                    'missing': 0,
                 }
             }
         }
@@ -651,7 +653,7 @@ def facet(self, field_name):
         )
 
         return OrderedDict([
-            (bucket['key'], bucket['doc_count'])
+            (bucket['key'] if bucket['key'] != 0 else None, bucket['doc_count'])
             for bucket in response['aggregations'][field_name]['buckets']
         ])
 
diff --git a/wagtail/search/index.py b/wagtail/search/index.py
--- a/wagtail/search/index.py
+++ b/wagtail/search/index.py
@@ -201,16 +201,44 @@ def get_type(self, cls):
 
         try:
             field = self.get_field(cls)
+
+            # Follow foreign keys to find underlying type
+            # We use a while loop as it's possible for a foreign key
+            # to target a foreign key in another model.
+            # (for example, a foreign key to a child page model will
+            # point to the `page_ptr_id` field so we need to follow this
+            # second foreign key to find the `id`` field in the Page model)
+            while isinstance(field, RelatedField):
+                field = field.target_field
+
             return field.get_internal_type()
+
         except models.fields.FieldDoesNotExist:
             return 'CharField'
 
     def get_value(self, obj):
+        from taggit.managers import TaggableManager
+
         try:
             field = self.get_field(obj.__class__)
             value = field.value_from_object(obj)
             if hasattr(field, 'get_searchable_content'):
                 value = field.get_searchable_content(value)
+            elif isinstance(field, TaggableManager):
+                # Special case for tags fields. Convert QuerySet of TaggedItems into QuerySet of Tags
+                Tag = field.remote_field.model
+                value = Tag.objects.filter(id__in=value.values_list('tag_id', flat=True))
+            elif isinstance(field, RelatedField):
+                # The type of the ForeignKey may have a get_searchable_content method that we should
+                # call. Firstly we need to find the field its referencing but it may be referencing
+                # another RelatedField (eg an FK to page_ptr_id) so we need to run this in a while
+                # loop to find the actual remote field.
+                remote_field = field
+                while isinstance(remote_field, RelatedField):
+                    remote_field = remote_field.target_field
+
+                if hasattr(remote_field, 'get_searchable_content'):
+                    value = remote_field.get_searchable_content(value)
             return value
         except models.fields.FieldDoesNotExist:
             value = getattr(obj, self.field_name, None)


Test patch:
diff --git a/wagtail/search/tests/elasticsearch_common_tests.py b/wagtail/search/tests/elasticsearch_common_tests.py
--- a/wagtail/search/tests/elasticsearch_common_tests.py
+++ b/wagtail/search/tests/elasticsearch_common_tests.py
@@ -191,8 +191,3 @@ def test_prefix_multiple_words(self):
     @unittest.expectedFailure
     def test_incomplete_plain_text(self):
         super().test_incomplete_plain_text()
-
-    @unittest.expectedFailure
-    def test_facet_tags(self):
-        # Failing on Elasticsearch as of #4508
-        super().test_facet_tags()
diff --git a/wagtail/search/tests/test_backends.py b/wagtail/search/tests/test_backends.py
--- a/wagtail/search/tests/test_backends.py
+++ b/wagtail/search/tests/test_backends.py
@@ -420,10 +420,19 @@ def test_facet_tags(self):
         # The test data doesn't contain any tags, add some
         FANTASY_BOOKS = [1, 2, 3, 4, 5, 6, 7]
         SCIFI_BOOKS = [10]
-        for book_id in FANTASY_BOOKS:
-            models.Book.objects.get(id=book_id).tags.add('Fantasy')
-        for book_id in SCIFI_BOOKS:
-            models.Book.objects.get(id=book_id).tags.add('Science Fiction')
+        for book in models.Book.objects.filter(id__in=FANTASY_BOOKS + SCIFI_BOOKS):
+            book = book.get_indexed_instance()
+
+            if book.id in FANTASY_BOOKS:
+                book.tags.add('Fantasy')
+            if book.id in SCIFI_BOOKS:
+                book.tags.add('Science Fiction')
+
+            self.backend.add(book)
+
+        index = self.backend.get_index_for_model(models.Book)
+        if index:
+            index.refresh()
 
         fantasy_tag = Tag.objects.get(name='Fantasy')
         scifi_tag = Tag.objects.get(name='Science Fiction')
diff --git a/wagtail/search/tests/test_elasticsearch2_backend.py b/wagtail/search/tests/test_elasticsearch2_backend.py
--- a/wagtail/search/tests/test_elasticsearch2_backend.py
+++ b/wagtail/search/tests/test_elasticsearch2_backend.py
@@ -531,6 +531,7 @@ def test_get_mapping(self):
                             'date_of_birth_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
                         },
                     },
+                    'authors_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'publication_date_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
                     'number_of_pages_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'tags': {
@@ -540,7 +541,7 @@ def test_get_mapping(self):
                             'slug_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
                         },
                     },
-                    'tags_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}
+                    'tags_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False}
                 }
             }
         }
@@ -572,6 +573,7 @@ def test_get_document(self):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
@@ -612,13 +614,15 @@ def test_get_mapping(self):
                     'searchtests_novel__protagonist': {
                         'type': 'nested',
                         'properties': {
-                            'name': {'type': 'string', 'boost': 0.5, 'include_in_all': True}
+                            'name': {'type': 'string', 'boost': 0.5, 'include_in_all': True},
+                            'novel_id_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False}
                         }
                     },
+                    'searchtests_novel__protagonist_id_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'searchtests_novel__characters': {
                         'type': 'nested',
                         'properties': {
-                            'name': {'type': 'string', 'boost': 0.25, 'include_in_all': True}
+                            'name': {'type': 'string', 'boost': 0.25, 'include_in_all': True},
                         }
                     },
 
@@ -636,6 +640,7 @@ def test_get_mapping(self):
                             'date_of_birth_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
                         },
                     },
+                    'authors_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'publication_date_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
                     'number_of_pages_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False},
                     'tags': {
@@ -645,7 +650,7 @@ def test_get_mapping(self):
                             'slug_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
                         },
                     },
-                    'tags_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}
+                    'tags_filter': {'index': 'not_analyzed', 'type': 'integer', 'include_in_all': False}
                 }
             }
         }
@@ -675,8 +680,10 @@ def test_get_document(self):
             # New
             'searchtests_novel__setting': "Middle Earth",
             'searchtests_novel__protagonist': {
-                'name': "Frodo Baggins"
+                'name': "Frodo Baggins",
+                'novel_id_filter': 4
             },
+            'searchtests_novel__protagonist_id_filter': 8,
             'searchtests_novel__characters': [
                 {
                     'name': "Bilbo Baggins"
@@ -704,6 +711,7 @@ def test_get_document(self):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
diff --git a/wagtail/search/tests/test_elasticsearch5_backend.py b/wagtail/search/tests/test_elasticsearch5_backend.py
--- a/wagtail/search/tests/test_elasticsearch5_backend.py
+++ b/wagtail/search/tests/test_elasticsearch5_backend.py
@@ -532,6 +532,7 @@ def test_get_mapping(self):
                             'date_of_birth_filter': {'type': 'date', 'include_in_all': False},
                         },
                     },
+                    'authors_filter': {'type': 'integer', 'include_in_all': False},
                     'publication_date_filter': {'type': 'date', 'include_in_all': False},
                     'number_of_pages_filter': {'type': 'integer', 'include_in_all': False},
                     'tags': {
@@ -541,7 +542,7 @@ def test_get_mapping(self):
                             'slug_filter': {'type': 'keyword', 'include_in_all': False},
                         },
                     },
-                    'tags_filter': {'type': 'keyword', 'include_in_all': False}
+                    'tags_filter': {'type': 'integer', 'include_in_all': False}
                 }
             }
         }
@@ -573,6 +574,7 @@ def test_get_document(self):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
@@ -613,9 +615,11 @@ def test_get_mapping(self):
                     'searchtests_novel__protagonist': {
                         'type': 'nested',
                         'properties': {
-                            'name': {'type': 'text', 'boost': 0.5, 'include_in_all': True}
+                            'name': {'type': 'text', 'boost': 0.5, 'include_in_all': True},
+                            'novel_id_filter': {'type': 'integer', 'include_in_all': False}
                         }
                     },
+                    'searchtests_novel__protagonist_id_filter': {'type': 'integer', 'include_in_all': False},
                     'searchtests_novel__characters': {
                         'type': 'nested',
                         'properties': {
@@ -637,6 +641,7 @@ def test_get_mapping(self):
                             'date_of_birth_filter': {'type': 'date', 'include_in_all': False},
                         },
                     },
+                    'authors_filter': {'type': 'integer', 'include_in_all': False},
                     'publication_date_filter': {'type': 'date', 'include_in_all': False},
                     'number_of_pages_filter': {'type': 'integer', 'include_in_all': False},
                     'tags': {
@@ -646,7 +651,7 @@ def test_get_mapping(self):
                             'slug_filter': {'type': 'keyword', 'include_in_all': False},
                         },
                     },
-                    'tags_filter': {'type': 'keyword', 'include_in_all': False}
+                    'tags_filter': {'type': 'integer', 'include_in_all': False}
                 }
             }
         }
@@ -676,8 +681,10 @@ def test_get_document(self):
             # New
             'searchtests_novel__setting': "Middle Earth",
             'searchtests_novel__protagonist': {
-                'name': "Frodo Baggins"
+                'name': "Frodo Baggins",
+                'novel_id_filter': 4
             },
+            'searchtests_novel__protagonist_id_filter': 8,
             'searchtests_novel__characters': [
                 {
                     'name': "Bilbo Baggins"
@@ -705,6 +712,7 @@ def test_get_document(self):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
diff --git a/wagtail/search/tests/test_elasticsearch6_backend.py b/wagtail/search/tests/test_elasticsearch6_backend.py
--- a/wagtail/search/tests/test_elasticsearch6_backend.py
+++ b/wagtail/search/tests/test_elasticsearch6_backend.py
@@ -533,6 +533,7 @@ def test_get_mapping(self):
                             'date_of_birth_filter': {'type': 'date'},
                         },
                     },
+                    'authors_filter': {'type': 'integer'},
                     'publication_date_filter': {'type': 'date'},
                     'number_of_pages_filter': {'type': 'integer'},
                     'tags': {
@@ -542,7 +543,7 @@ def test_get_mapping(self):
                             'slug_filter': {'type': 'keyword'},
                         },
                     },
-                    'tags_filter': {'type': 'keyword'}
+                    'tags_filter': {'type': 'integer'}
                 }
             }
         }
@@ -574,6 +575,7 @@ def test_get_document(self):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
@@ -614,9 +616,11 @@ def test_get_mapping(self):
                     'searchtests_novel__protagonist': {
                         'type': 'nested',
                         'properties': {
-                            'name': {'type': 'text', 'boost': 0.5, 'copy_to': '_all_text'}
+                            'name': {'type': 'text', 'boost': 0.5, 'copy_to': '_all_text'},
+                            'novel_id_filter': {'type': 'integer'}
                         }
                     },
+                    'searchtests_novel__protagonist_id_filter': {'type': 'integer'},
                     'searchtests_novel__characters': {
                         'type': 'nested',
                         'properties': {
@@ -639,6 +643,7 @@ def test_get_mapping(self):
                             'date_of_birth_filter': {'type': 'date'},
                         },
                     },
+                    'authors_filter': {'type': 'integer'},
                     'publication_date_filter': {'type': 'date'},
                     'number_of_pages_filter': {'type': 'integer'},
                     'tags': {
@@ -648,7 +653,7 @@ def test_get_mapping(self):
                             'slug_filter': {'type': 'keyword'},
                         },
                     },
-                    'tags_filter': {'type': 'keyword'}
+                    'tags_filter': {'type': 'integer'}
                 }
             }
         }
@@ -678,8 +683,10 @@ def test_get_document(self):
             # New
             'searchtests_novel__setting': "Middle Earth",
             'searchtests_novel__protagonist': {
-                'name': "Frodo Baggins"
+                'name': "Frodo Baggins",
+                'novel_id_filter': 4
             },
+            'searchtests_novel__protagonist_id_filter': 8,
             'searchtests_novel__characters': [
                 {
                     'name': "Bilbo Baggins"
@@ -707,6 +714,7 @@ def test_get_document(self):
                     'date_of_birth_filter': datetime.date(1892, 1, 3)
                 }
             ],
+            'authors_filter': [2],
             'publication_date_filter': datetime.date(1954, 7, 29),
             'number_of_pages_filter': 423,
             'tags': [],
diff --git a/wagtail/tests/customuser/fields.py b/wagtail/tests/customuser/fields.py
--- a/wagtail/tests/customuser/fields.py
+++ b/wagtail/tests/customuser/fields.py
@@ -68,3 +68,8 @@ def get_db_prep_value(self, value, connection, prepared=False):
         if not value:
             return
         return ConvertedValue(value).db_value
+
+    def get_searchable_content(self, value):
+        if not value:
+            return
+        return ConvertedValue(value).db_value
diff --git a/wagtail/tests/search/models.py b/wagtail/tests/search/models.py
--- a/wagtail/tests/search/models.py
+++ b/wagtail/tests/search/models.py
@@ -28,6 +28,7 @@ class Book(index.Indexed, models.Model):
         index.SearchField('title', partial_match=True, boost=2.0),
         index.AutocompleteField('title'),
         index.FilterField('title'),
+        index.FilterField('authors'),
         index.RelatedFields('authors', Author.search_fields),
         index.FilterField('publication_date'),
         index.FilterField('number_of_pages'),
@@ -73,10 +74,6 @@ class Character(models.Model):
     name = models.CharField(max_length=255)
     novel = models.ForeignKey('Novel', related_name='characters', on_delete=models.CASCADE)
 
-    search_fields = [
-        index.SearchField('name'),
-    ]
-
     def __str__(self):
         return self.name
 
@@ -92,7 +89,9 @@ class Novel(Book):
         ]),
         index.RelatedFields('protagonist', [
             index.SearchField('name', boost=0.5),
+            index.FilterField('novel'),
         ]),
+        index.FilterField('protagonist'),
     ]