Repository: wagtail/wagtail

Issue:
Give database and elasticsearch backends the same API
Currently, when you search with the Elasiticsearch backend, you get an `ElasticSearchResults` object which is quite different from the `QuerySet` object the database backend gives you.

The issue with this is if somebody is using the DB backend in development and the ES backend in production, they have to be very careful to not use any methods in `QuerySet` that are not also in `ElasticSearchResults`.

Another difference is the database backend won't check if a field being filtered on has been defined in `search_fields` as a `FilterField` whereas `ElasticSearchResults` does.

We should create a `BaseSearchResults` class which `DBSearchResults` and `ElasticSearchResults` can inherit from. Most of the functionality of `ElasticSearchResults` can be moved there.



Discussion:

Resolving patch:
diff --git a/wagtail/wagtailsearch/backends/base.py b/wagtail/wagtailsearch/backends/base.py
--- a/wagtail/wagtailsearch/backends/base.py
+++ b/wagtail/wagtailsearch/backends/base.py
@@ -1,34 +1,218 @@
+from six import text_type
+
+from django.db import models
 from django.db.models.query import QuerySet
 from django.core.exceptions import ImproperlyConfigured
 
+# Django 1.7 lookups
+try:
+    from django.db.models.lookups import Lookup
+except ImportError:
+    Lookup = None
+
+from django.db.models.sql.where import SubqueryConstraint, WhereNode
+
 from wagtail.wagtailsearch.index import class_is_indexed
 from wagtail.wagtailsearch.utils import normalise_query_string
 
 
+class FilterError(Exception):
+    pass
+
+
+class FieldError(Exception):
+    pass
+
+
+class BaseSearchQuery(object):
+    def __init__(self, queryset, query_string, fields=None):
+        self.queryset = queryset
+        self.query_string = query_string
+        self.fields = fields
+
+    def _get_searchable_field(self, field_attname):
+        # Get field
+        field = dict(
+            (field.get_attname(self.queryset.model), field)
+            for field in self.queryset.model.get_searchable_search_fields()
+        ).get(field_attname, None)
+
+        return field
+
+    def _get_filterable_field(self, field_attname):
+        # Get field
+        field = dict(
+            (field.get_attname(self.queryset.model), field)
+            for field in self.queryset.model.get_filterable_search_fields()
+        ).get(field_attname, None)
+
+        return field
+
+    def _process_lookup(self, field, lookup, value):
+        raise NotImplementedError
+
+    def _connect_filters(self, filters, connector, negated):
+        raise NotImplementedError
+
+    def _process_filter(self, field_attname, lookup, value):
+        # Get the field
+        field = self._get_filterable_field(field_attname)
+
+        if field is None:
+            raise FieldError('Cannot filter search results with field "' + field_attname + '". Please add index.FilterField(\'' + field_attname + '\') to ' + self.queryset.model.__name__ + '.search_fields.')
+
+        # Process the lookup
+        result = self._process_lookup(field, lookup, value)
+
+        if result is None:
+            raise FilterError('Could not apply filter on search results: "' + field_attname + '__' + lookup + ' = ' + text_type(value) + '". Lookup "' + lookup + '"" not recognosed.')
+
+        return result
+
+    def _get_filters_from_where_node(self, where_node):
+        # Check if this is a leaf node
+        if isinstance(where_node, tuple): # Django 1.6 and below
+            field_attname = where_node[0].col
+            lookup = where_node[1]
+            value = where_node[3]
+
+            # Process the filter
+            return self._process_filter(field_attname, lookup, value)
+
+        elif Lookup is not None and isinstance(where_node, Lookup): # Django 1.7 and above
+            field_attname = where_node.lhs.target.attname
+            lookup = where_node.lookup_name
+            value = where_node.rhs
+
+            # Process the filter
+            return self._process_filter(field_attname, lookup, value)
+
+        elif isinstance(where_node, SubqueryConstraint):
+            raise FilterError('Could not apply filter on search results: Subqueries are not allowed.')
+
+        elif isinstance(where_node, WhereNode):
+            # Get child filters
+            connector = where_node.connector
+            child_filters = [self._get_filters_from_where_node(child) for child in where_node.children]
+            child_filters = [child_filter for child_filter in child_filters if child_filter]
+
+            return self._connect_filters(child_filters, connector, where_node.negated)
+
+        else:
+            raise FilterError('Could not apply filter on search results: Unknown where node: ' + str(type(where_node)))
+
+    def _get_filters_from_queryset(self):
+        return self._get_filters_from_where_node(self.queryset.query.where)
+
+
+class BaseSearchResults(object):
+    def __init__(self, backend, query, prefetch_related=None):
+        self.backend = backend
+        self.query = query
+        self.prefetch_related = prefetch_related
+        self.start = 0
+        self.stop = None
+        self._results_cache = None
+        self._count_cache = None
+
+    def _set_limits(self, start=None, stop=None):
+        if stop is not None:
+            if self.stop is not None:
+                self.stop = min(self.stop, self.start + stop)
+            else:
+                self.stop = self.start + stop
+
+        if start is not None:
+            if self.stop is not None:
+                self.start = min(self.stop, self.start + start)
+            else:
+                self.start = self.start + start
+
+    def _clone(self):
+        klass = self.__class__
+        new = klass(self.backend, self.query, prefetch_related=self.prefetch_related)
+        new.start = self.start
+        new.stop = self.stop
+        return new
+
+    def _do_search(self):
+        raise NotImplementedError
+
+    def _do_count(self):
+        raise NotImplementedError
+
+    def results(self):
+        if self._results_cache is None:
+            self._results_cache = self._do_search()
+        return self._results_cache
+
+    def count(self):
+        if self._count_cache is None:
+            if self._results_cache is not None:
+                self._count_cache = len(self._results_cache)
+            else:
+                self._count_cache = self._do_count()
+        return self._count_cache
+
+    def __getitem__(self, key):
+        new = self._clone()
+
+        if isinstance(key, slice):
+            # Set limits
+            start = int(key.start) if key.start else None
+            stop = int(key.stop) if key.stop else None
+            new._set_limits(start, stop)
+
+            # Copy results cache
+            if self._results_cache is not None:
+                new._results_cache = self._results_cache[key]
+
+            return new
+        else:
+            if self._results_cache is not None:
+                return self._results_cache[key]
+
+            new.start = key
+            new.stop = key + 1
+            return list(new)[0]
+
+    def __iter__(self):
+        return iter(self.results())
+
+    def __len__(self):
+        return len(self.results())
+
+    def __repr__(self):
+        data = list(self[:21])
+        if len(data) > 20:
+            data[-1] = "...(remaining elements truncated)..."
+        return repr(data)
+
+
 class BaseSearch(object):
     def __init__(self, params):
         pass
 
     def reset_index(self):
-        return NotImplemented
+        raise NotImplementedError
 
     def add_type(self, model):
-        return NotImplemented
+        raise NotImplementedError
 
     def refresh_index(self):
-        return NotImplemented
+        raise NotImplementedError
 
     def add(self, obj):
-        return NotImplemented
+        raise NotImplementedError
 
     def add_bulk(self, model, obj_list):
-        return NotImplemented
+        raise NotImplementedError
 
     def delete(self, obj):
-        return NotImplemented
+        raise NotImplementedError
 
     def _search(self, queryset, query_string, fields=None):
-        return NotImplemented
+        raise NotImplementedError
 
     def search(self, query_string, model_or_queryset, fields=None, filters=None, prefetch_related=None):
         # Find model/queryset
diff --git a/wagtail/wagtailsearch/backends/db.py b/wagtail/wagtailsearch/backends/db.py
--- a/wagtail/wagtailsearch/backends/db.py
+++ b/wagtail/wagtailsearch/backends/db.py
@@ -1,40 +1,41 @@
 from django.db import models
 
-from wagtail.wagtailsearch.backends.base import BaseSearch
+from wagtail.wagtailsearch.backends.base import BaseSearch, BaseSearchQuery, BaseSearchResults
 
 
-class DBSearch(BaseSearch):
-    def __init__(self, params):
-        super(DBSearch, self).__init__(params)
+class DBSearchQuery(BaseSearchQuery):
+    def _process_lookup(self, field, lookup, value):
+        return models.Q(**{field.get_attname(self.queryset.model) + '__' + lookup: value})
 
-    def reset_index(self):
-        pass # Not needed
+    def _connect_filters(self, filters, connector, negated):
+        if connector == 'AND':
+            q = models.Q(*filters)
+        elif connector == 'OR':
+            q = models.Q(filters[0])
+            for fil in filters[1:]:
+                q |= fil
+        else:
+            return
 
-    def add_type(self, model):
-        pass # Not needed
+        if negated:
+            q = ~q
 
-    def refresh_index(self):
-        pass # Not needed
+        return q
 
-    def add(self, obj):
-        pass # Not needed
+    def get_q(self):
+        # Get filters as a q object
+        q = self._get_filters_from_queryset()
 
-    def add_bulk(self, model, obj_list):
-        pass # Not needed
-
-    def delete(self, obj):
-        pass # Not needed
+        model = self.queryset.model
 
-    def _search(self, queryset, query_string, fields=None):
-        if query_string is not None:
+        if self.query_string is not None:
             # Get fields
-            if fields is None:
-                fields = [field.field_name for field in queryset.model.get_searchable_search_fields()]
+            fields = self.fields or [field.field_name for field in model.get_searchable_search_fields()]
 
             # Get terms
-            terms = query_string.split()
+            terms = self.query_string.split()
             if not terms:
-                return queryset.model.objects.none()
+                return model.objects.none()
 
             # Filter by terms
             for term in terms:
@@ -42,16 +43,53 @@ def _search(self, queryset, query_string, fields=None):
                 for field_name in fields:
                     # Check if the field exists (this will filter out indexed callables)
                     try:
-                        queryset.model._meta.get_field_by_name(field_name)
+                        model._meta.get_field_by_name(field_name)
                     except:
                         continue
 
                     # Filter on this field
                     term_query |= models.Q(**{'%s__icontains' % field_name: term})
 
-                queryset = queryset.filter(term_query)
+                q &= term_query
+
+        return q
+
+
+class DBSearchResults(BaseSearchResults):
+    def get_queryset(self):
+        model = self.query.queryset.model
+        q = self.query.get_q()
+
+        return model.objects.filter(q).distinct()[self.start:self.stop]
+
+    def _do_search(self):
+        return self.get_queryset()
+
+    def _do_count(self):
+        return self.get_queryset().count()
 
-            # Distinct
-            queryset = queryset.distinct()
 
-        return queryset
+class DBSearch(BaseSearch):
+    def __init__(self, params):
+        super(DBSearch, self).__init__(params)
+
+    def reset_index(self):
+        pass # Not needed
+
+    def add_type(self, model):
+        pass # Not needed
+
+    def refresh_index(self):
+        pass # Not needed
+
+    def add(self, obj):
+        pass # Not needed
+
+    def add_bulk(self, model, obj_list):
+        return # Not needed
+
+    def delete(self, obj):
+        pass # Not needed
+
+    def _search(self, queryset, query_string, fields=None):
+        return DBSearchResults(self, DBSearchQuery(queryset, query_string, fields=fields))
diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -3,21 +3,13 @@
 import json
 
 from six.moves.urllib.parse import urlparse
-from six import text_type
 
 from django.db import models
-from django.db.models.sql.where import SubqueryConstraint, WhereNode
-
-# Django 1.7 lookups
-try:
-    from django.db.models.lookups import Lookup
-except ImportError:
-    Lookup = None
 
 from elasticsearch import Elasticsearch, NotFoundError, RequestError
 from elasticsearch.helpers import bulk
 
-from wagtail.wagtailsearch.backends.base import BaseSearch
+from wagtail.wagtailsearch.backends.base import BaseSearch, BaseSearchQuery, BaseSearchResults
 from wagtail.wagtailsearch.index import Indexed, SearchField, FilterField, class_is_indexed
 
 
@@ -118,31 +110,8 @@ def __repr__(self):
         return '<ElasticSearchMapping: %s>' % (self.model.__name__, )
 
 
-class FilterError(Exception):
-    pass
-
-
-class FieldError(Exception):
-    pass
-
-
-class ElasticSearchQuery(object):
-    def __init__(self, queryset, query_string, fields=None):
-        self.queryset = queryset
-        self.query_string = query_string
-        self.fields = fields
-
-    def _process_lookup(self, field_attname, lookup, value):
-        # Get field
-        field = dict(
-            (field.get_attname(self.queryset.model), field)
-            for field in self.queryset.model.get_filterable_search_fields()
-        ).get(field_attname, None)
-
-        # Give error if the field doesn't exist
-        if field is None:
-            raise FieldError('Cannot filter ElasticSearch results with field "' + field_attname + '". Please add FilterField(\'' + field_attname + '\') to ' + self.queryset.model.__name__ + '.search_fields.')
-
+class ElasticSearchQuery(BaseSearchQuery):
+    def _process_lookup(self, field, lookup, value):
         # Get the name of the field in the index
         field_index_name = field.get_index_name(self.queryset.model)
 
@@ -211,72 +180,23 @@ def _process_lookup(self, field_attname, lookup, value):
                 }
             }
 
-        raise FilterError('Could not apply filter on ElasticSearch results: "' + field_attname + '__' + lookup + ' = ' + text_type(value) + '". Lookup "' + lookup + '"" not recognised.')
-
-    def _get_filters_from_where(self, where_node):
-        # Check if this is a leaf node
-        if isinstance(where_node, tuple): # Django 1.6 and below
-            field_attname = where_node[0].col
-            lookup = where_node[1]
-            value = where_node[3]
-
-            # Process the filter
-            return self._process_lookup(field_attname, lookup, value)
-
-        elif Lookup is not None and isinstance(where_node, Lookup): # Django 1.7 and above
-            field_attname = where_node.lhs.target.attname
-            lookup = where_node.lookup_name
-            value = where_node.rhs
-
-            # Process the filter
-            return self._process_lookup(field_attname, lookup, value)
-
-        elif isinstance(where_node, SubqueryConstraint):
-            raise FilterError('Could not apply filter on ElasticSearch results: Subqueries are not allowed.')
-
-        elif isinstance(where_node, WhereNode):
-            # Get child filters
-            connector = where_node.connector
-            child_filters = [self._get_filters_from_where(child) for child in where_node.children]
-            child_filters = [child_filter for child_filter in child_filters if child_filter]
-
-            # Connect them
-            if child_filters:
-                if len(child_filters) == 1:
-                    filter_out = child_filters[0]
-                else:
-                    filter_out = {
-                        connector.lower(): [
-                            fil for fil in child_filters if fil is not None
-                        ]
-                    }
-
-                if where_node.negated:
-                    filter_out = {
-                        'not': filter_out
-                    }
-
-                return filter_out
-        else:
-            raise FilterError('Could not apply filter on ElasticSearch results: Unknown where node: ' + str(type(where_node)))
-
-    def _get_filters(self):
-        # Filters
-        filters = []
-
-        # Filter by content type
-        filters.append({
-            'prefix': {
-                'content_type': self.queryset.model.indexed_get_content_type()
-            }
-        })
+    def _connect_filters(self, filters, connector, negated):
+        if filters:
+            if len(filters) == 1:
+                filter_out = filters[0]
+            else:
+                filter_out = {
+                    connector.lower(): [
+                        fil for fil in filters if fil is not None
+                    ]
+                }
 
-        # Apply filters from queryset
-        queryset_filters = self._get_filters_from_where(self.queryset.query.where)
-        if queryset_filters:
-            filters.append(queryset_filters)
+            if negated:
+                filter_out = {
+                    'not': filter_out
+                }
 
-        return filters
+            return filter_out
 
     def to_es(self):
         # Query
@@ -302,7 +222,20 @@ def to_es(self):
             }
 
         # Filters
-        filters = self._get_filters()
+        filters = []
+
+        # Filter by content type
+        filters.append({
+            'prefix': {
+                'content_type': self.queryset.model.indexed_get_content_type()
+            }
+        })
+
+        # Apply filters from queryset
+        queryset_filters = self._get_filters_from_queryset()
+        if queryset_filters:
+            filters.append(queryset_filters)
+
         if len(filters) == 1:
             query = {
                 'filtered': {
@@ -326,36 +259,7 @@ def __repr__(self):
         return json.dumps(self.to_es())
 
 
-class ElasticSearchResults(object):
-    def __init__(self, backend, query, prefetch_related=None):
-        self.backend = backend
-        self.query = query
-        self.prefetch_related = prefetch_related
-        self.start = 0
-        self.stop = None
-        self._results_cache = None
-        self._count_cache = None
-
-    def _set_limits(self, start=None, stop=None):
-        if stop is not None:
-            if self.stop is not None:
-                self.stop = min(self.stop, self.start + stop)
-            else:
-                self.stop = self.start + stop
-
-        if start is not None:
-            if self.stop is not None:
-                self.start = min(self.stop, self.start + start)
-            else:
-                self.start = self.start + start
-
-    def _clone(self):
-        klass = self.__class__
-        new = klass(self.backend, self.query, prefetch_related=self.prefetch_related)
-        new.start = self.start
-        new.stop = self.stop
-        return new
-
+class ElasticSearchResults(BaseSearchResults):
     def _do_search(self):
         # Params for elasticsearch query
         params = dict(
@@ -417,53 +321,6 @@ def _do_count(self):
 
         return max(hit_count, 0)
 
-    def results(self):
-        if self._results_cache is None:
-            self._results_cache = self._do_search()
-        return self._results_cache
-
-    def count(self):
-        if self._count_cache is None:
-            if self._results_cache is not None:
-                self._count_cache = len(self._results_cache)
-            else:
-                self._count_cache = self._do_count()
-        return self._count_cache
-
-    def __getitem__(self, key):
-        new = self._clone()
-
-        if isinstance(key, slice):
-            # Set limits
-            start = int(key.start) if key.start else None
-            stop = int(key.stop) if key.stop else None
-            new._set_limits(start, stop)
-
-            # Copy results cache
-            if self._results_cache is not None:
-                new._results_cache = self._results_cache[key]
-
-            return new
-        else:
-            if self._results_cache is not None:
-                return self._results_cache[key]
-
-            new.start = key
-            new.stop = key + 1
-            return list(new)[0]
-
-    def __iter__(self):
-        return iter(self.results())
-
-    def __len__(self):
-        return len(self.results())
-
-    def __repr__(self):
-        data = list(self[:21])
-        if len(data) > 20:
-            data[-1] = "...(remaining elements truncated)..."
-        return repr(data)
-
 
 class ElasticSearch(BaseSearch):
     def __init__(self, params):


Test patch:
diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
--- a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -26,13 +26,13 @@ def test_search_with_spaces_only(self):
 
     def test_filter_on_non_filterindex_field(self):
         # id is not listed in the search_fields for SearchTest; this should raise a FieldError
-        from wagtail.wagtailsearch.backends.elasticsearch import FieldError
+        from wagtail.wagtailsearch.backends.base import FieldError
 
         with self.assertRaises(FieldError):
             results = list(self.backend.search("Hello", models.SearchTest, filters=dict(id=42)))
 
     def test_filter_with_unsupported_lookup_type(self):
-        from wagtail.wagtailsearch.backends.elasticsearch import FilterError
+        from wagtail.wagtailsearch.backends.base import FilterError
 
         with self.assertRaises(FilterError):
             results = list(self.backend.search("Hello", models.SearchTest, filters=dict(title__iregex='h(ea)llo')))