Repository: wagtail/wagtail

Issue:
Add OPTIONS argument to WAGTAILSEARCH_BACKENDS
Currently, any options passed to WAGTAILSEARCH_BACKENDS that Wagtail doesn't recognised are passed straight through to Elasticsearch. I think it should be made more explicit which settings are recognised by Wagtail and which are recognised by the underlying search library.

See the bit that mentions `max_retries` in http://docs.wagtail.io/en/v1.5.2/topics/search/backends.html#elasticsearch-backend

I think we should nest the options for Elasticsearch in a sub dict called `OPTIONS`:

For example:

```
WAGTAILSEARCH_BACKENDS = {
    'default': {
        'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch',
        'URLS': ['http://localhost:9200'],
        'INDEX': 'wagtail',
        'OPTIONS': {
            'max_retries': 1,
        }
    }
}
```

Obviously, care would need to be taken to ensure backwards compatibility.



Discussion:

Resolving patch:
diff --git a/wagtail/utils/utils.py b/wagtail/utils/utils.py
new file mode 100644
--- /dev/null
+++ b/wagtail/utils/utils.py
@@ -0,0 +1,23 @@
+from __future__ import absolute_import, unicode_literals
+
+import collections
+import sys
+
+
+def deep_update(source, overrides):
+    """Update a nested dictionary or similar mapping.
+
+    Modify ``source`` in place.
+    """
+    if sys.version_info >= (3, 0):
+        items = overrides.items()
+    else:
+        items = overrides.iteritems()
+
+    for key, value in items:
+        if isinstance(value, collections.Mapping) and value:
+            returned = deep_update(source.get(key, {}), value)
+            source[key] = returned
+        else:
+            source[key] = overrides[key]
+    return source
diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import json
+import warnings
 
 from django.db import models
 from django.utils.crypto import get_random_string
@@ -8,6 +9,8 @@
 from elasticsearch import Elasticsearch, NotFoundError
 from elasticsearch.helpers import bulk
 
+from wagtail.utils.deprecation import RemovedInWagtail110Warning
+from wagtail.utils.utils import deep_update
 from wagtail.wagtailsearch.backends.base import (
     BaseSearchBackend, BaseSearchQuery, BaseSearchResults)
 from wagtail.wagtailsearch.index import (
@@ -760,12 +763,24 @@ def __init__(self, params):
                     'http_auth': http_auth,
                 })
 
+        self.settings = self.settings.copy()  # Make the class settings attribute as instance settings attribute
+        self.settings = deep_update(self.settings, params.pop("INDEX_SETTINGS", {}))
+
         # Get Elasticsearch interface
         # Any remaining params are passed into the Elasticsearch constructor
+        options = params.pop('OPTIONS', {})
+        if not options and params:
+            options = params
+
+            warnings.warn(
+                "Any extra parameter for the ElasticSearch constructor must be passed through the OPTIONS dictionary.",
+                category=RemovedInWagtail110Warning, stacklevel=2
+            )
+
         self.es = Elasticsearch(
             hosts=self.hosts,
             timeout=self.timeout,
-            **params)
+            **options)
 
     def get_index_for_model(self, model):
         return self.index_class(self, self.index_name)


Test patch:
diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
--- a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -981,6 +981,33 @@ def test_urls(self):
         self.assertEqual(backend.hosts[3]['use_ssl'], True)
         self.assertEqual(backend.hosts[3]['url_prefix'], '/hello')
 
+    def test_default_index_settings_override(self):
+        backend = ElasticsearchSearchBackend(params={
+            'INDEX_SETTINGS': {
+                "settings": {  # Already existing key
+                    "number_of_shards": 2,  # New key
+                    "analysis": {  # Already existing key
+                        "analyzer": {  # Already existing key
+                            "edgengram_analyzer": {  # Already existing key
+                                "tokenizer": "standard"  # Key redefinition
+                            }
+                        }
+                    }
+                }
+            }
+        })
+
+        # Check structure
+        self.assertIn("number_of_shards", backend.settings["settings"].keys())
+        self.assertIn("analysis", backend.settings["settings"].keys())
+        self.assertIn("analyzer", backend.settings["settings"]["analysis"].keys())
+        self.assertIn("edgengram_analyzer", backend.settings["settings"]["analysis"]["analyzer"].keys())
+        self.assertIn("tokenizer", backend.settings["settings"]["analysis"]["analyzer"]["edgengram_analyzer"].keys())
+        # Check values
+        self.assertEqual(backend.settings["settings"]["number_of_shards"], 2)
+        self.assertEqual(backend.settings["settings"]["analysis"]["analyzer"]["edgengram_analyzer"]["tokenizer"], "standard")
+        self.assertEqual(backend.settings["settings"]["analysis"]["analyzer"]["edgengram_analyzer"]["type"], "custom")  # Check if a default setting still exists
+
 
 @unittest.skipUnless(os.environ.get('ELASTICSEARCH_URL', False), "ELASTICSEARCH_URL not set")
 @unittest.skipUnless(os.environ.get('ELASTICSEARCH_VERSION', '1') == '1', "ELASTICSEARCH_VERSION not set to 1")