Repository: pypi/warehouse

Issue:
Optimize reindexing
It's possible to tweak the index settings before and after a full reindex speed up reindexing.

Things you can do before reindexing:
- Set `num_replicas` to `0`. During reindexing this will stop the network traffic of Elasticsearch replicating your data across the nodes. This can happen in bulk when you set `num_replicas` back to a value greater than `0` after reindexing is finished.
- Set `refresh_interval` to `-1`. I.e. disable index refreshes completely during bulk indexing.  (See: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html#bulk)

Things you can do after reindexing:
- Perform a `force_merge` on the index. The defaults should be fine. See: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html
- Set the `num_replicas` back to its default value.
- Set the `refresh_interval` back to its default value.

[WIP] Add some optimizations to reindex
WORK IN PROGRESS

I consider the code to be relatively complete (besides the tests), but local testing is revealing a lot of timeout errors connecting to the Elasticsearch cluster. I don't know if a change here causes this or something else.



Discussion:
Attempted fix in https://github.com/pypa/warehouse/pull/1226

Just gonna say that the general idea and the changes look good to me, once you work out the issues :]


Resolving patch:
diff --git a/warehouse/cli/search/reindex.py b/warehouse/cli/search/reindex.py
--- a/warehouse/cli/search/reindex.py
+++ b/warehouse/cli/search/reindex.py
@@ -57,6 +57,8 @@ def reindex(config, **kwargs):
     """
     client = config.registry["elasticsearch.client"]
     db = Session(bind=config.registry["sqlalchemy.engine"])
+    number_of_replicas = config.registry.get("elasticsearch.replicas", 0)
+    refresh_interval = config.registry.get("elasticsearch.interval", "1s")
 
     # We use a randomly named index so that we can do a zero downtime reindex.
     # Essentially we'll use a randomly named index which we will use until all
@@ -68,12 +70,16 @@ def reindex(config, **kwargs):
     random_token = binascii.hexlify(os.urandom(5)).decode("ascii")
     new_index_name = "{}-{}".format(index_base, random_token)
     doc_types = config.registry.get("search.doc_types", set())
+
+    # Create the new index with zero replicas and index refreshes disabled
+    # while we are bulk indexing.
     new_index = get_index(
         new_index_name,
         doc_types,
         using=client,
         shards=config.registry.get("elasticsearch.shards", 1),
-        replicas=config.registry.get("elasticsearch.replicas", 0),
+        replicas=0,
+        interval="-1",
     )
     new_index.create()
 
@@ -91,8 +97,20 @@ def reindex(config, **kwargs):
         db.rollback()
         db.close()
 
-    # Now that we've finished indexing all of our data, we'll point the alias
-    # at our new randomly named index and delete the old index.
+    # Now that we've finished indexing all of our data we can optimize it and
+    # update the replicas and refresh intervals.
+    client.indices.forcemerge(index=new_index_name)
+    client.indices.put_settings(
+        index=new_index_name,
+        body={
+            "index": {
+                "number_of_replicas": number_of_replicas,
+                "refresh_interval": refresh_interval,
+            }
+        }
+    )
+
+    # Point the alias at our new randomly named index and delete the old index.
     if client.indices.exists_alias(name=index_base):
         to_delete = set()
         actions = []
diff --git a/warehouse/search.py b/warehouse/search.py
--- a/warehouse/search.py
+++ b/warehouse/search.py
@@ -29,11 +29,15 @@ def callback(scanner, _name, item):
     return cls
 
 
-def get_index(name, doc_types, *, using, shards=1, replicas=0):
+def get_index(name, doc_types, *, using, shards=1, replicas=0, interval="1s"):
     index = Index(name, using=using)
     for doc_type in doc_types:
         index.doc_type(doc_type)
-    index.settings(number_of_shards=shards, number_of_replicas=replicas)
+    index.settings(
+        number_of_shards=shards,
+        number_of_replicas=replicas,
+        refresh_interval=interval,
+    )
     return index
 
 
@@ -58,6 +62,8 @@ def includeme(config):
         [urllib.parse.urlunparse(p[:2] + ("",) * 4)],
         verify_certs=True,
         ca_certs=certifi.where(),
+        timeout=30,
+        retry_on_timeout=True,
     )
     config.registry["elasticsearch.index"] = p.path.strip("/")
     config.registry["elasticsearch.shards"] = int(qs.get("shards", ["1"])[0])


Test patch:
diff --git a/tests/unit/cli/search/test_reindex.py b/tests/unit/cli/search/test_reindex.py
--- a/tests/unit/cli/search/test_reindex.py
+++ b/tests/unit/cli/search/test_reindex.py
@@ -54,6 +54,9 @@ def __init__(self):
         self.indices = {}
         self.aliases = {}
 
+        self.put_settings = pretend.call_recorder(lambda *a, **kw: None)
+        self.forcemerge = pretend.call_recorder(lambda *a, **kw: None)
+
     def create(self, index, body):
         self.indices[index] = body
 
@@ -146,6 +149,8 @@ def parallel_bulk(client, iterable):
         assert sess_obj.rollback.calls == [pretend.call()]
         assert sess_obj.close.calls == [pretend.call()]
         assert es_client.indices.indices == {}
+        assert es_client.indices.put_settings.calls == []
+        assert es_client.indices.forcemerge.calls == []
 
     def test_successfully_indexes_and_adds_new(self, monkeypatch, cli):
         sess_obj = pretend.stub(
@@ -198,6 +203,20 @@ def project_docs(db):
         assert es_client.indices.aliases == {
             "warehouse": ["warehouse-cbcbcbcbcb"],
         }
+        assert es_client.indices.put_settings.calls == [
+            pretend.call(
+                index='warehouse-cbcbcbcbcb',
+                body={
+                    'index': {
+                        'number_of_replicas': 0,
+                        'refresh_interval': '1s',
+                    },
+                },
+            )
+        ]
+        assert es_client.indices.forcemerge.calls == [
+            pretend.call(index='warehouse-cbcbcbcbcb')
+        ]
 
     def test_successfully_indexes_and_replaces(self, monkeypatch, cli):
         sess_obj = pretend.stub(
@@ -252,3 +271,17 @@ def project_docs(db):
         assert es_client.indices.aliases == {
             "warehouse": ["warehouse-cbcbcbcbcb"],
         }
+        assert es_client.indices.put_settings.calls == [
+            pretend.call(
+                index='warehouse-cbcbcbcbcb',
+                body={
+                    'index': {
+                        'number_of_replicas': 0,
+                        'refresh_interval': '1s',
+                    },
+                },
+            )
+        ]
+        assert es_client.indices.forcemerge.calls == [
+            pretend.call(index='warehouse-cbcbcbcbcb')
+        ]
diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py
--- a/tests/unit/test_search.py
+++ b/tests/unit/test_search.py
@@ -42,6 +42,10 @@ def test_es(monkeypatch):
     assert index_cls.calls == [pretend.call("warehouse", using=client)]
     assert index_obj.doc_type.calls == [pretend.call(d) for d in doc_types]
     assert index_obj.settings.calls == [
-        pretend.call(number_of_shards=1, number_of_replicas=0),
+        pretend.call(
+            number_of_shards=1,
+            number_of_replicas=0,
+            refresh_interval="1s",
+        )
     ]
     assert index_obj.search.calls == [pretend.call()]