Repository: pypi/warehouse

Issue:
Tune Elasticsearch client for reindex separately from main search client
#3892 was necessary to alleviate the hard outage experienced due to ES cluster being down.

We were waiting 30s per request for Elasticsearch then retrying. This was added in #1471 to handle reindex issues.


Discussion:

Resolving patch:
diff --git a/warehouse/search/tasks.py b/warehouse/search/tasks.py
--- a/warehouse/search/tasks.py
+++ b/warehouse/search/tasks.py
@@ -11,11 +11,15 @@
 # limitations under the License.
 
 import binascii
+import urllib
 import os
 
 from elasticsearch.helpers import parallel_bulk
+from elasticsearch_dsl import serializer
 from sqlalchemy import and_, func
 from sqlalchemy.orm import aliased
+import certifi
+import elasticsearch
 
 from warehouse.packaging.models import (
     Classifier, Project, Release, release_classifiers)
@@ -98,7 +102,15 @@ def reindex(request):
     """
     Recreate the Search Index.
     """
-    client = request.registry["elasticsearch.client"]
+    p = urllib.parse.urlparse(request.registry.settings["elasticsearch.url"])
+    client = elasticsearch.Elasticsearch(
+        [urllib.parse.urlunparse(p[:2] + ("",) * 4)],
+        verify_certs=True,
+        ca_certs=certifi.where(),
+        timeout=30,
+        retry_on_timeout=True,
+        serializer=serializer.serializer,
+    )
     number_of_replicas = request.registry.get("elasticsearch.replicas", 0)
     refresh_interval = request.registry.get("elasticsearch.interval", "1s")
 


Test patch:
diff --git a/tests/unit/search/test_tasks.py b/tests/unit/search/test_tasks.py
--- a/tests/unit/search/test_tasks.py
+++ b/tests/unit/search/test_tasks.py
@@ -114,10 +114,17 @@ def project_docs(db):
 
         db_request.registry.update(
             {
-                "elasticsearch.client": es_client,
                 "elasticsearch.index": "warehouse",
             },
         )
+        db_request.registry.settings = {
+            "elasticsearch.url": "http://some.url",
+        }
+        monkeypatch.setattr(
+            warehouse.search.tasks.elasticsearch,
+            "Elasticsearch",
+            lambda *a, **kw: es_client
+        )
 
         class TestException(Exception):
             pass
@@ -158,11 +165,18 @@ def project_docs(db):
 
         db_request.registry.update(
             {
-                "elasticsearch.client": es_client,
                 "elasticsearch.index": "warehouse",
                 "elasticsearch.shards": 42,
             }
         )
+        db_request.registry.settings = {
+            "elasticsearch.url": "http://some.url",
+        }
+        monkeypatch.setattr(
+            warehouse.search.tasks.elasticsearch,
+            "Elasticsearch",
+            lambda *a, **kw: es_client
+        )
 
         parallel_bulk = pretend.call_recorder(lambda client, iterable: [None])
         monkeypatch.setattr(
@@ -224,12 +238,19 @@ def project_docs(db):
 
         db_request.registry.update(
             {
-                "elasticsearch.client": es_client,
                 "elasticsearch.index": "warehouse",
                 "elasticsearch.shards": 42,
                 "sqlalchemy.engine": db_engine,
             },
         )
+        db_request.registry.settings = {
+            "elasticsearch.url": "http://some.url",
+        }
+        monkeypatch.setattr(
+            warehouse.search.tasks.elasticsearch,
+            "Elasticsearch",
+            lambda *a, **kw: es_client
+        )
 
         parallel_bulk = pretend.call_recorder(lambda client, iterable: [None])
         monkeypatch.setattr(