Repository: readthedocs/readthedocs.org

Issue:
Fix or abandon reindex_elasticsearch
Currently the managment command `reindex_elasticsearch` [located here](https://github.com/rtfd/readthedocs.org/blob/master/readthedocs/core/management/commands/reindex_elasticsearch.py) is broken in at least two ways:
1. We give a [filter argument to the version's manager public method](https://github.com/rtfd/readthedocs.org/blob/adf227ae3810b0820d0ff548651dcd223f4f2388/readthedocs/core/management/commands/reindex_elasticsearch.py#L31). But that does not do any filtering. Therefore only selecting one project that should be updated will result in all projects to be reindexed.
2. We do not distinguish between mkdocs/sphinx projects. The used `parse_json.process_all_json_files` will only work correctly for Sphinx based projects.

We should either remove the command completely as long as it no longer used/needed. Or fix the issues outlined above.



Discussion:
This is the only way we have to reindex our search cluster, so it's definitely something we need to keep around.


Resolving patch:
diff --git a/readthedocs/core/management/commands/reindex_elasticsearch.py b/readthedocs/core/management/commands/reindex_elasticsearch.py
--- a/readthedocs/core/management/commands/reindex_elasticsearch.py
+++ b/readthedocs/core/management/commands/reindex_elasticsearch.py
@@ -2,12 +2,12 @@
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
 from django.conf import settings
 
 from readthedocs.builds.constants import LATEST
 from readthedocs.builds.models import Version
-from readthedocs.search import parse_json
-from readthedocs.restapi.utils import index_search_request
+from readthedocs.projects.tasks import update_search
 
 log = logging.getLogger(__name__)
 
@@ -27,13 +27,17 @@ def handle(self, *args, **options):
         '''
         project = options['project']
 
+        queryset = Version.objects.public()
+
         if project:
-            queryset = Version.objects.public(project__slug=project)
+            queryset = queryset.filter(project__slug=project)
+            if not queryset.exists():
+                raise CommandError(
+                    'No project with slug: {slug}'.format(slug=project))
             log.info("Building all versions for %s" % project)
         elif getattr(settings, 'INDEX_ONLY_LATEST', True):
-            queryset = Version.objects.public().filter(slug=LATEST)
-        else:
-            queryset = Version.objects.public()
+            queryset = queryset.filter(slug=LATEST)
+
         for version in queryset:
             log.info("Reindexing %s" % version)
             try:
@@ -41,10 +45,9 @@ def handle(self, *args, **options):
             except:
                 # This will happen on prod
                 commit = None
+
             try:
-                page_list = parse_json.process_all_json_files(version, build_dir=False)
-                index_search_request(
-                    version=version, page_list=page_list, commit=commit,
-                    project_scale=0, page_scale=0, section=False, delete=False)
+                update_search(version.pk, commit,
+                              delete_non_commit_files=False)
             except Exception:
-                log.error('Build failed for %s' % version, exc_info=True)
+                log.error('Reindex failed for %s' % version, exc_info=True)
diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py
--- a/readthedocs/projects/tasks.py
+++ b/readthedocs/projects/tasks.py
@@ -613,11 +613,12 @@ def move_files(version_pk, hostname, html=False, localmedia=False, search=False,
 
 
 @task(queue='web')
-def update_search(version_pk, commit):
+def update_search(version_pk, commit, delete_non_commit_files=True):
     """Task to update search indexes
 
     :param version_pk: Version id to update
     :param commit: Commit that updated index
+    :param delete_non_commit_files: Delete files not in commit from index
     """
     version = Version.objects.get(pk=version_pk)
 
@@ -642,6 +643,7 @@ def update_search(version_pk, commit):
         # Don't index sections to speed up indexing.
         # They aren't currently exposed anywhere.
         section=False,
+        delete=delete_non_commit_files,
     )
 
 


Test patch: