Repository: readthedocs/readthedocs.org
Issue: Fix or abandon reindex_elasticsearch

Currently the managment command `reindex_elasticsearch` [located here](https://github.com/rtfd/readthedocs.org/blob/master/readthedocs/core/management/commands/reindex_elasticsearch.py) is broken in at least two ways:
1. We give a [filter argument to the version's manager public method](https://github.com/rtfd/readthedocs.org/blob/adf227ae3810b0820d0ff548651dcd223f4f2388/readthedocs/core/management/commands/reindex_elasticsearch.py#L31). But that does not do any filtering. Therefore only selecting one project that should be updated will result in all projects to be reindexed.
2. We do not distinguish between mkdocs/sphinx projects. The used `parse_json.process_all_json_files` will only work correctly for Sphinx based projects.

We should either remove the command completely as long as it no longer used/needed. Or fix the issues outlined above.


Changed files:
readthedocs/projects/tasks.py, readthedocs/core/management/commands/reindex_elasticsearch.py

diff --git a/readthedocs/core/management/commands/reindex_elasticsearch.py b/readthedocs/core/management/commands/reindex_elasticsearch.py
index 76f25dbd3..83ba626bd 100644
--- a/readthedocs/core/management/commands/reindex_elasticsearch.py
+++ b/readthedocs/core/management/commands/reindex_elasticsearch.py
@@ -2,12 +2,12 @@ import logging
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
 from django.conf import settings
 
 from readthedocs.builds.constants import LATEST
 from readthedocs.builds.models import Version
-from readthedocs.search import parse_json
-from readthedocs.restapi.utils import index_search_request
+from readthedocs.projects.tasks import update_search
 
 log = logging.getLogger(__name__)
 
@@ -27,13 +27,17 @@ class Command(BaseCommand):
         '''
         project = options['project']
 
+        queryset = Version.objects.public()
+
         if project:
-            queryset = Version.objects.public(project__slug=project)
+            queryset = queryset.filter(project__slug=project)
+            if not queryset.exists():
+                raise CommandError(
+                    'No project with slug: {slug}'.format(slug=project))
             log.info("Building all versions for %s" % project)
         elif getattr(settings, 'INDEX_ONLY_LATEST', True):
-            queryset = Version.objects.public().filter(slug=LATEST)
-        else:
-            queryset = Version.objects.public()
+            queryset = queryset.filter(slug=LATEST)
+
         for version in queryset:
             log.info("Reindexing %s" % version)
             try:
@@ -41,10 +45,9 @@ class Command(BaseCommand):
             except:
                 # This will happen on prod
                 commit = None
+
             try:
-                page_list = parse_json.process_all_json_files(version, build_dir=False)
-                index_search_request(
-                    version=version, page_list=page_list, commit=commit,
-                    project_scale=0, page_scale=0, section=False, delete=False)
+                update_search(version.pk, commit,
+                              delete_non_commit_files=False)
             except Exception:
-                log.error('Build failed for %s' % version, exc_info=True)
+                log.error('Reindex failed for %s' % version, exc_info=True)
diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py
index f39279997..55bda8fff 100644
--- a/readthedocs/projects/tasks.py
+++ b/readthedocs/projects/tasks.py
@@ -613,11 +613,12 @@ def move_files(version_pk, hostname, html=False, localmedia=False, search=False,
 
 
 @task(queue='web')
-def update_search(version_pk, commit):
+def update_search(version_pk, commit, delete_non_commit_files=True):
     """Task to update search indexes
 
     :param version_pk: Version id to update
     :param commit: Commit that updated index
+    :param delete_non_commit_files: Delete files not in commit from index
     """
     version = Version.objects.get(pk=version_pk)
 
@@ -642,6 +643,7 @@ def update_search(version_pk, commit):
         # Don't index sections to speed up indexing.
         # They aren't currently exposed anywhere.
         section=False,
+        delete=delete_non_commit_files,
     )