Repository: readthedocs/readthedocs.org
Issue: Search isn't indexing the index page of nested pages in htmldir builder 

You can test with this project

https://anymail.readthedocs.io/en/latest/search/?q=Sending+email&check_keywords=yes&area=default

There aren't results from this page https://anymail.readthedocs.io/en/latest/sending/

This is kind of related to the doctype that we want to remove https://github.com/rtfd/readthedocs.org/issues/4638 and kind of the same as https://github.com/rtfd/readthedocs.org/issues/5254

In a more minimal example this is more visible, our search returns 0 results and fallback to sphinx.

The explanation is that we try to index the file `sending.fjson` but that file doesn't exists, what we want is to index `sending/index.fjson`

Changed files:
readthedocs/projects/models.py

diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py
index e1662fc63..5991a41bb 100644
--- a/readthedocs/projects/models.py
+++ b/readthedocs/projects/models.py
@@ -11,7 +11,6 @@ from django.contrib.auth.models import User
 from django.core.files.storage import get_storage_class
 from django.db import models
 from django.urls import NoReverseMatch, reverse
-from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
 from guardian.shortcuts import assign
@@ -1143,31 +1142,34 @@ class HTMLFile(ImportedFile):
 
     objects = HTMLFileManager()
 
-    @cached_property
-    def json_file_path(self):
+    def get_processed_json(self):
+        """
+        Get the parsed JSON for search indexing.
+
+        Check for two paths for each index file
+        This is because HTMLDir can generate a file from two different places:
+
+        * foo.rst
+        * foo/index.rst
+
+        Both lead to `foo/index.html`
+        https://github.com/rtfd/readthedocs.org/issues/5368
+        """
+        paths = []
         basename = os.path.splitext(self.path)[0]
-        if self.project.documentation_type == 'sphinx_htmldir' and basename.endswith('/index'):
+        paths.append(basename + '.fjson')
+        if basename.endswith('/index'):
             new_basename = re.sub(r'\\/index$', '', basename)
-            log.info(
-                'Adjusted json file path: %s -> %s',
-                basename,
-                new_basename,
-            )
-            basename = new_basename
-
-        file_path = basename + '.fjson'
+            paths.append(new_basename + '.fjson')
 
         full_json_path = self.project.get_production_media_path(
             type_='json', version_slug=self.version.slug, include_file=False
         )
-
-        file_path = os.path.join(full_json_path, file_path)
-        return file_path
-
-    def get_processed_json(self):
-        file_path = self.json_file_path
         try:
-            return process_file(file_path)
+            for path in paths:
+                file_path = os.path.join(full_json_path, path)
+                if os.path.exists(file_path):
+                    return process_file(file_path)
         except Exception:
             log.warning(
                 'Unhandled exception during search processing file: %s',
@@ -1181,7 +1183,7 @@ class HTMLFile(ImportedFile):
             'sections': [],
         }
 
-    @cached_property
+    @property
     def processed_json(self):
         return self.get_processed_json()