Repository: openfun/richie

Issue:
Children of unpublished categories can be indexed in Elasticsearch and shouldn't
## Bug Report

**Expected behavior/code**
When a category (or meta category) is unpublished (ie not published for all languages), we should not index it or any of its descendants.

**Actual Behavior**
The category indexers loops on all "published" categories and indexes them without checking the publication status of their ancestors.

**Steps to Reproduce**
1. Prepare a category tree in which a leaf category is published and one of its ancestors is not published in any language.
2. Generate the Elasticsearch index
3. Notice that the leaf category is indexed but shouldn't

**Environment**
- Richie version: 1.0.0-beta.9
- Platform: Docker

**Possible Solution**
The category indexer should walk the category tree from the meta categories and down the tree, level by level, checking each time that the category it is indexing is published and skipping on children of unpublished categories.



Discussion:

Resolving patch:
diff --git a/src/richie/apps/courses/models/course.py b/src/richie/apps/courses/models/course.py
--- a/src/richie/apps/courses/models/course.py
+++ b/src/richie/apps/courses/models/course.py
@@ -363,7 +363,7 @@ def get_categories(self, language=None):
             Category, CategoryPluginModel, language=language
         )
 
-    def get_root_to_leaf_category_pages(self):
+    def get_root_to_leaf_public_category_pages(self):
         """
         Build a query retrieving all pages linked to this course's categories or one of their
         ancestors excluding the meta category itself.
@@ -375,7 +375,9 @@ def get_root_to_leaf_category_pages(self):
         category_query = self.get_categories()
 
         # 1. We want the pages directly related to a category of the course
-        page_query = Page.objects.filter(publisher_draft__category__in=category_query)
+        page_query = Page.objects.filter(
+            publisher_draft__category__in=category_query, title_set__published=True
+        )
         # 2. We want the pages related to one of the ancestors of the categories of the course
         for category in category_query.select_related(
             "public_extension__extended_object"
@@ -386,7 +388,8 @@ def get_root_to_leaf_category_pages(self):
             # Don't include the meta category as it materializes a "filter bank" and not a
             # search option
             page_query = page_query | ancestor_query.filter(
-                node__parent__cms_pages__category__isnull=False
+                node__parent__cms_pages__category__isnull=False,
+                title_set__published=True,
             )
         return page_query.distinct()
 
diff --git a/src/richie/apps/search/indexers/courses.py b/src/richie/apps/search/indexers/courses.py
--- a/src/richie/apps/search/indexers/courses.py
+++ b/src/richie/apps/search/indexers/courses.py
@@ -490,7 +490,7 @@ def get_es_document_for_course(cls, course, index=None, action="index"):
         # Prepare categories, making sure we get title information for categories
         # in the same query
         category_pages = (
-            course.get_root_to_leaf_category_pages()
+            course.get_root_to_leaf_public_category_pages()
             .select_related("node")
             .prefetch_related(
                 Prefetch(


Test patch:
diff --git a/tests/apps/courses/test_models_course.py b/tests/apps/courses/test_models_course.py
--- a/tests/apps/courses/test_models_course.py
+++ b/tests/apps/courses/test_models_course.py
@@ -1143,7 +1143,7 @@ def test_models_course_get_programs_several_languages(self):
         self.assertEqual(Program.objects.count(), 2)
         self.assertEqual(course.get_programs().count(), 1)
 
-    def test_models_course_get_root_to_leaf_category_pages_leaf(self):
+    def test_models_course_get_root_to_leaf_public_category_pages_leaf(self):
         """
         A course linked to a leaf category, in a nested category tree, should be associated
         with all the category's ancestors.
@@ -1166,9 +1166,11 @@ def test_models_course_get_root_to_leaf_category_pages_leaf(self):
             parent_category.public_extension.extended_object,
             leaf_category.public_extension.extended_object,
         ]
-        self.assertEqual(expected_pages, list(course.get_root_to_leaf_category_pages()))
+        self.assertEqual(
+            expected_pages, list(course.get_root_to_leaf_public_category_pages())
+        )
 
-    def test_models_course_get_root_to_leaf_category_pages_parent(self):
+    def test_models_course_get_root_to_leaf_public_category_pages_parent(self):
         """
         A course linked to a parent category, in a nested category tree, should be associated
         with all the category's ancestors, but not we the child.
@@ -1188,9 +1190,11 @@ def test_models_course_get_root_to_leaf_category_pages_parent(self):
         )
 
         expected_pages = [parent_category.public_extension.extended_object]
-        self.assertEqual(expected_pages, list(course.get_root_to_leaf_category_pages()))
+        self.assertEqual(
+            expected_pages, list(course.get_root_to_leaf_public_category_pages())
+        )
 
-    def test_models_course_get_root_to_leaf_category_pages_duplicate(self):
+    def test_models_course_get_root_to_leaf_public_category_pages_duplicate(self):
         """
         If the course is linked to several categories, the ancestor categories should not get
         duplicated.
@@ -1217,7 +1221,9 @@ def test_models_course_get_root_to_leaf_category_pages_duplicate(self):
             leaf_category1.public_extension.extended_object,
             leaf_category2.public_extension.extended_object,
         ]
-        self.assertEqual(expected_pages, list(course.get_root_to_leaf_category_pages()))
+        self.assertEqual(
+            expected_pages, list(course.get_root_to_leaf_public_category_pages())
+        )
 
     # Fields: effort
 
diff --git a/tests/apps/search/test_indexers_categories.py b/tests/apps/search/test_indexers_categories.py
--- a/tests/apps/search/test_indexers_categories.py
+++ b/tests/apps/search/test_indexers_categories.py
@@ -141,6 +141,57 @@ def test_indexers_categories_get_es_documents(self, _mock_picture):
             ],
         )
 
+    def test_indexers_categories_get_es_documents_unpublished(self):
+        """
+        Unpublished categories and children of unpublished categories should not be indexed
+        """
+        # Our meta category and its page
+        meta = CategoryFactory(
+            page_parent=create_i18n_page("Categories", published=True),
+            page_reverse_id="subjects",
+            page_title="Subjects",
+            should_publish=True,
+        )
+        parent = CategoryFactory(page_parent=meta.extended_object, should_publish=True)
+        CategoryFactory(
+            page_parent=parent.extended_object,
+            page_title="my second subject",
+            should_publish=True,
+        )
+
+        # Unpublish the parent category
+        self.assertTrue(parent.extended_object.unpublish("en"))
+
+        # The unpublished parent category and its child should not get indexed
+        self.assertEqual(
+            list(
+                CategoriesIndexer.get_es_documents(
+                    index="some_index", action="some_action"
+                )
+            ),
+            [
+                {
+                    "_id": "P-00010001",
+                    "_index": "some_index",
+                    "_op_type": "some_action",
+                    "_type": "category",
+                    "absolute_url": {
+                        "en": "/en/categories/subjects/",
+                        "fr": "/fr/categories/subjects/",
+                    },
+                    "complete": {"en": ["Subjects"]},
+                    "description": {},
+                    "icon": {},
+                    "is_meta": True,
+                    "kind": None,
+                    "logo": {},
+                    "nb_children": 1,
+                    "path": "00010001",
+                    "title": {"en": "Subjects"},
+                },
+            ],
+        )
+
     def test_indexers_categories_get_es_documents_language_fallback(self):
         """Absolute urls should be computed as expected with language fallback."""
         # Our meta category and its page
diff --git a/tests/apps/search/test_indexers_courses.py b/tests/apps/search/test_indexers_courses.py
--- a/tests/apps/search/test_indexers_courses.py
+++ b/tests/apps/search/test_indexers_courses.py
@@ -9,6 +9,7 @@
 import pytz
 from cms.api import add_plugin
 
+from richie.apps.core.helpers import create_i18n_page
 from richie.apps.courses.cms_plugins import CategoryPlugin
 from richie.apps.courses.defaults import HOUR, MINUTE, WEEK
 from richie.apps.courses.factories import (
@@ -24,6 +25,8 @@
 from richie.apps.search.indexers.organizations import OrganizationsIndexer
 from richie.plugins.simple_picture.cms_plugins import SimplePicturePlugin
 
+# pylint: disable=too-many-public-methods
+
 
 class CoursesIndexersTestCase(TestCase):
     """
@@ -81,6 +84,82 @@ def test_indexers_courses_get_es_documents_no_course_run(self):
         self.assertEqual(len(indexed_courses), 1)
         self.assertEqual(indexed_courses[0]["course_runs"], [])
 
+    def test_indexers_courses_get_es_documents_unpublished_course(self):
+        """Unpublished courses should not be indexed"""
+        CourseFactory()
+
+        self.assertEqual(
+            list(
+                CoursesIndexer.get_es_documents(
+                    index="some_index", action="some_action"
+                )
+            ),
+            [],
+        )
+
+    def test_indexers_courses_get_es_documents_unpublished_category(self):
+        """
+        Unpublished categories and children of unpublished categories should not be indexed
+        """
+        # Create a child category
+        meta = CategoryFactory(
+            page_parent=create_i18n_page("Categories", published=True),
+            page_reverse_id="subjects",
+            page_title="Subjects",
+            should_publish=True,
+        )
+        parent = CategoryFactory(page_parent=meta.extended_object, should_publish=True)
+        category = CategoryFactory(
+            page_parent=parent.extended_object,
+            page_title="my second subject",
+            should_publish=True,
+        )
+
+        CourseFactory(fill_categories=[category], should_publish=True)
+
+        # Unpublish the parent category
+        self.assertTrue(parent.extended_object.unpublish("en"))
+
+        course_document = list(
+            CoursesIndexer.get_es_documents(index="some_index", action="some_action")
+        )[0]
+
+        # Neither the category not its parent should be linked to the course
+        self.assertEqual(course_document["categories"], [])
+        self.assertEqual(course_document["categories_names"], {})
+
+    def test_indexers_courses_get_es_documents_unpublished_organization(self):
+        """Unpublished organizations should not be indexed."""
+        organization = OrganizationFactory(should_publish=True)
+        CourseFactory(fill_organizations=[organization], should_publish=True)
+
+        # Unpublish the organization
+        self.assertTrue(organization.extended_object.unpublish("en"))
+
+        course_document = list(
+            CoursesIndexer.get_es_documents(index="some_index", action="some_action")
+        )[0]
+
+        # The unpublished organization should not be linked to the course
+        self.assertEqual(course_document["organizations"], [])
+        self.assertEqual(course_document["organizations_names"], {})
+
+    def test_indexers_courses_get_es_documents_unpublished_person(self):
+        """Unpublished persons should not be indexed."""
+        person = PersonFactory(should_publish=True)
+        CourseFactory(fill_team=[person], should_publish=True)
+
+        # Unpublish the person
+        self.assertTrue(person.extended_object.unpublish("en"))
+
+        course_document = list(
+            CoursesIndexer.get_es_documents(index="some_index", action="some_action")
+        )[0]
+
+        # The unpublished person should not be linked to the course
+        self.assertEqual(course_document["persons"], [])
+        self.assertEqual(course_document["persons_names"], {})
+
     def test_indexers_courses_get_es_documents_snapshots(self):
         """
         Course snapshots should not get indexed.
diff --git a/tests/apps/search/test_indexers_organizations.py b/tests/apps/search/test_indexers_organizations.py
--- a/tests/apps/search/test_indexers_organizations.py
+++ b/tests/apps/search/test_indexers_organizations.py
@@ -121,6 +121,20 @@ def test_indexers_organizations_get_es_documents(self, _mock_picture):
             ],
         )
 
+    def test_indexers_organizations_get_es_documents_unpublished(self):
+        """Unpublished organizations should not be indexed"""
+        OrganizationFactory()
+
+        # The unpublished organization should not get indexed
+        self.assertEqual(
+            list(
+                OrganizationsIndexer.get_es_documents(
+                    index="some_index", action="some_action"
+                )
+            ),
+            [],
+        )
+
     def test_indexers_organizations_get_es_documents_language_fallback(self):
         """Absolute urls should be computed as expected with language fallback."""
         OrganizationFactory(
diff --git a/tests/apps/search/test_indexers_persons.py b/tests/apps/search/test_indexers_persons.py
--- a/tests/apps/search/test_indexers_persons.py
+++ b/tests/apps/search/test_indexers_persons.py
@@ -96,6 +96,20 @@ def test_indexers_persons_get_es_documents(self, _mock_picture):
             ],
         )
 
+    def test_indexers_persons_get_es_documents_unpublished(self):
+        """Unpublished persons should not be indexed"""
+        PersonFactory()
+
+        # The unpublished person should not get indexed
+        self.assertEqual(
+            list(
+                PersonsIndexer.get_es_documents(
+                    index="some_index", action="some_action"
+                )
+            ),
+            [],
+        )
+
     def test_indexers_persons_get_es_documents_language_fallback(self):
         """Absolute urls should be computed as expected with language fallback."""
         PersonFactory(