Repository: learningequality/kolibri

Issue:
num_coach_contents is not annotated on existing channels
### Observed behavior
Since the merging of #5497 num_coach_contents is no longer a dynamically calculated field, and instead is calculated on channel import and annotation.

This means that the result will be 0 for any content that was imported prior to 0.12.4.

### Expected behavior
num_coach_contents should accurately represent the number of coach contents available.

### User-facing consequences
Minor confusion.


Discussion:
This could be fixed by running `scanforcontent` on every server startup, but this is an excessively time consuming process. What would be better is to have some sort of version specific upgrade mechanism as implemented in #5494 - but unsure if this is too risky to target for 0.12.4?

Resolving patch:
diff --git a/kolibri/core/content/utils/annotation.py b/kolibri/core/content/utils/annotation.py
--- a/kolibri/core/content/utils/annotation.py
+++ b/kolibri/core/content/utils/annotation.py
@@ -70,6 +70,7 @@ def update_channel_metadata():
                     )
                 )
     fix_multiple_trees_with_id_one()
+    update_num_coach_contents()
     connection.close()
 
 
@@ -134,6 +135,88 @@ def fix_multiple_trees_with_id_one():
             )
 
 
+def update_num_coach_contents():
+    """
+    Function to set num_coach_content on all topic trees to account for
+    those that were imported before annotations were performed
+    """
+    bridge = Bridge(app_name=CONTENT_APP_NAME)
+
+    ContentNodeClass = bridge.get_class(ContentNode)
+
+    ContentNodeTable = bridge.get_table(ContentNode)
+
+    connection = bridge.get_connection()
+
+    child = ContentNodeTable.alias()
+
+    logger.info("Updating num_coach_content on existing channels")
+
+    # start a transaction
+
+    trans = connection.begin()
+
+    # Update all leaf ContentNodes to have num_coach_content to 1 or 0
+    connection.execute(
+        ContentNodeTable.update()
+        .where(
+            # That are not topics
+            ContentNodeTable.c.kind
+            != content_kinds.TOPIC
+        )
+        .values(num_coach_contents=cast(ContentNodeTable.c.coach_content, Integer()))
+    )
+
+    # Expression to capture all available child nodes of a contentnode
+    available_nodes = select([child.c.available]).where(
+        and_(
+            child.c.available == True,  # noqa
+            ContentNodeTable.c.id == child.c.parent_id,
+        )
+    )
+
+    # Expression that sums the total number of coach contents for each child node
+    # of a contentnode
+    coach_content_num = select([func.sum(child.c.num_coach_contents)]).where(
+        and_(
+            child.c.available == True,  # noqa
+            ContentNodeTable.c.id == child.c.parent_id,
+        )
+    )
+
+    for channel_id in ChannelMetadata.objects.all().values_list("id", flat=True):
+
+        node_depth = (
+            bridge.session.query(func.max(ContentNodeClass.level))
+            .filter_by(channel_id=channel_id)
+            .scalar()
+        )
+
+        # Go from the deepest level to the shallowest
+        for level in range(node_depth, 0, -1):
+
+            # Only modify topic availability here
+            connection.execute(
+                ContentNodeTable.update()
+                .where(
+                    and_(
+                        ContentNodeTable.c.level == level - 1,
+                        ContentNodeTable.c.channel_id == channel_id,
+                        ContentNodeTable.c.kind == content_kinds.TOPIC,
+                    )
+                )
+                # Because we have set availability to False on all topics as a starting point
+                # we only need to make updates to topics with available children.
+                .where(exists(available_nodes))
+                .values(num_coach_contents=coach_content_num)
+            )
+
+    # commit the transaction
+    trans.commit()
+
+    bridge.end()
+
+
 def set_leaf_node_availability_from_local_file_availability(channel_id):
     bridge = Bridge(app_name=CONTENT_APP_NAME)
 
@@ -291,7 +374,11 @@ def recurse_annotation_up_tree(channel_id):
 
     connection = bridge.get_connection()
 
-    node_depth = bridge.session.query(func.max(ContentNodeClass.level)).scalar()
+    node_depth = (
+        bridge.session.query(func.max(ContentNodeClass.level))
+        .filter_by(channel_id=channel_id)
+        .scalar()
+    )
 
     logger.info(
         "Annotating ContentNode objects with children for {levels} levels".format(
@@ -334,49 +421,49 @@ def recurse_annotation_up_tree(channel_id):
         .values(available=False)
     )
 
-    # Go from the deepest level to the shallowest
-    for level in range(node_depth, 0, -1):
+    # Expression to capture all available child nodes of a contentnode
+    available_nodes = select([child.c.available]).where(
+        and_(
+            child.c.available == True,  # noqa
+            ContentNodeTable.c.id == child.c.parent_id,
+        )
+    )
 
-        # Expression to capture all available child nodes of a contentnode
-        available_nodes = select([child.c.available]).where(
+    # Expressions for annotation of coach content
+
+    # Expression that will resolve a boolean value for all the available children
+    # of a content node, whereby if they all have coach_content flagged on them, it will be true,
+    # but otherwise false.
+    # Everything after the select statement should be identical to the available_nodes expression above.
+    if bridge.engine.name == "sqlite":
+        # Use a min function to simulate an AND.
+        coach_content_nodes = select([func.min(child.c.coach_content)]).where(
             and_(
                 child.c.available == True,  # noqa
                 ContentNodeTable.c.id == child.c.parent_id,
             )
         )
-
-        # Expressions for annotation of coach content
-
-        # Expression that will resolve a boolean value for all the available children
-        # of a content node, whereby if they all have coach_content flagged on them, it will be true,
-        # but otherwise false.
-        # Everything after the select statement should be identical to the available_nodes expression above.
-        if bridge.engine.name == "sqlite":
-            # Use a min function to simulate an AND.
-            coach_content_nodes = select([func.min(child.c.coach_content)]).where(
-                and_(
-                    child.c.available == True,  # noqa
-                    ContentNodeTable.c.id == child.c.parent_id,
-                )
-            )
-        elif bridge.engine.name == "postgresql":
-            # Use the postgres boolean AND operator
-            coach_content_nodes = select([func.bool_and(child.c.coach_content)]).where(
-                and_(
-                    child.c.available == True,  # noqa
-                    ContentNodeTable.c.id == child.c.parent_id,
-                )
-            )
-
-        # Expression that sums the total number of coach contents for each child node
-        # of a contentnode
-        coach_content_num = select([func.sum(child.c.num_coach_contents)]).where(
+    elif bridge.engine.name == "postgresql":
+        # Use the postgres boolean AND operator
+        coach_content_nodes = select([func.bool_and(child.c.coach_content)]).where(
             and_(
                 child.c.available == True,  # noqa
                 ContentNodeTable.c.id == child.c.parent_id,
             )
         )
 
+    # Expression that sums the total number of coach contents for each child node
+    # of a contentnode
+    coach_content_num = select([func.sum(child.c.num_coach_contents)]).where(
+        and_(
+            child.c.available == True,  # noqa
+            ContentNodeTable.c.id == child.c.parent_id,
+        )
+    )
+
+    # Go from the deepest level to the shallowest
+    for level in range(node_depth, 0, -1):
+
         logger.info(
             "Annotating ContentNode objects with children for level {level}".format(
                 level=level
diff --git a/kolibri/core/content/utils/sqlalchemybridge.py b/kolibri/core/content/utils/sqlalchemybridge.py
--- a/kolibri/core/content/utils/sqlalchemybridge.py
+++ b/kolibri/core/content/utils/sqlalchemybridge.py
@@ -252,6 +252,10 @@ def get_class(self, DjangoModel):
         return get_class(DjangoModel, self.Base)
 
     def get_table(self, DjangoModel):
+        """
+        Return the SQLAlchemy Table object associated with this Django Model
+        https://docs.sqlalchemy.org/en/13/orm/extensions/declarative/table_config.html#using-a-hybrid-approach-with-table
+        """
         return self.get_class(DjangoModel).__table__
 
     def get_connection(self):


Test patch:
diff --git a/kolibri/core/content/test/test_annotation.py b/kolibri/core/content/test/test_annotation.py
--- a/kolibri/core/content/test/test_annotation.py
+++ b/kolibri/core/content/test/test_annotation.py
@@ -27,6 +27,7 @@
     set_leaf_node_availability_from_local_file_availability,
 )
 from kolibri.core.content.utils.annotation import set_local_file_availability_from_disk
+from kolibri.core.content.utils.annotation import update_num_coach_contents
 
 
 def get_engine(connection_string):
@@ -666,6 +667,125 @@ def test_two_extra_channels_one_contentdb_exists(self, path_mock, import_mock):
         import_mock.assert_called_with(root_node_2.channel_id)
 
 
+@patch("kolibri.core.content.utils.sqlalchemybridge.get_engine", new=get_engine)
+class UpdateNumCoachContents(TransactionTestCase):
+
+    fixtures = ["content_test.json"]
+
+    def setUp(self):
+        super(UpdateNumCoachContents, self).setUp()
+        ContentNode.objects.all().update(available=False)
+
+    def test_no_content_nodes_coach_content(self):
+        ContentNode.objects.all().update(available=True)
+        ContentNode.objects.all().update(coach_content=False)
+        update_num_coach_contents()
+        root = ChannelMetadata.objects.get(id=test_channel_id).root
+        self.assertEqual(root.num_coach_contents, 0)
+
+    def test_all_root_content_nodes_coach_content(self):
+        ContentNode.objects.all().update(available=True, coach_content=False)
+        root_node = ContentNode.objects.get(parent__isnull=True)
+        ContentNode.objects.filter(parent=root_node).exclude(
+            kind=content_kinds.TOPIC
+        ).update(coach_content=True)
+        update_num_coach_contents()
+        root_node.refresh_from_db()
+        self.assertEqual(root_node.num_coach_contents, 2)
+
+    def test_one_root_content_node_coach_content(self):
+        ContentNode.objects.all().update(available=True, coach_content=False)
+        root_node = ContentNode.objects.get(parent__isnull=True)
+        node = (
+            ContentNode.objects.filter(parent=root_node)
+            .exclude(kind=content_kinds.TOPIC)
+            .first()
+        )
+        node.coach_content = True
+        node.save()
+        update_num_coach_contents()
+        root_node.refresh_from_db()
+        self.assertEqual(root_node.num_coach_contents, 1)
+
+    def test_one_root_topic_node_coach_content(self):
+        ContentNode.objects.all().update(available=True, coach_content=False)
+        root_node = ContentNode.objects.get(parent__isnull=True)
+        node = ContentNode.objects.filter(
+            parent=root_node, kind=content_kinds.TOPIC
+        ).first()
+        node.coach_content = True
+        node.save()
+        update_num_coach_contents()
+        root_node.refresh_from_db()
+        self.assertEqual(root_node.num_coach_contents, 0)
+
+    def test_one_child_node_coach_content(self):
+        ContentNode.objects.all().update(available=True, coach_content=False)
+        root_node = ContentNode.objects.get(parent__isnull=True)
+        node = ContentNode.objects.filter(
+            parent=root_node, kind=content_kinds.TOPIC
+        ).first()
+        ContentNode.objects.create(
+            title="test1",
+            id=uuid.uuid4().hex,
+            content_id=uuid.uuid4().hex,
+            channel_id=root_node.channel_id,
+            parent=node,
+            kind=content_kinds.VIDEO,
+            available=True,
+            coach_content=True,
+        )
+        update_num_coach_contents()
+        root_node.refresh_from_db()
+        node.refresh_from_db()
+        self.assertEqual(root_node.num_coach_contents, 1)
+        self.assertEqual(node.num_coach_contents, 1)
+
+    def test_one_child_coach_content_parent_no_siblings(self):
+        ContentNode.objects.all().update(available=True, coach_content=False)
+        root_node = ContentNode.objects.get(parent__isnull=True)
+        topic_node = ContentNode.objects.filter(
+            parent=root_node, kind=content_kinds.TOPIC
+        ).first()
+        parent_node = ContentNode.objects.create(
+            title="test1",
+            id=uuid.uuid4().hex,
+            content_id=uuid.uuid4().hex,
+            channel_id=root_node.channel_id,
+            parent=topic_node,
+            kind=content_kinds.TOPIC,
+            available=True,
+            coach_content=False,
+        )
+        ContentNode.objects.create(
+            title="test2",
+            id=uuid.uuid4().hex,
+            content_id=uuid.uuid4().hex,
+            channel_id=root_node.channel_id,
+            parent=parent_node,
+            kind=content_kinds.VIDEO,
+            available=True,
+            coach_content=True,
+        )
+        ContentNode.objects.create(
+            title="test3",
+            id=uuid.uuid4().hex,
+            content_id=uuid.uuid4().hex,
+            channel_id=root_node.channel_id,
+            parent=parent_node,
+            kind=content_kinds.VIDEO,
+            available=True,
+            coach_content=False,
+        )
+        update_num_coach_contents()
+        parent_node.refresh_from_db()
+        self.assertEqual(parent_node.num_coach_contents, 1)
+
+    def tearDown(self):
+        call_command("flush", interactive=False)
+        super(UpdateNumCoachContents, self).tearDown()
+
+
 class CalculateChannelFieldsTestCase(TestCase):
     def setUp(self):
         self.node = ContentNode.objects.create(
diff --git a/kolibri/core/content/test/test_enumerate_channel.py b/kolibri/core/content/test/test_enumerate_channel.py
--- a/kolibri/core/content/test/test_enumerate_channel.py
+++ b/kolibri/core/content/test/test_enumerate_channel.py
@@ -36,8 +36,11 @@ def create_corrupted_database_file(self, db_dir):
             f.write("test corrupted database file")
         return db_file
 
+    @patch("kolibri.core.content.utils.annotation.update_num_coach_contents")
     @patch("kolibri.core.content.utils.annotation.logger.warning")
-    def test_corrupted_database_file_server_start(self, logger_mock):
+    def test_corrupted_database_file_server_start(
+        self, logger_mock, coach_contents_mock
+    ):
         db_file = self.create_corrupted_database_file(get_content_database_dir_path())
         update_channel_metadata()
         message_list = [message[0][0] for message in logger_mock.call_args_list]
diff --git a/kolibri/core/logger/test/test_handler.py b/kolibri/utils/tests/test_handler.py
similarity index 87%
rename from kolibri/core/logger/test/test_handler.py
rename to kolibri/utils/tests/test_handler.py
--- a/kolibri/core/logger/test/test_handler.py
+++ b/kolibri/utils/tests/test_handler.py
@@ -15,9 +15,9 @@ def test_do_rollover(self):
         # Temporarily set the rotation time of the log file to be every second
         settings.LOGGING["handlers"]["file"]["when"] = "s"
         # make sure that kolibri will be running for more than one second
-        cli.main(["language", "setdefault", "en"])
+        cli.main(["--skipupdate", "manage", "help"])
         sleep(1)
-        cli.main(["language", "setdefault", "en"])
+        cli.main(["--skipupdate", "manage", "help"])
         # change back to the original rotation time
         settings.LOGGING["handlers"]["file"]["when"] = orig_value