Repository: learningequality/kolibri

Issue:
Reinstate loading of additional resources in the topics page
## Observed behavior
Currently we display at most 4 items within a topic in the topics page - but have no option on how to show any more.

## Expected behavior
If there are 4 more results available locally, then it should immediately display them.
If not, it should load more from the backend and show a loading state until the request is completed.

## User-facing consequences
Confusion as only 4 items in a topic are ever shown!



Discussion:

Resolving patch:
diff --git a/kolibri/core/content/api.py b/kolibri/core/content/api.py
--- a/kolibri/core/content/api.py
+++ b/kolibri/core/content/api.py
@@ -810,19 +810,21 @@ def resume(self, request, **kwargs):
         return Response(self.serialize(queryset))
 
 
+# The max recursed page size should be less than 25 for a couple of reasons:
+# 1. At this size the query appears to be relatively performant, and will deliver most of the tree
+#    data to the frontend in a single query.
+# 2. In the case where the tree topology means that this will not produce the full query, the limit of
+#    25 immediate children and 25 grand children means that we are at most using 1 + 25 + 25 * 25 = 651
+#    SQL parameters in the query to get the nodes for serialization - this means that we should not ever
+#    run into an issue where we hit a SQL parameters limit in the queries in here.
+# If we find that this page size is too high, we should lower it, but for the reasons noted above, we
+# should not raise it.
+NUM_CHILDREN = 12
+NUM_GRANDCHILDREN_PER_CHILD = 12
+
+
 @method_decorator(cache_forever, name="dispatch")
 class ContentNodeTreeViewset(BaseContentNodeMixin, BaseValuesViewset):
-    # We fix the page size at 25 for a couple of reasons:
-    # 1. At this size the query appears to be relatively performant, and will deliver most of the tree
-    #    data to the frontend in a single query.
-    # 2. In the case where the tree topology means that this will not produce the full query, the limit of
-    #    25 immediate children and 25 grand children means that we are at most using 1 + 25 + 25 * 25 = 651
-    #    SQL parameters in the query to get the nodes for serialization - this means that we should not ever
-    #    run into an issue where we hit a SQL parameters limit in the queries in here.
-    # If we find that this page size is too high, we should lower it, but for the reasons noted above, we
-    # should not raise it.
-    page_size = 25
-
     def validate_and_return_params(self, request):
         depth = request.query_params.get("depth", 2)
         next__gt = request.query_params.get("next__gt")
@@ -843,9 +845,10 @@ def validate_and_return_params(self, request):
                 raise ValidationError(
                     "next__gt query parameter must be a positive integer if specified"
                 )
+
         return depth, next__gt
 
-    def get_grandchild_ids(self, child_ids, depth):
+    def get_grandchild_ids(self, child_ids, depth, page_size):
         if depth == 2:
             # Use this to keep track of how many grand children we have accumulated per child of the parent node
             gc_by_parent = {}
@@ -863,11 +866,11 @@ def get_grandchild_ids(self, child_ids, depth):
                 # then we keep on adding them to both lists
                 # If not, we just skip this node, as we have already hit the page limit for the node that is
                 # its parent.
-                if len(gc_by_parent[gc["parent_id"]]) < self.page_size:
+                if len(gc_by_parent[gc["parent_id"]]) < page_size:
                     gc_by_parent[gc["parent_id"]].append(gc["id"])
                     yield gc["id"]
 
-    def retrieve(self, request, **kwargs):
+    def retrieve(self, request, pk=None):
         """
         A nested, paginated representation of the children and grandchildren of a specific node
 
@@ -892,28 +895,30 @@ def retrieve(self, request, **kwargs):
         """
 
         # Get the model for the parent node here - we do this so that we trigger a 404 immediately if the node
-        # does not exist (or exists but is not available), and so that we have the Django model object later to
-        # use for the `get_ancestors` MPTT method.
-        parent_model = self.get_object()
+        # does not exist (or exists but is not available).
+        parent_id = pk if pk and self.get_queryset().filter(id=pk).exists() else None
+
+        if parent_id is None:
+            raise Http404
 
         depth, next__gt = self.validate_and_return_params(request)
 
         # Get a list of child_ids of the parent node up to the pagination limit
-        child_qs = self.get_queryset().filter(parent=parent_model)
+        child_qs = self.get_queryset().filter(parent_id=parent_id)
         if next__gt is not None:
             child_qs = child_qs.filter(lft__gt=next__gt)
         child_ids = child_qs.values_list("id", flat=True).order_by("lft")[
-            0 : self.page_size
+            0:NUM_CHILDREN
         ]
 
         # Get a flat list of ids for grandchildren we will be returning
-        gc_ids = self.get_grandchild_ids(child_ids, depth)
+        gc_ids = self.get_grandchild_ids(child_ids, depth, NUM_GRANDCHILDREN_PER_CHILD)
 
         # We explicitly order by lft here, so that the nodes are in tree traversal order, so we can iterate over them and build
         # out our nested representation, being sure that any ancestors have already been processed.
         nodes = self.serialize(
             self.filter_queryset(self.get_queryset())
-            .filter(Q(id=parent_model.id) | Q(id__in=child_ids) | Q(id__in=gc_ids))
+            .filter(Q(id=parent_id) | Q(id__in=child_ids) | Q(id__in=gc_ids))
             .order_by("lft")
         )
 
@@ -929,7 +934,7 @@ def retrieve(self, request, **kwargs):
         for desc in nodes[1:]:
             # First check to see whether it is a direct child of the
             # parent node that we initially queried
-            if desc["parent"] == parent_model.id:
+            if desc["parent"] == parent_id:
                 # If so add them to the children_by_id map so that
                 # grandchildren descendants can reference them later
                 children_by_id[desc["id"]] = desc
@@ -939,6 +944,9 @@ def retrieve(self, request, **kwargs):
                 # When we request more results for pagination, we want to return
                 # both nodes at this level, and the nodes at the lower level
                 more_depth = 2
+                # For the parent node the page size is the maximum number of children
+                # we are returning (regardless of whether they have a full representation)
+                page_size = NUM_CHILDREN
             elif desc["parent"] in children_by_id:
                 # Otherwise, check to see if our descendant's parent is in the
                 # children_by_id map - if it failed the first condition,
@@ -947,6 +955,9 @@ def retrieve(self, request, **kwargs):
                 # When we request more results for pagination, we only want to return
                 # nodes at this level, and not any of its children
                 more_depth = 1
+                # For a child node, the page size is the maximum number of grandchildren
+                # per node that we are returning if it is a recursed node
+                page_size = NUM_GRANDCHILDREN_PER_CHILD
             else:
                 # If we get to here, we have a node that is not in the tree subsection we are
                 # trying to return, so we just ignore it. This shouldn't happen.
@@ -959,7 +970,7 @@ def retrieve(self, request, **kwargs):
             desc_parent["children"]["results"].append(desc)
             # Only bother updating the URL for more if we have hit the page size limit
             # otherwise it will just continue to be None
-            if len(desc_parent["children"]["results"]) == self.page_size:
+            if len(desc_parent["children"]["results"]) == page_size:
                 # Any subsequent queries to get siblings of this node can restrict themselves
                 # to looking for nodes with lft greater than the rght value of this descendant
                 next__gt = desc["rght"]


Test patch:
diff --git a/kolibri/core/content/test/test_content_app.py b/kolibri/core/content/test/test_content_app.py
--- a/kolibri/core/content/test/test_content_app.py
+++ b/kolibri/core/content/test/test_content_app.py
@@ -398,7 +398,7 @@ def _recurse_and_assert(self, data, nodes, recursion_depth=0):
                     self.assertGreater(len(child_nodes), len(children["results"]))
                     self.assertEqual(children["more"]["id"], expected.id)
                     self.assertEqual(
-                        children["more"]["params"]["next__gt"], child_nodes[24].rght
+                        children["more"]["params"]["next__gt"], child_nodes[11].rght
                     )
                     self.assertEqual(
                         children["more"]["params"]["depth"], 2 - recursion_depth
@@ -445,18 +445,18 @@ def test_contentnode_tree_depth_1(self):
         "Skipping postgres as not as vulnerable to large queries and large insertions are less performant",
     )
     def test_contentnode_tree_next__gt(self):
-        builder = ChannelBuilder(levels=2, num_children=30)
+        builder = ChannelBuilder(levels=2, num_children=17)
         builder.insert_into_default_db()
         content.ContentNode.objects.all().update(available=True)
         root = content.ContentNode.objects.get(id=builder.root_node["id"])
-        next__gt = content.ContentNode.objects.filter(parent=root)[24].rght
+        next__gt = content.ContentNode.objects.filter(parent=root)[11].rght
         response = self.client.get(
             reverse("kolibri:core:contentnode_tree-detail", kwargs={"pk": root.id}),
             data={"next__gt": next__gt},
         )
         self.assertEqual(len(response.data["children"]["results"]), 5)
         self.assertIsNone(response.data["children"]["more"])
-        first_node = content.ContentNode.objects.filter(parent=root)[25]
+        first_node = content.ContentNode.objects.filter(parent=root)[12]
         self._recurse_and_assert(
             [response.data["children"]["results"][0]], [first_node], recursion_depth=1
         )
@@ -467,7 +467,7 @@ def test_contentnode_tree_next__gt(self):
         "Skipping postgres as not as vulnerable to large queries and large insertions are less performant",
     )
     def test_contentnode_tree_more(self):
-        builder = ChannelBuilder(levels=2, num_children=30)
+        builder = ChannelBuilder(levels=2, num_children=17)
         builder.insert_into_default_db()
         content.ContentNode.objects.all().update(available=True)
         root = content.ContentNode.objects.get(id=builder.root_node["id"])