Repository: OkunaOrg/okuna-api

Issue:
Get timeline posts API is slow
It's taking up to 10 seconds to load.

Optimise the db queries, remove all looping if possible and on the frontend most likely bring caching of initial posts back.


Discussion:

Resolving patch:
diff --git a/openbook_auth/models.py b/openbook_auth/models.py
--- a/openbook_auth/models.py
+++ b/openbook_auth/models.py
@@ -1310,35 +1310,46 @@ def get_timeline_posts(self, lists_ids=None, circles_ids=None, max_id=None):
         :param username:
         :return:
         """
-        # If there's no circles or lists filters, add all posts
-        if circles_ids:
-            timeline_posts_query = Q(creator=self.pk, circles__id__in=circles_ids)
-        elif lists_ids:
-            timeline_posts_query = Q()
-        else:
-            timeline_posts_query = Q(creator_id=self.pk)
 
-        follows_related_query = self.follows.select_related('followed_user')
+        timeline_posts_query = Q()
+
+        if not circles_ids and not lists_ids:
+            # All of our posts
+            timeline_posts_query.add(Q(creator_id=self.pk), Q.OR)
+            # All of the posts of communities we're part of
+            timeline_posts_query.add(Q(community__memberships__user__id=self.pk), Q.OR)
+        elif circles_ids:
+            # All of our posts for the given circles
+            timeline_posts_query.add(Q(creator_id=self.pk, circles__id__in=circles_ids), Q.OR)
+
+        foreign_posts_query = Q()
+
+        # Posts from people we are following
+        following_users_posts = Q(creator__followers__user_id=self.pk)
 
-        # If there's lists filters, filter follows with it
         if lists_ids:
-            follows = follows_related_query.filter(lists__id__in=lists_ids)
+            following_users_posts.add(Q(creator__followers__lists__id__in=lists_ids), Q.AND)
+
+        foreign_posts_query.add(following_users_posts, Q.AND)
+
+        if circles_ids:
+            posts_visibility_query = Q(circles__id=self._get_world_circle_id(),
+                                       creator__connections__target_connection__circles__id__in=circles_ids)
         else:
-            follows = follows_related_query.all()
-
-        for follow in follows:
-            followed_user = follow.followed_user
-            if circles_ids:
-                # Check that the user belongs to the filtered circles
-                if self.is_connected_with_user_with_id_in_circles_with_ids(followed_user.pk, circles_ids):
-                    followed_user_posts_query = self._make_get_posts_query_for_user(followed_user, )
-                    timeline_posts_query.add(followed_user_posts_query, Q.OR)
-            else:
-                followed_user_posts_query = self._make_get_posts_query_for_user(followed_user, )
-                timeline_posts_query.add(followed_user_posts_query, Q.OR)
+            posts_visibility_query = Q(circles__id=self._get_world_circle_id())
 
-        if not circles_ids and not lists_ids:
-            timeline_posts_query.add(Q(community__memberships__user__id=self.pk), Q.OR)
+        # Posts from people we are connected with
+        connections_posts_query = Q(circles__connections__target_connection__user_id=self.pk,
+                                    circles__connections__target_connection__circles__isnull=False)
+
+        if circles_ids:
+            connections_posts_query.add(Q(circles__connections__target_connection__circles__id__in=circles_ids), Q.AND)
+
+        posts_visibility_query.add(connections_posts_query, Q.OR)
+
+        foreign_posts_query.add(posts_visibility_query, Q.AND)
+
+        timeline_posts_query.add(foreign_posts_query, Q.OR)
 
         if max_id:
             timeline_posts_query.add(Q(id__lt=max_id), Q.AND)
diff --git a/openbook_posts/models.py b/openbook_posts/models.py
--- a/openbook_posts/models.py
+++ b/openbook_posts/models.py
@@ -87,33 +87,20 @@ def create_post(cls, creator, circles_ids=None, community_name=None, image=None,
 
     @classmethod
     def get_emoji_counts_for_post_with_id(cls, post_id, emoji_id=None, reactor_id=None):
-        PostReaction = get_post_reaction_model()
         Emoji = get_emoji_model()
 
-        emoji_query = Q(reactions__post_id=post_id)
+        emoji_query = Q(reactions__post_id=post_id, )
 
         if emoji_id:
             emoji_query.add(Q(reactions__emoji_id=emoji_id), Q.AND)
 
-        emojis_reacted_with = Emoji.objects.filter(emoji_query).distinct()
-
-        emoji_counts = []
-
-        for emoji in emojis_reacted_with:
-            reaction_query = Q(post_id=post_id, emoji_id=emoji.pk)
-
-            if reactor_id:
-                reaction_query.add(Q(reactor_id=reactor_id), Q.AND)
-
-            emoji_count = PostReaction.objects.filter(reaction_query).count()
-            emoji_counts.append({
-                'emoji': emoji,
-                'count': emoji_count
-            })
+        if reactor_id:
+            emoji_query.add(Q(reactions__reactor_id=reactor_id), Q.AND)
 
-        emoji_counts.sort(key=lambda x: x['count'], reverse=True)
+        emojis = Emoji.objects.filter(emoji_query).annotate(Count('reactions')).distinct().order_by(
+            '-reactions__count').all()
 
-        return emoji_counts
+        return [{'emoji': emoji, 'count': emoji.reactions__count} for emoji in emojis]
 
     @classmethod
     def get_trending_posts(cls):


Test patch:
diff --git a/openbook_posts/tests/views/test_posts.py b/openbook_posts/tests/views/test_posts.py
--- a/openbook_posts/tests/views/test_posts.py
+++ b/openbook_posts/tests/views/test_posts.py
@@ -879,7 +879,7 @@ def test_retrieves_own_posts_of_own_filtered_circle(self):
 
         response_posts = json.loads(response.content)
 
-        self.assertEqual(len(response_posts), len(posts_ids))
+        self.assertEqual(len(posts_ids), len(response_posts))
 
         response_posts_ids = [post['id'] for post in response_posts]