Repository: zulip/zulip

Issue:
get_messages: Fix interaction between include_history and use_first_unread_anchor
@borisyankov helped me discover a subtle bug where `use_first_unread_anchor` and `include_history` don't interact correctly in a situation where the only messages on a topic were before the current user subscribed to that stream.  Basically if you set things up like this:
* Send a single message to a new topic "test" to a public stream Hamlet is not subscribed to ("Denmark" in this example).
* Subscribe hamlet to that stream
* Do the API query the mobile app does to try to view topic "test". (e.g. stream Denmark, topic "test", use_first_unread_anchor=True, anchor=0, num_before=25, num_after=25).  Then the API query will return 0 messages (!).  So effectively one sees messages in the webapp, but none appear on mobile.

Here's how this happens:
* Because `use_first_unread_anchor` is True, we end up going down the non-`include_history` code path in `get_messages_backend` and thus joining with UserMessage.  The user has 0 rows for that topic in UserMessage (since they weren't subscribed when the message was sent), so everything from there on returns no messages.

What we should actually be doing is in `get_messages_backend`:
* The main query should just be on `Message` (i.e. we should get rid of the `and not use_first_unread_anchor` part of the `include_history` line)
* In the `use_first_unread_anchor` code block, we should check `include_history`, and if True, we do the join against UserMessage in constructing `first_unread_query`, since we do need that join for getting the first unread message ID (just not for fetching the messages themselves, i.e. no changes to the main query).  This should require a moderately sized refactor.
* The rest of the code path correctly splits the logic based on whether `include_history` is True (i.e. does a UserMessage query to splice in the `flags` data as needed).

@showell since you were just in this code, would you be up for tackling this in the next day or so?  Ideally we'd have a fix in before the 1.8 release (aka Monday).



Discussion:
Hello @zulip/server-api members, this issue was labeled with the "area: api" label, so you may want to check it out!

<!-- areaLabelNotification -->

Resolving patch:
diff --git a/zerver/views/messages.py b/zerver/views/messages.py
--- a/zerver/views/messages.py
+++ b/zerver/views/messages.py
@@ -557,6 +557,33 @@ def exclude_muting_conditions(user_profile: UserProfile,
 
     return conditions
 
+def get_base_query_for_search(user_profile: UserProfile,
+                              need_message: bool,
+                              need_user_message: bool) -> Tuple[Query, ColumnElement]:
+    if need_message and need_user_message:
+        query = select([column("message_id"), column("flags")],
+                       column("user_profile_id") == literal(user_profile.id),
+                       join(table("zerver_usermessage"), table("zerver_message"),
+                            literal_column("zerver_usermessage.message_id") ==
+                            literal_column("zerver_message.id")))
+        inner_msg_id_col = column("message_id")
+        return (query, inner_msg_id_col)
+
+    if need_user_message:
+        query = select([column("message_id"), column("flags")],
+                       column("user_profile_id") == literal(user_profile.id),
+                       table("zerver_usermessage"))
+        inner_msg_id_col = column("message_id")
+        return (query, inner_msg_id_col)
+
+    else:
+        assert(need_message)
+        query = select([column("id").label("message_id")],
+                       None,
+                       table("zerver_message"))
+        inner_msg_id_col = literal_column("zerver_message.id")
+        return (query, inner_msg_id_col)
+
 def add_narrow_conditions(user_profile: UserProfile,
                           inner_msg_id_col: ColumnElement,
                           query: Query,
@@ -595,10 +622,32 @@ def add_narrow_conditions(user_profile: UserProfile,
     return (query, is_search)
 
 def find_first_unread_anchor(sa_conn: Any,
-                             inner_msg_id_col: ColumnElement,
                              user_profile: UserProfile,
-                             narrow: List[Dict[str, Any]],
-                             query: Query) -> int:
+                             narrow: List[Dict[str, Any]]) -> int:
+    # We always need UserMessage in our query, because it has the unread
+    # flag for the user.
+    need_user_message = True
+
+    # TODO:
+    # We err on the side of putting Message in our query, but there are
+    # probably situations that we don't need it.  We may eventually try
+    # to make add_narrow_conditions() and similar functions help us make
+    # possible optimizations.
+    need_message = True
+
+    query, inner_msg_id_col = get_base_query_for_search(
+        user_profile=user_profile,
+        need_message=need_message,
+        need_user_message=need_user_message,
+    )
+
+    query, is_search = add_narrow_conditions(
+        user_profile=user_profile,
+        inner_msg_id_col=inner_msg_id_col,
+        query=query,
+        narrow=narrow,
+    )
+
     condition = column("flags").op("&")(UserMessage.flags.read.mask) == 0
 
     # We exclude messages on muted topics when finding the first unread
@@ -643,30 +692,29 @@ def get_messages_backend(request: HttpRequest, user_profile: UserProfile,
                          apply_markdown: bool=REQ(validator=check_bool, default=True)) -> HttpResponse:
     include_history = ok_to_include_history(narrow, user_profile)
 
-    if include_history and not use_first_unread_anchor:
+    if include_history:
         # The initial query in this case doesn't use `zerver_usermessage`,
         # and isn't yet limited to messages the user is entitled to see!
         #
         # This is OK only because we've made sure this is a narrow that
         # will cause us to limit the query appropriately later.
         # See `ok_to_include_history` for details.
-        query = select([column("id").label("message_id")], None, table("zerver_message"))
-        inner_msg_id_col = literal_column("zerver_message.id")
-    elif narrow is None and not use_first_unread_anchor:
-        # This is limited to messages the user received, as recorded in `zerver_usermessage`.
-        query = select([column("message_id"), column("flags")],
-                       column("user_profile_id") == literal(user_profile.id),
-                       table("zerver_usermessage"))
-        inner_msg_id_col = column("message_id")
+        need_message = True
+        need_user_message = False
+    elif narrow is None:
+        # We need to limit to messages the user has received, but we don't actually
+        # need any fields from Message
+        need_message = False
+        need_user_message = True
     else:
-        # This is limited to messages the user received, as recorded in `zerver_usermessage`.
-        # TODO: Don't do this join if we're not doing a search
-        query = select([column("message_id"), column("flags")],
-                       column("user_profile_id") == literal(user_profile.id),
-                       join(table("zerver_usermessage"), table("zerver_message"),
-                            literal_column("zerver_usermessage.message_id") ==
-                            literal_column("zerver_message.id")))
-        inner_msg_id_col = column("message_id")
+        need_message = True
+        need_user_message = True
+
+    query, inner_msg_id_col = get_base_query_for_search(
+        user_profile=user_profile,
+        need_message=need_message,
+        need_user_message=need_user_message,
+    )
 
     query, is_search = add_narrow_conditions(
         user_profile=user_profile,
@@ -690,10 +738,8 @@ def get_messages_backend(request: HttpRequest, user_profile: UserProfile,
     if use_first_unread_anchor:
         anchor = find_first_unread_anchor(
             sa_conn,
-            inner_msg_id_col,
             user_profile,
             narrow,
-            query
         )
 
     anchored_to_left = (anchor == 0)


Test patch:
diff --git a/zerver/tests/test_narrow.py b/zerver/tests/test_narrow.py
--- a/zerver/tests/test_narrow.py
+++ b/zerver/tests/test_narrow.py
@@ -1736,7 +1736,7 @@ def test_find_first_unread_anchor(self) -> None:
         self.make_stream('England')
 
         # Send a few messages that Hamlet won't have UserMessage rows for.
-        self.send_stream_message(cordelia.email, 'England')
+        unsub_message_id = self.send_stream_message(cordelia.email, 'England')
         self.send_personal_message(cordelia.email, othello.email)
 
         self.subscribe(hamlet, 'England')
@@ -1747,37 +1747,50 @@ def test_find_first_unread_anchor(self) -> None:
         set_topic_mutes(hamlet, muted_topics)
 
         # send a muted message
-        self.send_stream_message(cordelia.email, 'England', topic_name='muted')
+        muted_message_id = self.send_stream_message(cordelia.email, 'England', topic_name='muted')
 
         # finally send Hamlet a "normal" message
         first_message_id = self.send_stream_message(cordelia.email, 'England')
 
         # send a few more messages
-        self.send_stream_message(cordelia.email, 'England')
+        extra_message_id = self.send_stream_message(cordelia.email, 'England')
         self.send_personal_message(cordelia.email, hamlet.email)
 
         sa_conn = get_sqlalchemy_connection()
 
         user_profile = hamlet
 
-        # TODO: Make it so that find_first_unread_anchor() does not require
-        #       the incoming query to join to zerver_usermessage.
-        query = select([column("message_id"), column("flags")],
-                       column("user_profile_id") == literal(user_profile.id),
-                       join(table("zerver_usermessage"), table("zerver_message"),
-                            literal_column("zerver_usermessage.message_id") ==
-                            literal_column("zerver_message.id")))
-        inner_msg_id_col = column("message_id")
-
         anchor = find_first_unread_anchor(
             sa_conn=sa_conn,
-            inner_msg_id_col=inner_msg_id_col,
             user_profile=user_profile,
             narrow=[],
-            query=query,
         )
         self.assertEqual(anchor, first_message_id)
 
+        # With the same data setup, we now want to test that a reasonable
+        # search still gets the first message sent to Hamlet (before he
+        # subscribed) and other recent messages to the stream.
+        query_params = dict(
+            use_first_unread_anchor='true',
+            anchor=0,
+            num_before=10,
+            num_after=10,
+            narrow='[["stream", "England"]]'
+        )
+        request = POSTRequestMock(query_params, user_profile)
+
+        payload = get_messages_backend(request, user_profile)
+        result = ujson.loads(payload.content)
+        self.assertEqual(result['anchor'], first_message_id)
+        self.assertEqual(result['found_newest'], True)
+        self.assertEqual(result['found_oldest'], True)
+
+        messages = result['messages']
+        self.assertEqual(
+            {msg['id'] for msg in messages},
+            {unsub_message_id, muted_message_id, first_message_id, extra_message_id}
+        )
+
     def test_use_first_unread_anchor_with_some_unread_messages(self) -> None:
         user_profile = self.example_user('hamlet')
 
@@ -1958,7 +1971,7 @@ def test_use_first_unread_anchor_with_muted_topics(self) -> None:
         # the `message_id = LARGER_THAN_MAX_MESSAGE_ID` hack.
         queries = [q for q in all_queries if '/* get_messages */' in q['sql']]
         self.assertEqual(len(queries), 1)
-        self.assertIn('AND message_id = %d' % (LARGER_THAN_MAX_MESSAGE_ID,),
+        self.assertIn('AND zerver_message.id = %d' % (LARGER_THAN_MAX_MESSAGE_ID,),
                       queries[0]['sql'])
 
     def test_exclude_muting_conditions(self) -> None: