Repository: zulip/zulip

Issue:
inline meniton for users, groups and streams fail if they come after an invalid mention
This bug was reported here:
[https://chat.zulip.org/#narrow/stream/9-issues/topic/markdown.20mention.20bug](https://chat.zulip.org/#narrow/stream/9-issues/topic/markdown.20mention.20bug)

If there is an invalid user name written inside mention syntax then all the subsequent valid mentions are also treated as invalid. This is true for user groups, and streams mentions too.

**An example**
for source:
```
@**invalid** @**iago**  @*invalid user group* @*hamletcharacters* #**invalid streams>little URLS isn't loading quickly**  #**sales>little URLS isn't loading quickly**
```
output:
![image](https://user-images.githubusercontent.com/63504956/110537832-39f6bf80-8149-11eb-87a3-5e4d3a4c7f45.png)



Discussion:
@zulipbot claim

Resolving patch:
diff --git a/zerver/lib/markdown/__init__.py b/zerver/lib/markdown/__init__.py
--- a/zerver/lib/markdown/__init__.py
+++ b/zerver/lib/markdown/__init__.py
@@ -147,7 +147,15 @@ def normal_compile(pattern: str) -> Pattern[str]:
 
 @one_time
 def get_compiled_stream_link_regex() -> Pattern[str]:
-    return verbose_compile(STREAM_LINK_REGEX)
+    # Not using verbose_compile as it adds ^(.*?) and
+    # (.*?)$ which cause extra overhead of matching
+    # pattern which is not required.
+    # With new InlineProcessor these extra patterns
+    # are not required.
+    return re.compile(
+        STREAM_LINK_REGEX,
+        re.DOTALL | re.UNICODE | re.VERBOSE,
+    )
 
 
 STREAM_TOPIC_LINK_REGEX = r"""
@@ -162,7 +170,15 @@ def get_compiled_stream_link_regex() -> Pattern[str]:
 
 @one_time
 def get_compiled_stream_topic_link_regex() -> Pattern[str]:
-    return verbose_compile(STREAM_TOPIC_LINK_REGEX)
+    # Not using verbose_compile as it adds ^(.*?) and
+    # (.*?)$ which cause extra overhead of matching
+    # pattern which is not required.
+    # With new InlineProcessor these extra patterns
+    # are not required.
+    return re.compile(
+        STREAM_TOPIC_LINK_REGEX,
+        re.DOTALL | re.UNICODE | re.VERBOSE,
+    )
 
 
 LINK_REGEX: Optional[Pattern[str]] = None
@@ -1796,8 +1812,10 @@ def handleMatch(self, m: Match[str]) -> Union[Element, str]:
         )
 
 
-class UserMentionPattern(markdown.inlinepatterns.Pattern):
-    def handleMatch(self, m: Match[str]) -> Optional[Element]:
+class UserMentionPattern(markdown.inlinepatterns.InlineProcessor):
+    def handleMatch(  # type: ignore[override] # supertype incompatible with supersupertype
+        self, m: Match[str], data: str
+    ) -> Union[Tuple[None, None, None], Tuple[Element, int, int]]:
         match = m.group("match")
         silent = m.group("silent") == "_"
 
@@ -1806,7 +1824,7 @@ def handleMatch(self, m: Match[str]) -> Optional[Element]:
             if match.startswith("**") and match.endswith("**"):
                 name = match[2:-2]
             else:
-                return None
+                return None, None, None
 
             wildcard = mention.user_mention_matches_wildcard(name)
 
@@ -1827,7 +1845,7 @@ def handleMatch(self, m: Match[str]) -> Optional[Element]:
                 user_id = str(user["id"])
             else:
                 # Don't highlight @mentions that don't refer to a valid user
-                return None
+                return None, None, None
 
             el = Element("span")
             el.set("data-user-id", user_id)
@@ -1838,15 +1856,17 @@ def handleMatch(self, m: Match[str]) -> Optional[Element]:
                 el.set("class", "user-mention")
                 text = f"@{text}"
             el.text = markdown.util.AtomicString(text)
-            return el
-        return None
-
+            return el, m.start(), m.end()
+        return None, None, None
 
-class UserGroupMentionPattern(markdown.inlinepatterns.Pattern):
-    def handleMatch(self, m: Match[str]) -> Optional[Element]:
-        match = m.group(2)
 
+class UserGroupMentionPattern(markdown.inlinepatterns.InlineProcessor):
+    def handleMatch(  # type: ignore[override] # supertype incompatible with supersupertype
+        self, m: Match[str], data: str
+    ) -> Union[Tuple[None, None, None], Tuple[Element, int, int]]:
+        match = m.group(1)
         db_data = self.md.zulip_db_data
+
         if self.md.zulip_message and db_data is not None:
             name = extract_user_group(match)
             user_group = db_data["mention_data"].get_user_group(name)
@@ -1857,18 +1877,25 @@ def handleMatch(self, m: Match[str]) -> Optional[Element]:
             else:
                 # Don't highlight @-mentions that don't refer to a valid user
                 # group.
-                return None
+                return None, None, None
 
             el = Element("span")
             el.set("class", "user-group-mention")
             el.set("data-user-group-id", user_group_id)
             text = f"@{name}"
             el.text = markdown.util.AtomicString(text)
-            return el
-        return None
+            return el, m.start(), m.end()
+        return None, None, None
 
 
-class StreamPattern(CompiledPattern):
+class StreamPattern(markdown.inlinepatterns.InlineProcessor):
+    def __init__(self, compiled_re: Pattern[str], md: markdown.Markdown) -> None:
+        # This is similar to the superclass's small __init__ function,
+        # but we skip the compilation step and let the caller give us
+        # a compiled regex.
+        self.compiled_re = compiled_re
+        self.md = md
+
     def find_stream_by_name(self, name: str) -> Optional[Dict[str, Any]]:
         db_data = self.md.zulip_db_data
         if db_data is None:
@@ -1876,13 +1903,15 @@ def find_stream_by_name(self, name: str) -> Optional[Dict[str, Any]]:
         stream = db_data["stream_names"].get(name)
         return stream
 
-    def handleMatch(self, m: Match[str]) -> Optional[Element]:
+    def handleMatch(  # type: ignore[override] # supertype incompatible with supersupertype
+        self, m: Match[str], data: str
+    ) -> Union[Tuple[None, None, None], Tuple[Element, int, int]]:
         name = m.group("stream_name")
 
         if self.md.zulip_message:
             stream = self.find_stream_by_name(name)
             if stream is None:
-                return None
+                return None, None, None
             el = Element("a")
             el.set("class", "stream")
             el.set("data-stream-id", str(stream["id"]))
@@ -1895,11 +1924,18 @@ def handleMatch(self, m: Match[str]) -> Optional[Element]:
             el.set("href", f"/#narrow/stream/{stream_url}")
             text = f"#{name}"
             el.text = markdown.util.AtomicString(text)
-            return el
-        return None
+            return el, m.start(), m.end()
+        return None, None, None
 
 
-class StreamTopicPattern(CompiledPattern):
+class StreamTopicPattern(markdown.inlinepatterns.InlineProcessor):
+    def __init__(self, compiled_re: Pattern[str], md: markdown.Markdown) -> None:
+        # This is similar to the superclass's small __init__ function,
+        # but we skip the compilation step and let the caller give us
+        # a compiled regex.
+        self.compiled_re = compiled_re
+        self.md = md
+
     def find_stream_by_name(self, name: str) -> Optional[Dict[str, Any]]:
         db_data = self.md.zulip_db_data
         if db_data is None:
@@ -1907,14 +1943,16 @@ def find_stream_by_name(self, name: str) -> Optional[Dict[str, Any]]:
         stream = db_data["stream_names"].get(name)
         return stream
 
-    def handleMatch(self, m: Match[str]) -> Optional[Element]:
+    def handleMatch(  # type: ignore[override] # supertype incompatible with supersupertype
+        self, m: Match[str], data: str
+    ) -> Union[Tuple[None, None, None], Tuple[Element, int, int]]:
         stream_name = m.group("stream_name")
         topic_name = m.group("topic_name")
 
         if self.md.zulip_message:
             stream = self.find_stream_by_name(stream_name)
             if stream is None or topic_name is None:
-                return None
+                return None, None, None
             el = Element("a")
             el.set("class", "stream-topic")
             el.set("data-stream-id", str(stream["id"]))
@@ -1924,8 +1962,8 @@ def handleMatch(self, m: Match[str]) -> Optional[Element]:
             el.set("href", link)
             text = f"#{stream_name} > {topic_name}"
             el.text = markdown.util.AtomicString(text)
-            return el
-        return None
+            return el, m.start(), m.end()
+        return None, None, None
 
 
 def possible_linked_stream_names(content: str) -> Set[str]:


Test patch:
diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py
--- a/zerver/tests/test_markdown.py
+++ b/zerver/tests/test_markdown.py
@@ -1845,6 +1845,36 @@ def test_mention_silent(self) -> None:
         )
         self.assertEqual(msg.mentions_user_ids, set())
 
+    def test_mention_invalid_followed_by_valid(self) -> None:
+        sender_user_profile = self.example_user("othello")
+        user_profile = self.example_user("hamlet")
+        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
+        user_id = user_profile.id
+
+        content = "@**Invalid user** and @**King Hamlet**"
+        self.assertEqual(
+            render_markdown(msg, content),
+            '<p>@<strong>Invalid user</strong> and <span class="user-mention" '
+            f'data-user-id="{user_id}">'
+            "@King Hamlet</span></p>",
+        )
+        self.assertEqual(msg.mentions_user_ids, {user_profile.id})
+
+    def test_silent_mention_invalid_followed_by_valid(self) -> None:
+        sender_user_profile = self.example_user("othello")
+        user_profile = self.example_user("hamlet")
+        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
+        user_id = user_profile.id
+
+        content = "@_**Invalid user** and @_**King Hamlet**"
+        self.assertEqual(
+            render_markdown(msg, content),
+            '<p>@_<strong>Invalid user</strong> and <span class="user-mention silent" '
+            f'data-user-id="{user_id}">'
+            "King Hamlet</span></p>",
+        )
+        self.assertEqual(msg.mentions_user_ids, set())
+
     def test_possible_mentions(self) -> None:
         def assert_mentions(content: str, names: Set[str], has_wildcards: bool = False) -> None:
             self.assertEqual(possible_mentions(content), (names, has_wildcards))
@@ -2026,6 +2056,27 @@ def test_user_group_mention_single(self) -> None:
         self.assertEqual(msg.mentions_user_ids, {user_profile.id})
         self.assertEqual(msg.mentions_user_group_ids, {user_group.id})
 
+    def test_invalid_user_group_followed_by_valid_mention_single(self) -> None:
+        sender_user_profile = self.example_user("othello")
+        user_profile = self.example_user("hamlet")
+        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
+        user_id = user_profile.id
+        user_group = self.create_user_group_for_test("support")
+
+        content = "@**King Hamlet** @*Invalid user group* @*support*"
+        self.assertEqual(
+            render_markdown(msg, content),
+            '<p><span class="user-mention" '
+            f'data-user-id="{user_id}">'
+            "@King Hamlet</span> "
+            "@<em>Invalid user group</em> "
+            '<span class="user-group-mention" '
+            f'data-user-group-id="{user_group.id}">'
+            "@support</span></p>",
+        )
+        self.assertEqual(msg.mentions_user_ids, {user_profile.id})
+        self.assertEqual(msg.mentions_user_group_ids, {user_group.id})
+
     def test_user_group_mention_atomic_string(self) -> None:
         sender_user_profile = self.example_user("othello")
         realm = get_realm("zulip")
@@ -2155,6 +2206,18 @@ def test_stream_single(self) -> None:
             ),
         )
 
+    def test_invalid_stream_followed_by_valid_mention(self) -> None:
+        denmark = get_stream("Denmark", get_realm("zulip"))
+        sender_user_profile = self.example_user("othello")
+        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
+        content = "#**Invalid** and #**Denmark**"
+        self.assertEqual(
+            render_markdown(msg, content),
+            '<p>#<strong>Invalid</strong> and <a class="stream" data-stream-id="{d.id}" href="/#narrow/stream/{d.id}-Denmark">#{d.name}</a></p>'.format(
+                d=denmark,
+            ),
+        )
+
     def test_stream_multiple(self) -> None:
         sender_user_profile = self.example_user("othello")
         msg = Message(sender=sender_user_profile, sending_client=get_client("test"))