Repository: python-discord/bot

Issue:
Watchlist Tokens & Watchlist Words merge and bug
Tokens and Words watchlist filters really should be just be combined I think. The name of `watch_tokens` being shown in mod alerts makes one think of bot tokens, not a character/word list. If there's any particular reason why it was done like this in the beginning though, feel free to let us know.

There's also a bug where if the `retarded` expression gets triggered but it matches one of it's exceptions such as "sorry for all the retarded questions", it'll return `False` right there and then, failing to process any other expressions after it. So technically "what's with the retarded question you dumb cunt" may not trigger the filter since `(re+)tar+(d+|t+)(ed)? questions?` is an exception and `cunts*` comes after it in the list of expressions to be processed.

This can be resolved by replacing `return False` with `continue`:
```diff
                    # stuff like "sorry for all the retarded questions"
                    elif re.search(RETARDED_QUESTIONS_RE, text, re.IGNORECASE):
-                       return False
+                       continue
```

As a side note, this is going to make for an interesting github history check if someone looks it over in the future.


Discussion:
re tokens and words, I copied it off Rowboat, basically. But we could just use words and manually stick word boundary signifiers in the expressions I guess. 
Now that we filter for all usages of the r-word, can we close this issue? Or is this bug still alive?
the second part of the bug is probably fixed now, yes, but the first point about how we shouldn't have tokens and words at all but just make it one thing is still very much relevant.
How about leaving the regex lists separate in the config but combining the filters in the code? Would keep the convenience of not requiring manual word boundaries.

Resolving patch:
diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py
--- a/bot/cogs/filtering.py
+++ b/bot/cogs/filtering.py
@@ -38,6 +38,7 @@
 TOKEN_WATCHLIST_PATTERNS = [
     re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist
 ]
+WATCHLIST_PATTERNS = WORD_WATCHLIST_PATTERNS + TOKEN_WATCHLIST_PATTERNS
 
 
 def expand_spoilers(text: str) -> str:
@@ -88,24 +89,18 @@ def __init__(self, bot: Bot):
                     f"Your URL has been removed because it matched a blacklisted domain. {staff_mistake_str}"
                 )
             },
+            "watch_regex": {
+                "enabled": Filter.watch_regex,
+                "function": self._has_watch_regex_match,
+                "type": "watchlist",
+                "content_only": True,
+            },
             "watch_rich_embeds": {
                 "enabled": Filter.watch_rich_embeds,
                 "function": self._has_rich_embed,
                 "type": "watchlist",
                 "content_only": False,
             },
-            "watch_words": {
-                "enabled": Filter.watch_words,
-                "function": self._has_watchlist_words,
-                "type": "watchlist",
-                "content_only": True,
-            },
-            "watch_tokens": {
-                "enabled": Filter.watch_tokens,
-                "function": self._has_watchlist_tokens,
-                "type": "watchlist",
-                "content_only": True,
-            },
         }
 
     @property
@@ -191,8 +186,8 @@ async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> No
                         else:
                             channel_str = f"in {msg.channel.mention}"
 
-                        # Word and match stats for watch_words and watch_tokens
-                        if filter_name in ("watch_words", "watch_tokens"):
+                        # Word and match stats for watch_regex
+                        if filter_name == "watch_regex":
                             surroundings = match.string[max(match.start() - 10, 0): match.end() + 10]
                             message_content = (
                                 f"**Match:** '{match[0]}'\n"
@@ -248,37 +243,24 @@ async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> No
                         break  # We don't want multiple filters to trigger
 
     @staticmethod
-    async def _has_watchlist_words(text: str) -> Union[bool, re.Match]:
+    async def _has_watch_regex_match(text: str) -> Union[bool, re.Match]:
         """
-        Returns True if the text contains one of the regular expressions from the word_watchlist in our filter config.
+        Return True if `text` matches any regex from `word_watchlist` or `token_watchlist` configs.
 
-        Only matches words with boundaries before and after the expression.
+        `word_watchlist`'s patterns are placed between word boundaries while `token_watchlist` is
+        matched as-is. Spoilers are expanded, if any, and URLs are ignored.
         """
         if SPOILER_RE.search(text):
             text = expand_spoilers(text)
-        for regex_pattern in WORD_WATCHLIST_PATTERNS:
-            match = regex_pattern.search(text)
-            if match:
-                return match  # match objects always have a boolean value of True
 
-        return False
-
-    @staticmethod
-    async def _has_watchlist_tokens(text: str) -> Union[bool, re.Match]:
-        """
-        Returns True if the text contains one of the regular expressions from the token_watchlist in our filter config.
+        # Make sure it's not a URL
+        if URL_RE.search(text):
+            return False
 
-        This will match the expression even if it does not have boundaries before and after.
-        """
-        for regex_pattern in TOKEN_WATCHLIST_PATTERNS:
-            match = regex_pattern.search(text)
+        for pattern in WATCHLIST_PATTERNS:
+            match = pattern.search(text)
             if match:
-
-                # Make sure it's not a URL
-                if not URL_RE.search(text):
-                    return match  # match objects always have a boolean value of True
-
-        return False
+                return match
 
     @staticmethod
     async def _has_urls(text: str) -> bool:
diff --git a/bot/constants.py b/bot/constants.py
--- a/bot/constants.py
+++ b/bot/constants.py
@@ -206,9 +206,8 @@ class Filter(metaclass=YAMLGetter):
     filter_zalgo: bool
     filter_invites: bool
     filter_domains: bool
+    watch_regex: bool
     watch_rich_embeds: bool
-    watch_words: bool
-    watch_tokens: bool
 
     # Notifications are not expected for "watchlist" type filters
     notify_user_zalgo: bool


Test patch: