Repository: python-discord/bot

Issue:
Strip spoiler tags for watchlist triggers
With the [recent Spoiler Tag addition](https://support.discordapp.com/hc/en-us/articles/360022320632-Spoiler-Tags-) to Discord, it's now possible to take memes to a whole new level. While the message filters are still operable, it's difficult to read the trigger messages in the mod log, particularly on mobile.

Spoilers are wrapped with `||` (e.g. `||text||`). Let's add a helper method to the modlog cog to strip these from the message content before they are sent.


Discussion:
I think we may want to extend this to other features as well. For instance, if some uses `||` in their username, e.g. `|| Hemlock's Cat ||`, then the username will also be spoilered in embeds posted by Python:

![2019-02-03_10-22-06](https://user-images.githubusercontent.com/33516116/52175029-b7e38200-279d-11e9-9726-b94cad563a39.png)

Now, as individual users, we can already prevent this by changing the spoiler option in our user settings (`User Settings > Text & Images > Show spoiler content`), but we may want to implement a more general approach that doesn't rely on individual settings. (Users on mobile can't reveal spoilers within embeds, so you can't reveal the username manually on mobile.)

One solution for the username problem is to wrap it in inline codeblock markup (\`). As shown in the embed above, the spoiler tags won't be rendered in that case.
~~Also we need to keep in mind that the alert system won't work anymore if someone prefixes a word with ||. The word filter splits by spaces so "||hello world||" would be split to `["||hello", "world||"]`. I don't think that the filters handle for this (I might be wrong) so is this potentially a way to evade filters?~~

See comment from sco1 below.

> On 3 Feb 2019, at 09:26, Sebastiaan Zeeff <notifications@github.com> wrote:
> 
> I think we may want to extend this to other features as well. For instance, if some uses || in their username, e.g. || Hemlock's Cat ||, then the username will also be spoilered in embeds posted by Python:
> 
> 
> 
> Now, as individual users, we can already prevent this by changing the spoiler option in our user settings (User Settings > Text & Images > Show spoiler content), but we may want to implement a more general approach that doesn't rely on individual settings. (Users on mobile can't reveal spoilers within embeds, so you can't reveal the username manually on mobile.)
> 
> One solution for the username problem is to wrap it in inline codeblock markup (`). As shown in the embed above, the spoiler tags won't be rendered in that case.
> 
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub, or mute the thread.

The word filter doesn't split by spaces. The regex is `\b{expression}\b`, and `|` is considered a word boundary. 

It should still be played around with a bit to see if it's exploitable but at first glance it doesn't seem like it is; the filter has already been successfully triggered for a spoilered watch word.
As an addendum, the filters should probably filter this out as well.

![Screenshot_20190705_145133_com discord](https://user-images.githubusercontent.com/204153/60726917-98ab7500-9f34-11e9-8ab9-d699d401faa2.png)


Resolving patch:
diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py
--- a/bot/cogs/filtering.py
+++ b/bot/cogs/filtering.py
@@ -6,6 +6,7 @@
 from dateutil.relativedelta import relativedelta
 from discord import Colour, DMChannel, Member, Message, TextChannel
 from discord.ext.commands import Cog
+from discord.utils import escape_markdown
 
 from bot.bot import Bot
 from bot.cogs.moderation import ModLog
@@ -27,6 +28,7 @@
     flags=re.IGNORECASE
 )
 
+SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL)
 URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE)
 ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]")
 
@@ -38,6 +40,14 @@
 ]
 
 
+def expand_spoilers(text: str) -> str:
+    """Return a string containing all interpretations of a spoilered message."""
+    split_text = SPOILER_RE.split(text)
+    return ''.join(
+        split_text[0::2] + split_text[1::2] + split_text
+    )
+
+
 class Filtering(Cog):
     """Filtering out invites, blacklisting domains, and warning us of certain regular expressions."""
 
@@ -186,8 +196,8 @@ async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> No
                             surroundings = match.string[max(match.start() - 10, 0): match.end() + 10]
                             message_content = (
                                 f"**Match:** '{match[0]}'\n"
-                                f"**Location:** '...{surroundings}...'\n"
-                                f"\n**Original Message:**\n{msg.content}"
+                                f"**Location:** '...{escape_markdown(surroundings)}...'\n"
+                                f"\n**Original Message:**\n{escape_markdown(msg.content)}"
                             )
                         else:  # Use content of discord Message
                             message_content = msg.content
@@ -244,6 +254,8 @@ async def _has_watchlist_words(text: str) -> Union[bool, re.Match]:
 
         Only matches words with boundaries before and after the expression.
         """
+        if SPOILER_RE.search(text):
+            text = expand_spoilers(text)
         for regex_pattern in WORD_WATCHLIST_PATTERNS:
             match = regex_pattern.search(text)
             if match:
diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py
--- a/bot/cogs/moderation/modlog.py
+++ b/bot/cogs/moderation/modlog.py
@@ -12,6 +12,7 @@
 from discord import Colour
 from discord.abc import GuildChannel
 from discord.ext.commands import Cog, Context
+from discord.utils import escape_markdown
 
 from bot.bot import Bot
 from bot.constants import Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, URLs
@@ -389,7 +390,8 @@ async def on_member_join(self, member: discord.Member) -> None:
         if member.guild.id != GuildConstant.id:
             return
 
-        message = f"{member} (`{member.id}`)"
+        member_str = escape_markdown(str(member))
+        message = f"{member_str} (`{member.id}`)"
         now = datetime.utcnow()
         difference = abs(relativedelta(now, member.created_at))
 
@@ -415,9 +417,10 @@ async def on_member_remove(self, member: discord.Member) -> None:
             self._ignored[Event.member_remove].remove(member.id)
             return
 
+        member_str = escape_markdown(str(member))
         await self.send_log_message(
             Icons.sign_out, Colours.soft_red,
-            "User left", f"{member} (`{member.id}`)",
+            "User left", f"{member_str} (`{member.id}`)",
             thumbnail=member.avatar_url_as(static_format="png"),
             channel_id=Channels.user_log
         )
@@ -432,9 +435,10 @@ async def on_member_unban(self, guild: discord.Guild, member: discord.User) -> N
             self._ignored[Event.member_unban].remove(member.id)
             return
 
+        member_str = escape_markdown(str(member))
         await self.send_log_message(
             Icons.user_unban, Colour.blurple(),
-            "User unbanned", f"{member} (`{member.id}`)",
+            "User unbanned", f"{member_str} (`{member.id}`)",
             thumbnail=member.avatar_url_as(static_format="png"),
             channel_id=Channels.mod_log
         )
@@ -526,7 +530,8 @@ async def on_member_update(self, before: discord.Member, after: discord.Member)
         for item in sorted(changes):
             message += f"{Emojis.bullet} {item}\n"
 
-        message = f"**{after}** (`{after.id}`)\n{message}"
+        member_str = escape_markdown(str(after))
+        message = f"**{member_str}** (`{after.id}`)\n{message}"
 
         await self.send_log_message(
             Icons.user_update, Colour.blurple(),
@@ -553,16 +558,17 @@ async def on_message_delete(self, message: discord.Message) -> None:
         if author.bot:
             return
 
+        author_str = escape_markdown(str(author))
         if channel.category:
             response = (
-                f"**Author:** {author} (`{author.id}`)\n"
+                f"**Author:** {author_str} (`{author.id}`)\n"
                 f"**Channel:** {channel.category}/#{channel.name} (`{channel.id}`)\n"
                 f"**Message ID:** `{message.id}`\n"
                 "\n"
             )
         else:
             response = (
-                f"**Author:** {author} (`{author.id}`)\n"
+                f"**Author:** {author_str} (`{author.id}`)\n"
                 f"**Channel:** #{channel.name} (`{channel.id}`)\n"
                 f"**Message ID:** `{message.id}`\n"
                 "\n"
@@ -649,6 +655,8 @@ async def on_message_edit(self, msg_before: discord.Message, msg_after: discord.
             return
 
         author = msg_before.author
+        author_str = escape_markdown(str(author))
+
         channel = msg_before.channel
         channel_name = f"{channel.category}/#{channel.name}" if channel.category else f"#{channel.name}"
 
@@ -680,7 +688,7 @@ async def on_message_edit(self, msg_before: discord.Message, msg_after: discord.
                 content_after.append(sub)
 
         response = (
-            f"**Author:** {author} (`{author.id}`)\n"
+            f"**Author:** {author_str} (`{author.id}`)\n"
             f"**Channel:** {channel_name} (`{channel.id}`)\n"
             f"**Message ID:** `{msg_before.id}`\n"
             "\n"
@@ -823,8 +831,9 @@ async def on_voice_state_update(
         if not changes:
             return
 
+        member_str = escape_markdown(str(member))
         message = "\n".join(f"{Emojis.bullet} {item}" for item in sorted(changes))
-        message = f"**{member}** (`{member.id}`)\n{message}"
+        message = f"**{member_str}** (`{member.id}`)\n{message}"
 
         await self.send_log_message(
             icon_url=icon,


Test patch: