Repository: sopel-irc/sopel

Issue:
`trigger` should have an easy way to get plain-text (no formatting)
Note that this is a distinct feature from #462, which requests the ability to match rules on the plain-text version of each line.

My notes from back when I wrote this idea down proposed `trigger.plain` as the home for this data. We could even parallel `trigger.group` with `trigger.plain.group`… Bikeshedding welcome, I guess. This isn't going into the next release (7.0), so there's time to consider and discuss.


Discussion:
I assume it makes sense, but right now I'd like an example of what you mean by "no formatting".
> an example of what you mean by "no formatting"

Original:

    \x0301,01Spoiler text\x03 with \x02BOLD\x02 assumptions!

No formatting:

    Spoiler text with BOLD assumptions!

This obviously doesn't show _all_ formatting types, but at minimum the "plain" representation of the trigger's text should remove all known formatting characters and (for colors) their values. Perhaps it should also remove all non-printable characters just to be safe against future formatting extensions?
Might be best implemented as a new function in `tools`, which could be used in #462 also. For reference, we'd do [something like ZNC's `CString::StripControls()` method](https://github.com/znc/znc/blob/5fd7584e7c4efadc5e1e7153340c18810f91e2a2/src/ZNCString.cpp#L1393-L1440) (`StripControls_n` implementation shown, because that's where the logic is).

Resolving patch:
diff --git a/sopel/formatting.py b/sopel/formatting.py
--- a/sopel/formatting.py
+++ b/sopel/formatting.py
@@ -8,6 +8,7 @@
 # Licensed under the Eiffel Forum License 2.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import re
 import string
 import sys
 
@@ -32,6 +33,7 @@
     'strikethrough',
     'monospace',
     'reverse',
+    'plain',
     # utility class
     'colors',
 ]
@@ -60,6 +62,58 @@
 CONTROL_REVERSE = '\x16'
 """The control code to start or end reverse-color formatting."""
 
+CONTROL_NON_PRINTING = [
+    '\x00',
+    '\x01',
+    '\x02',  # CONTROL_BOLD
+    '\x03',  # CONTROL_COLOR
+    '\x04',  # CONTROL_HEX_COLOR
+    '\x05',
+    '\x06',
+    '\x07',
+    '\x08',
+    '\x09',
+    '\x0a',
+    '\x0b',
+    '\x0c',
+    '\x0d',
+    '\x0e',
+    '\x0f',  # CONTROL_NORMAL
+    '\x10',
+    '\x11',  # CONTROL_MONOSPACE
+    '\x12',
+    '\x13',
+    '\x14',
+    '\x15',
+    '\x16',  # CONTROL_REVERSE
+    '\x17',
+    '\x18',
+    '\x19',
+    '\x1a',
+    '\x1b',
+    '\x1c',
+    '\x1d',  # CONTROL_ITALIC
+    '\x1e',  # CONTROL_STRIKETHROUGH
+    '\x1f',  # CONTROL_UNDERLINE
+    '\x7f',
+]
+
+# Regex to detect Control Pattern
+COLOR_PATTERN = re.escape(CONTROL_COLOR) + r'((\d{1,2},\d{2})|\d{2})?'
+HEX_COLOR_PATTERN = '%s(%s)?' % (
+    re.escape(CONTROL_HEX_COLOR),
+    '|'.join([
+        '(' + ','.join([r'[a-fA-F0-9]{6}', r'[a-fA-F0-9]{6}']) + ')',
+        r'[a-fA-F0-9]{6}'
+    ])
+)
+
+PLAIN_PATTERN = '|'.join([
+    '(' + COLOR_PATTERN + ')',
+    '(' + HEX_COLOR_PATTERN + ')',
+])
+PLAIN_REGEX = re.compile(PLAIN_PATTERN)
+
 
 # TODO when we can move to 3.3+ completely, make this an Enum.
 class colors:
@@ -247,3 +301,14 @@ def reverse(text):
         that understand it (e.g. mIRC) can be unpredictable. Use it carefully.
     """
     return ''.join([CONTROL_REVERSE, text, CONTROL_REVERSE])
+
+
+def plain(text):
+    """Return the text without any IRC formatting.
+
+    :param str text: text with potential IRC formatting control code(s)
+    :rtype: str
+    """
+    if '\x03' in text or '\x04' in text:
+        text = PLAIN_REGEX.sub('', text)
+    return ''.join(c for c in text if ord(c) >= 0x20 and c != '\x7F')
diff --git a/sopel/trigger.py b/sopel/trigger.py
--- a/sopel/trigger.py
+++ b/sopel/trigger.py
@@ -6,7 +6,7 @@
 import re
 import sys
 
-from sopel import tools
+from sopel import formatting, tools
 from sopel.tools import web
 
 
@@ -90,6 +90,10 @@ class PreTrigger(object):
         This is for ``PRIVMSG`` and ``NOTICE`` messages only. For other
         messages, this will be an empty ``tuple``.
 
+    .. py:attribute:: plain
+
+        The last argument of the IRC command with control codes stripped.
+
     .. py:attribute:: time
 
         The time when the message was received.
@@ -109,6 +113,7 @@ def __init__(self, own_nick, line, url_schemes=None):
         line = line.strip('\r\n')
         self.line = line
         self.urls = tuple()
+        self.plain = ''
 
         # Break off IRCv3 message tags, if present
         self.tags = {}
@@ -190,6 +195,10 @@ def __init__(self, own_nick, line, url_schemes=None):
             # Account is the second arg `...JOIN #Sopel account :realname`
             self.tags['account'] = self.args[1]
 
+        # get plain text message
+        if self.args:
+            self.plain = formatting.plain(self.args[-1])
+
 
 class Trigger(unicode):
     """A line from the server, which has matched a callable's rules.
@@ -338,6 +347,13 @@ class Trigger(unicode):
     URLs are listed only for ``PRIVMSG`` or a ``NOTICE``, otherwise this is
     an empty tuple.
     """
+    plain = property(lambda self: self._pretrigger.plain)
+    """The text without formatting control codes.
+
+    :type: str
+
+    This is the text of the trigger object without formatting control codes.
+    """
     tags = property(lambda self: self._pretrigger.tags)
     """A map of the IRCv3 message tags on the message.
 


Test patch:
diff --git a/test/test_formatting.py b/test/test_formatting.py
--- a/test/test_formatting.py
+++ b/test/test_formatting.py
@@ -4,7 +4,20 @@
 
 import pytest
 
-from sopel.formatting import bold, color, colors, hex_color, italic, monospace, reverse, strikethrough, underline
+from sopel.formatting import (
+    bold,
+    color,
+    colors,
+    CONTROL_NON_PRINTING,
+    CONTROL_NORMAL,
+    hex_color,
+    italic,
+    monospace,
+    plain,
+    reverse,
+    strikethrough,
+    underline,
+)
 
 
 def test_color():
@@ -54,3 +67,76 @@ def test_monospace():
 def test_reverse():
     text = 'Hello World'
     assert reverse(text) == '\x16' + text + '\x16'
+
+
+def test_plain_color():
+    text = 'some text'
+    assert plain(color(text, colors.PINK)) == text
+    assert plain(color(text, colors.PINK, colors.TEAL)) == text
+
+    tpl = 'b %s a'
+    expected = tpl % text
+    assert plain(tpl % color(text, colors.PINK)) == expected
+    assert plain(tpl % color(text, colors.PINK, colors.TEAL)) == expected
+
+
+def test_plain_hex_color():
+    text = 'some text'
+    assert plain(hex_color(text, 'ff0098')) == text
+    assert plain(hex_color(text, 'ff0098', '00b571')) == text
+
+    tpl = 'b %s a'
+    expected = tpl % text
+    assert plain(tpl % hex_color(text, 'ff0098')) == expected
+    assert plain(tpl % hex_color(text, 'ff0098', '00b571')) == expected
+
+
+def test_plain_bold():
+    text = 'some text'
+    assert plain(bold(text)) == text
+
+
+def test_plain_italic():
+    text = 'some text'
+    assert plain(italic(text)) == text
+
+
+def test_plain_underline():
+    text = 'some text'
+    assert plain(underline(text)) == text
+
+
+def test_plain_strikethrough():
+    text = 'some text'
+    assert plain(strikethrough(text)) == text
+
+
+def test_plain_monospace():
+    text = 'some text'
+    assert plain(monospace(text)) == text
+
+
+def test_plain_reverse():
+    text = 'some text'
+    assert plain(reverse(text)) == text
+
+
+def test_plain_reset():
+    text = 'some%s text' % CONTROL_NORMAL
+    assert plain(text) == 'some text'
+
+
+@pytest.mark.parametrize('code', CONTROL_NON_PRINTING)
+def test_plain_non_printing(code):
+    text = 'some%s text' % code
+    assert plain(text) == 'some text'
+
+
+def test_plain_unknown():
+    text = 'some \x99text'
+    assert plain(text) == text, 'An unknown control code must not be stripped'
+
+
+def test_plain_emoji():
+    text = 'some emoji 💪 in here'
+    assert plain(text) == text
diff --git a/test/test_trigger.py b/test/test_trigger.py
--- a/test/test_trigger.py
+++ b/test/test_trigger.py
@@ -41,6 +41,7 @@ def test_basic_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['#Sopel', 'Hello, world']
     assert pretrigger.text == 'Hello, world'
+    assert pretrigger.plain == 'Hello, world'
     assert pretrigger.event == 'PRIVMSG'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -56,6 +57,7 @@ def test_pm_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['Sopel', 'Hello, world']
     assert pretrigger.text == 'Hello, world'
+    assert pretrigger.plain == 'Hello, world'
     assert pretrigger.event == 'PRIVMSG'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -71,6 +73,7 @@ def test_quit_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['quit message text']
     assert pretrigger.text == 'quit message text'
+    assert pretrigger.plain == 'quit message text'
     assert pretrigger.event == 'QUIT'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -86,6 +89,7 @@ def test_join_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['#Sopel']
     assert pretrigger.text == '#Sopel'
+    assert pretrigger.plain == '#Sopel'
     assert pretrigger.event == 'JOIN'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -103,6 +107,7 @@ def test_tags_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['#Sopel', 'Hello, world']
     assert pretrigger.text == 'Hello, world'
+    assert pretrigger.plain == 'Hello, world'
     assert pretrigger.event == 'PRIVMSG'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -118,6 +123,7 @@ def test_intents_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['#Sopel', 'Hello, world']
     assert pretrigger.text == 'Hello, world'
+    assert pretrigger.plain == 'Hello, world'
     assert pretrigger.event == 'PRIVMSG'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -133,6 +139,7 @@ def test_unusual_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == []
     assert pretrigger.text == 'PING'
+    assert pretrigger.plain == ''
     assert pretrigger.event == 'PING'
 
 
@@ -144,6 +151,7 @@ def test_ctcp_intent_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['Sopel', '']
     assert pretrigger.text == '\x01VERSION\x01'
+    assert pretrigger.plain == ''
     assert pretrigger.event == 'PRIVMSG'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -159,6 +167,7 @@ def test_ctcp_data_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['Sopel', '1123321']
     assert pretrigger.text == '\x01PING 1123321\x01'
+    assert pretrigger.plain == '1123321'
     assert pretrigger.event == 'PRIVMSG'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -166,6 +175,22 @@ def test_ctcp_data_pretrigger(nick):
     assert pretrigger.sender == Identifier('Foo')
 
 
+def test_ctcp_action_pretrigger(nick):
+    line = ':Foo!foo@example.com PRIVMSG #Sopel :\x01ACTION Hello, world\x01'
+    pretrigger = PreTrigger(nick, line)
+    assert pretrigger.tags == {'intent': 'ACTION'}
+    assert pretrigger.hostmask == 'Foo!foo@example.com'
+    assert pretrigger.line == line
+    assert pretrigger.args == ['#Sopel', 'Hello, world']
+    assert pretrigger.text == '\x01ACTION Hello, world\x01'
+    assert pretrigger.plain == 'Hello, world'
+    assert pretrigger.event == 'PRIVMSG'
+    assert pretrigger.nick == Identifier('Foo')
+    assert pretrigger.user == 'foo'
+    assert pretrigger.host == 'example.com'
+    assert pretrigger.sender == '#Sopel'
+
+
 def test_ircv3_extended_join_pretrigger(nick):
     line = ':Foo!foo@example.com JOIN #Sopel bar :Real Name'
     pretrigger = PreTrigger(nick, line)
@@ -174,6 +199,7 @@ def test_ircv3_extended_join_pretrigger(nick):
     assert pretrigger.line == line
     assert pretrigger.args == ['#Sopel', 'bar', 'Real Name']
     assert pretrigger.text == 'Real Name'
+    assert pretrigger.plain == 'Real Name'
     assert pretrigger.event == 'JOIN'
     assert pretrigger.nick == Identifier('Foo')
     assert pretrigger.user == 'foo'
@@ -202,6 +228,7 @@ def test_ircv3_extended_join_trigger(nick, configfactory):
     assert trigger.group == fakematch.group
     assert trigger.groups == fakematch.groups
     assert trigger.args == ['#Sopel', 'bar', 'Real Name']
+    assert trigger.plain == 'Real Name'
     assert trigger.account == 'bar'
     assert trigger.tags == {'account': 'bar'}
     assert trigger.owner is True
@@ -229,6 +256,7 @@ def test_ircv3_intents_trigger(nick, configfactory):
     assert trigger.groups == fakematch.groups
     assert trigger.groupdict == fakematch.groupdict
     assert trigger.args == ['#Sopel', 'Hello, world']
+    assert trigger.plain == 'Hello, world'
     assert trigger.tags == {'intent': 'ACTION'}
     assert trigger.account is None
     assert trigger.admin is True