Repository: sopel-irc/sopel

Issue:
tools: command-regexp functions shouldn't be public
The major rule-system rework in #1873 is a great excuse to take the `get_*_regexp()` and `get_*_pattern()` functions from `sopel.tools` and either shove them into someplace like `sopel.plugins.rules.utils` or encapsulate the pattern generation into the different rule object classes.

@Exirel let's come back to this after #1873 is finished.

**Full to-deprecate list (for 7.1):**
- [x] `compile_rule()` [#2027]
- [x] `get_command_regexp()` [#1944]
- [x] `get_command_pattern()` [#1944]
- [x] `get_nickname_command_regexp()` [#1944]
- [x] `get_nickname_command_pattern()` [#1944]
- [x] `get_action_command_regexp()` [#1944]
- [x] `get_action_command_pattern()` [#1944]

**Full to-delete list (for 8.0):**
- [x] `compile_rule()` [#2141]
- [x] `get_command_regexp()` [#2141]
- [x] `get_command_pattern()` [#2141]
- [x] `get_nickname_command_regexp()` [#2141]
- [x] `get_nickname_command_pattern()` [#2141]
- [x] `get_action_command_regexp()` [#2141]
- [x] `get_action_command_pattern()` [#2141]


Discussion:

Resolving patch:
diff --git a/sopel/plugins/rules.py b/sopel/plugins/rules.py
--- a/sopel/plugins/rules.py
+++ b/sopel/plugins/rules.py
@@ -56,6 +56,7 @@
     'URLCallback',
 ]
 
+
 LOGGER = logging.getLogger(__name__)
 
 IGNORE_RATE_LIMIT = 1  # equivalent to sopel.plugin.NOLIMIT
@@ -1017,6 +1018,24 @@ class Command(NamedRuleMixin, Rule):
         <Bot> You just invoked the command 'dummy' (as 'dummy-alias')
 
     """
+    # This regexp matches equivalently and produces the same
+    # groups 1 and 2 as the old regexp: r'^%s(%s)(?: +(.*))?$'
+    # The only differences should be handling all whitespace
+    # like spaces and the addition of groups 3-6.
+    PATTERN_TEMPLATE = r"""
+        (?:{prefix})({command}) # Command as group 1.
+        (?:\s+              # Whitespace to end command.
+        (                   # Rest of the line as group 2.
+        (?:(\S+))?          # Parameters 1-4 as groups 3-6.
+        (?:\s+(\S+))?
+        (?:\s+(\S+))?
+        (?:\s+(\S+))?
+        .*                  # Accept anything after the parameters.
+                            # Leave it up to the plugin to parse the line.
+        ))?                 # Group 2 must be None, if there are no parameters.
+        $                   # EoL, so there are no partial matches.
+    """
+
     @classmethod
     def from_callable(cls, settings, handler):
         prefix = settings.core.prefix
@@ -1093,13 +1112,18 @@ def get_rule_regex(self):
         * the rule's name (escaped for regex if needed),
         * all of its aliases (escaped for regex if needed),
 
-        and everything is then given to :func:`sopel.tools.get_command_regexp`,
-        which creates a compiled regex to return.
+        to create a compiled regex to return.
         """
         name = [self.escape_name(self._name)]
         aliases = [self.escape_name(alias) for alias in self._aliases]
         pattern = r'|'.join(name + aliases)
-        return tools.get_command_regexp(self._prefix, pattern)
+
+        # Escape all whitespace with a single backslash.
+        # This ensures that regexp in the prefix is treated as it was before
+        # the actual regexp was changed to use the verbose syntax.
+        prefix = re.sub(r"(\s)", r"\\\1", self._prefix)
+        pattern = self.PATTERN_TEMPLATE.format(prefix=prefix, command=pattern)
+        return re.compile(pattern, re.IGNORECASE | re.VERBOSE)
 
 
 class NickCommand(NamedRuleMixin, Rule):
@@ -1124,6 +1148,22 @@ class NickCommand(NamedRuleMixin, Rule):
 
     Apart from that, it behaves exactly like a :class:`generic rule <Rule>`.
     """
+    PATTERN_TEMPLATE = r"""
+        ^
+        $nickname[:,]? # Nickname.
+        \s+({command}) # Command as group 1.
+        (?:\s+         # Whitespace to end command.
+        (              # Rest of the line as group 2.
+        (?:(\S+))?     # Parameters 1-4 as groups 3-6.
+        (?:\s+(\S+))?
+        (?:\s+(\S+))?
+        (?:\s+(\S+))?
+        .*             # Accept anything after the parameters. Leave it up to
+                       # the plugin to parse the line.
+        ))?            # Group 1 must be None, if there are no parameters.
+        $              # EoL, so there are no partial matches.
+    """
+
     @classmethod
     def from_callable(cls, settings, handler):
         nick = settings.core.nick
@@ -1196,15 +1236,15 @@ def get_rule_regex(self):
         * the rule's name (escaped for regex),
         * all of its aliases (escaped for regex),
 
-        and everything is then given to
-        :func:`sopel.tools.get_nickname_command_regexp`, which creates a
-        compiled regex to return.
+        to create a compiled regex to return.
         """
         name = [self.escape_name(self._name)]
         aliases = [self.escape_name(alias) for alias in self._aliases]
         pattern = r'|'.join(name + aliases)
-        return tools.get_nickname_command_regexp(
-            self._nick, pattern, self._nick_aliases)
+        return tools.compile_rule(
+            self._nick,
+            self.PATTERN_TEMPLATE.format(command=pattern),
+            self._nick_aliases)
 
 
 class ActionCommand(NamedRuleMixin, Rule):
@@ -1226,6 +1266,21 @@ class ActionCommand(NamedRuleMixin, Rule):
     Apart from that, it behaves exactly like a :class:`generic rule <Rule>`.
     """
     INTENT_REGEX = re.compile(r'ACTION', re.IGNORECASE)
+    PATTERN_TEMPLATE = r"""
+        ({command})         # Command as group 1.
+        (?:\s+              # Whitespace to end command.
+        (                   # Rest of the line as group 2.
+        (?:(\S+))?          # Parameters 1-4 as groups 3-6.
+        (?:\s+(\S+))?
+        (?:\s+(\S+))?
+        (?:\s+(\S+))?
+        .*                  # Accept anything after the parameters.
+                            # Leave it up to the plugin to parse
+                            # the line.
+        ))?                 # Group 2 must be None, if there are no
+                            # parameters.
+        $                   # EoL, so there are no partial matches.
+    """
 
     @classmethod
     def from_callable(cls, settings, handler):
@@ -1267,14 +1322,13 @@ def get_rule_regex(self):
         * the rule's name (escaped for regex if needed),
         * all of its aliases (escaped for regex if needed),
 
-        and everything is then given to
-        :func:`sopel.tools.get_action_command_regexp`, which creates a compiled
-        regex to return.
+        to create a compiled regex to return.
         """
         name = [self.escape_name(self._name)]
         aliases = [self.escape_name(alias) for alias in self._aliases]
         pattern = r'|'.join(name + aliases)
-        return tools.get_action_command_regexp(pattern)
+        pattern = self.PATTERN_TEMPLATE.format(command=pattern)
+        return re.compile(pattern, re.IGNORECASE | re.VERBOSE)
 
     def match_intent(self, intent):
         """Tell if ``intent`` is an ``ACTION``.
diff --git a/sopel/tools/__init__.py b/sopel/tools/__init__.py
--- a/sopel/tools/__init__.py
+++ b/sopel/tools/__init__.py
@@ -225,6 +225,7 @@ def compile_rule(nick, pattern, alias_nicks):
     return re.compile(pattern, flags)
 
 
+@deprecated('command regexp tools are now private', '7.1', '8.0')
 def get_command_regexp(prefix, command):
     """Get a compiled regexp object that implements the command.
 
@@ -233,15 +234,13 @@ def get_command_regexp(prefix, command):
     :return: a compiled regexp object that implements the command
     :rtype: :ref:`re.Pattern <python:re-objects>`
     """
-    # Escape all whitespace with a single backslash. This ensures that regexp
-    # in the prefix is treated as it was before the actual regexp was changed
-    # to use the verbose syntax.
-    prefix = re.sub(r"(\s)", r"\\\1", prefix)
-
-    pattern = get_command_pattern(prefix, command)
-    return re.compile(pattern, re.IGNORECASE | re.VERBOSE)
+    # Must defer import to avoid cyclic dependency
+    from sopel.plugins.rules import Command
+    rule = Command(name=command, prefix=prefix)
+    return rule.get_rule_regex()
 
 
+@deprecated('command regexp tools are now private', '7.1', '8.0')
 def get_command_pattern(prefix, command):
     """Get the uncompiled regex pattern for standard commands.
 
@@ -250,27 +249,12 @@ def get_command_pattern(prefix, command):
     :return: a regex pattern that will match the given command
     :rtype: str
     """
-    # This regexp matches equivalently and produces the same
-    # groups 1 and 2 as the old regexp: r'^%s(%s)(?: +(.*))?$'
-    # The only differences should be handling all whitespace
-    # like spaces and the addition of groups 3-6.
-    return r"""
-        (?:{prefix})({command}) # Command as group 1.
-        (?:\s+              # Whitespace to end command.
-        (                   # Rest of the line as group 2.
-        (?:(\S+))?          # Parameters 1-4 as groups 3-6.
-        (?:\s+(\S+))?
-        (?:\s+(\S+))?
-        (?:\s+(\S+))?
-        .*                  # Accept anything after the parameters.
-                            # Leave it up to the plugin to parse
-                            # the line.
-        ))?                 # Group 2 must be None, if there are no
-                            # parameters.
-        $                   # EoL, so there are no partial matches.
-        """.format(prefix=prefix, command=command)
+    # Must defer import to avoid cyclic dependency
+    from sopel.plugins.rules import Command
+    return Command.PATTERN_TEMPLATE.format(prefix=prefix, command=command)
 
 
+@deprecated('command regexp tools are now private', '7.1', '8.0')
 def get_nickname_command_regexp(nick, command, alias_nicks):
     """Get a compiled regexp object that implements the nickname command.
 
@@ -281,14 +265,19 @@ def get_nickname_command_regexp(nick, command, alias_nicks):
     :return: a compiled regex pattern that implements the given nickname command
     :rtype: :ref:`re.Pattern <python:re-objects>`
     """
+    # Must defer import to avoid cyclic dependency
+    from sopel.plugins.rules import NickCommand
+
     if isinstance(alias_nicks, unicode):
         alias_nicks = [alias_nicks]
     elif not isinstance(alias_nicks, (list, tuple)):
         raise ValueError('A list or string is required.')
 
-    return compile_rule(nick, get_nickname_command_pattern(command), alias_nicks)
+    rule = NickCommand(nick=nick, name=command, nick_aliases=alias_nicks)
+    return rule.get_rule_regex()
 
 
+@deprecated('command regexp tools are now private', '7.1', '8.0')
 def get_nickname_command_pattern(command):
     """Get the uncompiled regex pattern for a nickname command.
 
@@ -296,23 +285,12 @@ def get_nickname_command_pattern(command):
     :return: a regex pattern that will match the given nickname command
     :rtype: str
     """
-    return r"""
-        ^
-        $nickname[:,]? # Nickname.
-        \s+({command}) # Command as group 1.
-        (?:\s+         # Whitespace to end command.
-        (              # Rest of the line as group 2.
-        (?:(\S+))?     # Parameters 1-4 as groups 3-6.
-        (?:\s+(\S+))?
-        (?:\s+(\S+))?
-        (?:\s+(\S+))?
-        .*             # Accept anything after the parameters. Leave it up to
-                       # the plugin to parse the line.
-        ))?            # Group 1 must be None, if there are no parameters.
-        $              # EoL, so there are no partial matches.
-        """.format(command=command)
+    # Must defer import to avoid cyclic dependency
+    from sopel.plugins.rules import NickCommand
+    return NickCommand.PATTERN_TEMPLATE.format(command=command)
 
 
+@deprecated('command regexp tools are now private', '7.1', '8.0')
 def get_action_command_regexp(command):
     """Get a compiled regexp object that implements the command.
 
@@ -320,10 +298,13 @@ def get_action_command_regexp(command):
     :return: a compiled regexp object that implements the command
     :rtype: :ref:`re.Pattern <python:re-objects>`
     """
-    pattern = get_action_command_pattern(command)
-    return re.compile(pattern, re.IGNORECASE | re.VERBOSE)
+    # Must defer import to avoid cyclic dependency
+    from sopel.plugins.rules import ActionCommand
+    rule = ActionCommand(name=command)
+    return rule.get_rule_regex()
 
 
+@deprecated('command regexp tools are now private', '7.1', '8.0')
 def get_action_command_pattern(command):
     """Get the uncompiled regex pattern for action commands.
 
@@ -331,25 +312,9 @@ def get_action_command_pattern(command):
     :return: a regex pattern that will match the given command
     :rtype: str
     """
-    # This regexp matches equivalently and produces the same
-    # groups 1 and 2 as the old regexp: r'^%s(%s)(?: +(.*))?$'
-    # The only differences should be handling all whitespace
-    # like spaces and the addition of groups 3-6.
-    return r"""
-        ({command}) # Command as group 1.
-        (?:\s+              # Whitespace to end command.
-        (                   # Rest of the line as group 2.
-        (?:(\S+))?          # Parameters 1-4 as groups 3-6.
-        (?:\s+(\S+))?
-        (?:\s+(\S+))?
-        (?:\s+(\S+))?
-        .*                  # Accept anything after the parameters.
-                            # Leave it up to the plugin to parse
-                            # the line.
-        ))?                 # Group 2 must be None, if there are no
-                            # parameters.
-        $                   # EoL, so there are no partial matches.
-        """.format(command=command)
+    # Must defer import to avoid cyclic dependency
+    from sopel.plugins.rules import ActionCommand
+    return ActionCommand.PATTERN_TEMPLATE.format(command=command)
 
 
 def get_sendable_message(text, max_length=400):


Test patch:
diff --git a/sopel/test_tools.py b/sopel/test_tools.py
--- a/sopel/test_tools.py
+++ b/sopel/test_tools.py
@@ -24,13 +24,7 @@
 except ImportError:
     import configparser as ConfigParser
 
-from sopel.bot import SopelWrapper
-import sopel.config
-import sopel.config.core_section
-import sopel.plugins
-import sopel.tools
-import sopel.tools.target
-import sopel.trigger
+from sopel import bot, config, loader, plugins, tools, trigger
 
 
 __all__ = [
@@ -47,14 +41,14 @@
     basestring = str
 
 
-class MockConfig(sopel.config.Config):
-    @sopel.tools.deprecated('use configfactory fixture instead', '7.0', '8.0')
+class MockConfig(config.Config):
+    @tools.deprecated('use configfactory fixture instead', '7.0', '8.0')
     def __init__(self):
         self.filename = tempfile.mkstemp()[1]
         self.parser = ConfigParser.RawConfigParser(allow_no_value=True)
         self.parser.add_section('core')
         self.parser.set('core', 'owner', 'Embolalia')
-        self.define_section('core', sopel.config.core_section.CoreSection)
+        self.define_section('core', config.core_section.CoreSection)
         self.get = self.parser.get
 
     def define_section(self, name, cls_):
@@ -64,20 +58,20 @@ def define_section(self, name, cls_):
 
 
 class MockSopel(object):
-    @sopel.tools.deprecated('use botfactory fixture instead', '7.0', '8.0')
+    @tools.deprecated('use botfactory fixture instead', '7.0', '8.0')
     def __init__(self, nick, admin=False, owner=False):
         self.nick = nick
         self.user = "sopel"
 
-        channel = sopel.tools.Identifier("#Sopel")
-        self.channels = sopel.tools.SopelIdentifierMemory()
-        self.channels[channel] = sopel.tools.target.Channel(channel)
+        channel = tools.Identifier("#Sopel")
+        self.channels = tools.SopelIdentifierMemory()
+        self.channels[channel] = tools.target.Channel(channel)
 
-        self.users = sopel.tools.SopelIdentifierMemory()
-        self.privileges = sopel.tools.SopelMemory()
+        self.users = tools.SopelIdentifierMemory()
+        self.privileges = tools.SopelMemory()
 
-        self.memory = sopel.tools.SopelMemory()
-        self.memory['url_callbacks'] = sopel.tools.SopelMemory()
+        self.memory = tools.SopelMemory()
+        self.memory['url_callbacks'] = tools.SopelMemory()
 
         self.config = MockConfig()
         self._init_config()
@@ -119,14 +113,14 @@ def unregister_url_callback(self, pattern, callback):
             pass
 
     def search_url_callbacks(self, url):
-        for regex, function in sopel.tools.iteritems(self.memory['url_callbacks']):
+        for regex, function in tools.iteritems(self.memory['url_callbacks']):
             match = regex.search(url)
             if match:
                 yield function, match
 
 
-class MockSopelWrapper(SopelWrapper):
-    @sopel.tools.deprecated('use sopel.bot.SopelWrapper instead', '7.0', '8.0')
+class MockSopelWrapper(bot.SopelWrapper):
+    @tools.deprecated('use sopel.bot.SopelWrapper instead', '7.0', '8.0')
     def __init__(self, *args, **kwargs):
         super(MockSopelWrapper, self).__init__(*args, **kwargs)
 
@@ -167,33 +161,33 @@ def test(configfactory, botfactory, ircfactory):
         )
         settings = configfactory('default.cfg', test_config)
         url_schemes = settings.core.auto_url_schemes
-        bot = botfactory(settings)
-        server = ircfactory(bot)
+        mockbot = botfactory(settings)
+        server = ircfactory(mockbot)
         server.channel_joined('#Sopel')
 
-        match = None
-        if hasattr(tested_func, "commands"):
-            for command in tested_func.commands:
-                regexp = sopel.tools.get_command_regexp(".", command)
-                match = regexp.match(msg)
-                if match:
-                    break
-        assert match, "Example did not match any command."
+        if not hasattr(tested_func, 'commands'):
+            raise AssertionError('Function is not a command.')
 
-        sender = bot.nick if privmsg else "#channel"
-        hostmask = "%s!%s@%s" % (bot.nick, "UserName", "example.com")
+        loader.clean_callable(tested_func, settings)
+        test_rule = plugins.rules.Command.from_callable(settings, tested_func)
+        parse_results = list(test_rule.parse(msg))
+        assert parse_results, "Example did not match any command."
+
+        match = parse_results[0]
+        sender = mockbot.nick if privmsg else "#channel"
+        hostmask = "%s!%s@%s" % (mockbot.nick, "UserName", "example.com")
 
         # TODO enable message tags
         full_message = ':{} PRIVMSG {} :{}'.format(hostmask, sender, msg)
-        pretrigger = sopel.trigger.PreTrigger(
-            bot.nick, full_message, url_schemes=url_schemes)
-        trigger = sopel.trigger.Trigger(bot.settings, pretrigger, match)
-        pattern = re.compile(r'^%s: ' % re.escape(bot.nick))
+        pretrigger = trigger.PreTrigger(
+            mockbot.nick, full_message, url_schemes=url_schemes)
+        test_trigger = trigger.Trigger(mockbot.settings, pretrigger, match)
+        pattern = re.compile(r'^%s: ' % re.escape(mockbot.nick))
 
         # setup module
         module = sys.modules[tested_func.__module__]
         if hasattr(module, 'setup'):
-            module.setup(bot)
+            module.setup(mockbot)
 
         def isnt_ignored(value):
             """Return True if value doesn't match any re in ignore list."""
@@ -204,12 +198,15 @@ def isnt_ignored(value):
         expected_output_count = 0
         for _i in range(repeat):
             expected_output_count += len(results)
-            wrapper = SopelWrapper(bot, trigger)
-            tested_func(wrapper, trigger)
+            wrapper = bot.SopelWrapper(mockbot, test_trigger)
+            tested_func(wrapper, test_trigger)
 
             output_triggers = (
-                sopel.trigger.PreTrigger(
-                    bot.nick, message.decode('utf-8'), url_schemes=url_schemes)
+                trigger.PreTrigger(
+                    mockbot.nick,
+                    message.decode('utf-8'),
+                    url_schemes=url_schemes,
+                )
                 for message in wrapper.backend.message_sent
             )
             output_texts = (
@@ -263,6 +260,7 @@ def insert_into_module(func, module_name, base_name, prefix):
     setattr(module, func.__name__, func)
 
 
+@tools.deprecated('pytest now runs @plugin.example tests directly', '7.1', '8.0')
 def run_example_tests(filename, tb='native', multithread=False, verbose=False):
     # These are only required when running tests, so import them here rather
     # than at the module level.
diff --git a/test/plugins/test_plugins_rules.py b/test/plugins/test_plugins_rules.py
--- a/test/plugins/test_plugins_rules.py
+++ b/test/plugins/test_plugins_rules.py
@@ -1881,6 +1881,33 @@ def test_nick_command_match(mockbot):
     assert match.group(6) is None
 
 
+def test_nick_command_match_args(mockbot):
+    line = (
+        ':Foo!foo@example.com PRIVMSG #sopel :'
+        'TestBot: hello arg1 arg2 arg3 arg4 arg5'
+    )
+    pretrigger = trigger.PreTrigger(mockbot.nick, line)
+
+    rule = rules.NickCommand('TestBot', 'hello')
+    matches = list(rule.match(mockbot, pretrigger))
+    assert len(matches) == 1, 'Exactly one match must be found'
+
+    match = matches[0]
+    assert match.group(0) == 'TestBot: hello arg1 arg2 arg3 arg4 arg5'
+    assert match.group(1) == 'hello'
+    assert match.group(2) == 'arg1 arg2 arg3 arg4 arg5', (
+        'The global arg list must include everything else')
+    # group 3-6 must match the 4 first arguments
+    assert match.group(3) == 'arg1'
+    assert match.group(4) == 'arg2'
+    assert match.group(5) == 'arg3'
+    assert match.group(6) == 'arg4'
+
+    # command regex doesn't match more than 4 extra args
+    with pytest.raises(IndexError):
+        match.group(7)
+
+
 def test_nick_command_has_alias(mockbot):
     rule = rules.NickCommand('TestBot', 'hello', aliases=['hi'])
     assert rule.has_alias('hi')
@@ -2068,6 +2095,33 @@ def test_action_command_match(mockbot):
     assert match.group(6) is None
 
 
+def test_action_command_match_args(mockbot):
+    line = (
+        ':Foo!foo@example.com PRIVMSG #sopel :'
+        '\x01ACTION hello arg1 arg2 arg3 arg4 arg5\x01'
+    )
+    pretrigger = trigger.PreTrigger(mockbot.nick, line)
+
+    rule = rules.ActionCommand('hello')
+    matches = list(rule.match(mockbot, pretrigger))
+    assert len(matches) == 1, 'Exactly one match must be found'
+
+    match = matches[0]
+    assert match.group(0) == 'hello arg1 arg2 arg3 arg4 arg5'
+    assert match.group(1) == 'hello'
+    assert match.group(2) == 'arg1 arg2 arg3 arg4 arg5', (
+        'The global arg list must include everything else')
+    # group 3-6 must match the 4 first arguments
+    assert match.group(3) == 'arg1'
+    assert match.group(4) == 'arg2'
+    assert match.group(5) == 'arg3'
+    assert match.group(6) == 'arg4'
+
+    # command regex doesn't match more than 4 extra args
+    with pytest.raises(IndexError):
+        match.group(7)
+
+
 def test_action_command_has_alias(mockbot):
     rule = rules.ActionCommand('hello', aliases=['hi'])
     assert rule.has_alias('hi')
diff --git a/test/test_tools.py b/test/test_tools.py
--- a/test/test_tools.py
+++ b/test/test_tools.py
@@ -63,6 +63,11 @@ def nickname_command_line(nick, command, groups):
     return "{}: {} {}".format(nick, command, ' '.join(groups.values()))
 
 
+@pytest.fixture
+def action_command_line(command, groups):
+    return "{} {}".format(command, ' '.join(groups.values()))
+
+
 def test_command_groups(prefix, command, groups, command_line):
     regex = tools.get_command_regexp(prefix, command)
     match = re.match(regex, command_line)
@@ -100,6 +105,18 @@ def test_nickname_command_aliased(command, nick, alias_nicks, groups, nickname_c
     assert match.group(6) == groups[6]
 
 
+def test_action_command_groups(command, groups, action_command_line):
+    regex = tools.get_action_command_regexp(command)
+    match = re.match(regex, action_command_line)
+    assert match.group(0) == action_command_line
+    assert match.group(1) == command
+    assert match.group(2) == ' '.join(groups.values())
+    assert match.group(3) == groups[3]
+    assert match.group(4) == groups[4]
+    assert match.group(5) == groups[5]
+    assert match.group(6) == groups[6]
+
+
 def test_get_sendable_message_default():
     initial = 'aaaa'
     text, excess = tools.get_sendable_message(initial)