Repository: nerdvegas/rez
Pull request #1040 (2021-03-12 20:21:38)

Extension plugins
### Feature Propose
Allowing people registering their own sub-command, forwarding function, configs.

### What's Added (updated)
* New type plugin "Command"
* New plugin loading mechanism : scan `sys.path` for module that has `rezplugins` sub-module
* Warning messages for plugin name clashing related issue

### Why scan modules for plugins ?
The new command type plugin enables registering custom sub-command, which may need to access additional module to do it's job. Hence we have to install the module which is required by the sub-command into Rez's venv, so that it could be imported in production installed Rez session which is ignoring `PYTHONPATH`. With these in mind, installing custom rezplugins as a sub-module would be the most desirable way. Now, how to let Rez register plugin that has been installed into `lib/site-packages` ? Instead of registering full path into `rezconfig.plugin_path`, simply scan modules from `sys.path` is most friendly.

### Note
When name clashing happen, plugins loaded by the new loading mechanism will take precedence over plugins that are registered in `rezconfig.plugin_path`.

~Please have a visit to my example plugin and have a look how this works 👉🏼 https://github.com/davidlatwe/rezbefoo~ (outdated)



Patch:
diff --git a/example_extensions/hello_cmd/README.md b/example_extensions/hello_cmd/README.md
new file mode 100644
index 00000000..dc4d6c6a
--- /dev/null
+++ b/example_extensions/hello_cmd/README.md
@@ -0,0 +1,12 @@
+Welcome to install your first rez extension!
+
+Currently, please pip install this Python package into Rez's venv, and run
+```
+$ rez -i
+```
+You should see a plugin named "world" in the plugin list.
+And now you could do
+```
+$ rez world -h
+```
+to see what you could do about it.
diff --git a/example_extensions/hello_cmd/__init__.py b/example_extensions/hello_cmd/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/example_extensions/hello_cmd/lib.py b/example_extensions/hello_cmd/lib.py
new file mode 100644
index 00000000..b4f2224c
--- /dev/null
+++ b/example_extensions/hello_cmd/lib.py
@@ -0,0 +1,5 @@
+
+def get_message_from_world():
+    from rez.config import config
+    message = config.plugins.command.world.message
+    return message
diff --git a/example_extensions/hello_cmd/rezplugins/__init__.py b/example_extensions/hello_cmd/rezplugins/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/example_extensions/hello_cmd/rezplugins/command/__init__.py b/example_extensions/hello_cmd/rezplugins/command/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/example_extensions/hello_cmd/rezplugins/command/rezconfig.py b/example_extensions/hello_cmd/rezplugins/command/rezconfig.py
new file mode 100644
index 00000000..3f3e82a6
--- /dev/null
+++ b/example_extensions/hello_cmd/rezplugins/command/rezconfig.py
@@ -0,0 +1,4 @@
+
+world = {
+    "message": "welcome to this world."
+}
diff --git a/example_extensions/hello_cmd/rezplugins/command/world.py b/example_extensions/hello_cmd/rezplugins/command/world.py
new file mode 100644
index 00000000..d481c8da
--- /dev/null
+++ b/example_extensions/hello_cmd/rezplugins/command/world.py
@@ -0,0 +1,37 @@
+"""
+Demoing Rez's command type plugin
+"""
+from rez.command import Command
+
+# This attribute is optional, default behavior will be applied if not present.
+command_behavior = {
+    "hidden": False,   # (bool): default False
+    "arg_mode": None,  # (str): "passthrough", "grouped", default None
+}
+
+
+def setup_parser(parser, completions=False):
+    parser.add_argument("-m", "--message", action="store_true",
+                        help="Print message from world.")
+
+
+def command(opts, parser=None, extra_arg_groups=None):
+    from hello_cmd import lib
+
+    if opts.message:
+        msg = lib.get_message_from_world()
+        print(msg)
+        return
+
+    print("Please use '-h' flag to see what you can do to this world !")
+
+
+class WorldCommand(Command):
+
+    @classmethod
+    def name(cls):
+        return "world"
+
+
+def register_plugin():
+    return WorldCommand
diff --git a/example_extensions/hello_cmd/setup.py b/example_extensions/hello_cmd/setup.py
new file mode 100644
index 00000000..827638f4
--- /dev/null
+++ b/example_extensions/hello_cmd/setup.py
@@ -0,0 +1,12 @@
+from __future__ import print_function, with_statement
+from setuptools import setup
+
+
+setup(
+    name="hello_cmd",
+    version="1.0",
+    package_dir={"hello_cmd": ""},
+    packages=["hello_cmd",
+              "hello_cmd.rezplugins",
+              "hello_cmd.rezplugins.command"],
+)
diff --git a/src/rez/cli/_main.py b/src/rez/cli/_main.py
index adbc4e89..b579b9de 100644
--- a/src/rez/cli/_main.py
+++ b/src/rez/cli/_main.py
@@ -101,8 +101,8 @@ def setup_parser():
 
     # add lazy subparsers
     subparser = parser.add_subparsers(dest='cmd', metavar='COMMAND')
-    for subcommand in subcommands:
-        module_name = "rez.cli.%s" % subcommand
+    for subcommand, data in subcommands.items():
+        module_name = data.get('module_name', 'rez.cli.%s' % subcommand)
 
         subparser.add_parser(
             subcommand,
diff --git a/src/rez/cli/_util.py b/src/rez/cli/_util.py
index a3d253a5..a1585bdf 100644
--- a/src/rez/cli/_util.py
+++ b/src/rez/cli/_util.py
@@ -63,6 +63,57 @@ subcommands = {
 }
 
 
+def load_plugin_cmd():
+    """Load subcommand from command type plugin
+
+    The command type plugin module should have attribute `command_behavior`,
+    and the value must be a dict if provided. For example:
+
+        # in your command plugin module
+        command_behavior = {
+            "hidden": False,   # (bool): default False
+            "arg_mode": None,  #  (str): "passthrough", "grouped", default None
+        }
+
+    If the attribute not present, default behavior will be given.
+
+    """
+    from rez.config import config
+    from rez.utils.logging_ import print_debug
+    from rez.plugin_managers import plugin_manager
+
+    ext_plugins = dict()
+
+    for plugin_name in plugin_manager.get_plugins("command"):
+        module = plugin_manager.get_plugin_module("command", plugin_name)
+
+        behavior = getattr(module, "command_behavior", None)
+        if behavior is None:
+            behavior = dict()
+
+            if config.debug("plugins"):
+                print_debug("Attribute 'command_behavior' not found in plugin "
+                            "module %s, registering with default behavior."
+                            % module.__name__)
+        try:
+            data = behavior.copy()
+            data.update({"module_name": module.__name__})
+            ext_plugins[plugin_name] = data
+
+        except Exception:
+            if config.debug("plugins"):
+                import traceback
+                from rez.vendor.six.six import StringIO
+                out = StringIO()
+                traceback.print_exc(file=out)
+                print_debug(out.getvalue())
+
+    return ext_plugins
+
+
+subcommands.update(load_plugin_cmd())
+
+
 class LazySubParsersAction(_SubParsersAction):
     """Argparse Action which calls the `setup_subparser` function provided to
     `LazyArgumentParser`.
diff --git a/src/rez/cli/complete.py b/src/rez/cli/complete.py
index 521fe257..cf8f4e54 100644
--- a/src/rez/cli/complete.py
+++ b/src/rez/cli/complete.py
@@ -80,7 +80,8 @@ def command(opts, parser, extra_arg_groups=None):
 
     # create parser for subcommand
     from rez.backport.importlib import import_module
-    module_name = "rez.cli.%s" % subcommand
+    data = subcommands[subcommand]
+    module_name = data.get("module_name", "rez.cli.%s" % subcommand)
     mod = import_module(module_name)
     parser = argparse.ArgumentParser()
     mod.setup_parser(parser, completions=True)
diff --git a/src/rez/command.py b/src/rez/command.py
new file mode 100644
index 00000000..aad26a9f
--- /dev/null
+++ b/src/rez/command.py
@@ -0,0 +1,52 @@
+
+from rez.config import config
+
+
+class Command(object):
+    """An interface for registering custom Rez subcommand
+
+    To register plugin and expose subcommand, the plugin module..
+
+    * MUST have a module-level docstring (used as the command help)
+    * MUST provide a `setup_parser()` function
+    * MUST provide a `command()` function
+    * MUST provide a `register_plugin()` function
+    * SHOULD have a module-level attribute `command_behavior`
+
+    For example, a plugin named 'foo' and this is the `foo.py`:
+
+        '''The docstring for command help, this is required.
+        '''
+        from rez.command import Command
+
+        command_behavior = {
+            "hidden": False,   # optional: bool
+            "arg_mode": None,  # optional: None, "passthrough", "grouped"
+        }
+
+        def setup_parser(parser, completions=False):
+            parser.add_argument("--hello", ...)
+
+        def command(opts, parser=None, extra_arg_groups=None):
+            if opts.hello:
+                print("world")
+
+        class CommandFoo(Command):
+            schema_dict = {}
+
+            @classmethod
+            def name(cls):
+                return "foo"
+
+        def register_plugin():
+            return CommandFoo
+
+    """
+    def __init__(self):
+        self.type_settings = config.plugins.extension
+        self.settings = self.type_settings.get(self.name())
+
+    @classmethod
+    def name(cls):
+        """Return the name of the Command and rez-subcommand."""
+        raise NotImplementedError
diff --git a/src/rez/plugin_managers.py b/src/rez/plugin_managers.py
index d787dbac..87a52de7 100644
--- a/src/rez/plugin_managers.py
+++ b/src/rez/plugin_managers.py
@@ -8,6 +8,7 @@ from rez.utils.data_utils import LazySingleton, cached_property, deep_update
 from rez.utils.logging_ import print_debug, print_warning
 from rez.vendor.six import six
 from rez.exceptions import RezPluginError
+import pkgutil
 import os.path
 import sys
 
@@ -52,22 +53,31 @@ def extend_path(path, name):
     init_py = "__init__" + os.extsep + "py"
     path = path[:]
 
-    for dir_ in config.plugin_path:
-        if not os.path.isdir(dir_):
-            if config.debug("plugins"):
-                print_debug("skipped nonexistant rez plugin path: %s" % dir_)
-            continue
+    def append_if_valid(dir_):
+        if os.path.isdir(dir_):
+            subdir = os.path.normcase(os.path.join(dir_, pname))
+            initfile = os.path.join(subdir, init_py)
+            if subdir not in path and os.path.isfile(initfile):
+                path.append(subdir)
+
+        elif config.debug("plugins"):
+            print_debug("skipped nonexistant rez plugin path: %s" % dir_)
 
-        subdir = os.path.join(dir_, pname)
-        # XXX This may still add duplicate entries to path on
-        # case-insensitive filesystems
-        initfile = os.path.join(subdir, init_py)
-        if subdir not in path and os.path.isfile(initfile):
-            path.append(subdir)
+    # Extend old-style plugins
+    for dir_ in config.plugin_path:
+        append_if_valid(dir_)
+    # Extend new-style plugins
+    for dir_ in plugin_manager.rezplugins_module_paths:
+        append_if_valid(dir_)
 
     return path
 
 
+def uncache_rezplugins_module_paths(instance=None):
+    instance = instance or plugin_manager
+    cached_property.uncache(instance, "rezplugins_module_paths")
+
+
 class RezPluginType(object):
     """An abstract base class representing a single type of plugin.
 
@@ -112,42 +122,66 @@ class RezPluginType(object):
             else package.__path__
 
         # reverse plugin path order, so that custom plugins have a chance to
-        # override the builtin plugins (from /rezplugins).
+        # be found before the builtin plugins (from /rezplugins).
         paths = reversed(paths)
 
         for path in paths:
             if config.debug("plugins"):
                 print_debug("searching plugin path %s...", path)
 
-            for loader, modname, ispkg in pkgutil.iter_modules(
+            for importer, modname, ispkg in pkgutil.iter_modules(
                     [path], package.__name__ + '.'):
 
-                if loader is None:
+                if importer is None:
                     continue
 
                 plugin_name = modname.split('.')[-1]
-                if plugin_name.startswith('_'):
+                if plugin_name.startswith('_') or plugin_name == 'rezconfig':
+                    continue
+
+                if plugin_name in self.plugin_modules:
+                    # same named plugins will have identical module name,
+                    # which will just reuse previous imported module from
+                    # `sys.modules` below. skipping the rest of the process
+                    # for good.
+                    if config.debug("plugins"):
+                        print_warning("skipped same named %s plugin at %s: %s"
+                                      % (self.type_name, path, modname))
                     continue
 
                 if config.debug("plugins"):
                     print_debug("loading %s plugin at %s: %s..."
                                 % (self.type_name, path, modname))
                 try:
+                    # nerdvegas/rez#218
                     # load_module will force reload the module if it's
                     # already loaded, so check for that
-                    module = sys.modules.get(modname)
-                    if module is None:
-                        module = loader.find_module(modname).load_module(modname)
-                    if hasattr(module, 'register_plugin') and \
-                            hasattr(module.register_plugin, '__call__'):
-                        plugin_class = module.register_plugin()
+                    plugin_module = sys.modules.get(modname)
+                    if plugin_module is None:
+                        loader = importer.find_module(modname)
+                        plugin_module = loader.load_module(modname)
+
+                    elif os.path.dirname(plugin_module.__file__) != path:
+                        if config.debug("plugins"):
+                            # this should not happen but if it does, tell why.
+                            print_warning(
+                                "plugin module %s is not loaded from current "
+                                "load path but reused from previous imported "
+                                "path: %s" % (modname, plugin_module.__file__))
+
+                    if (hasattr(plugin_module, "register_plugin")
+                            and callable(plugin_module.register_plugin)):
+
+                        plugin_class = plugin_module.register_plugin()
                         if plugin_class is not None:
-                            self.register_plugin(plugin_name, plugin_class, module)
+                            self.register_plugin(plugin_name,
+                                                 plugin_class,
+                                                 plugin_module)
                         else:
                             if config.debug("plugins"):
                                 print_warning(
-                                    "'register_plugin' function at %s: %s did not return a class."
-                                    % (path, modname))
+                                    "'register_plugin' function at %s: %s did "
+                                    "not return a class." % (path, modname))
                     else:
                         if config.debug("plugins"):
                             print_warning(
@@ -261,6 +295,19 @@ class RezPluginManager(object):
     def __init__(self):
         self._plugin_types = {}
 
+    @cached_property
+    def rezplugins_module_paths(self):
+        paths = []
+        for importer, name, ispkg in pkgutil.iter_modules():
+            if not ispkg:
+                continue
+
+            module_path = os.path.join(importer.path, name)
+            if os.path.isdir(os.path.join(module_path, "rezplugins")):
+                paths.append(module_path)
+
+        return paths
+
     # -- plugin types
 
     def _get_plugin_type(self, plugin_type):
@@ -380,6 +427,12 @@ class BuildProcessPluginType(RezPluginType):
     type_name = "build_process"
 
 
+class CommandPluginType(RezPluginType):
+    """Support for different custom Rez applications/subcommands.
+    """
+    type_name = "command"
+
+
 plugin_manager = RezPluginManager()
 
 
@@ -389,6 +442,7 @@ plugin_manager.register_plugin_type(ReleaseHookPluginType)
 plugin_manager.register_plugin_type(BuildSystemPluginType)
 plugin_manager.register_plugin_type(PackageRepositoryPluginType)
 plugin_manager.register_plugin_type(BuildProcessPluginType)
+plugin_manager.register_plugin_type(CommandPluginType)
 
 
 # Copyright 2013-2016 Allan Johns.
diff --git a/src/rezplugins/command/__init__.py b/src/rezplugins/command/__init__.py
new file mode 100644
index 00000000..6dab8423
--- /dev/null
+++ b/src/rezplugins/command/__init__.py
@@ -0,0 +1,18 @@
+from rez.plugin_managers import extend_path
+__path__ = extend_path(__path__, __name__)
+
+
+# Copyright 2013-2016 Allan Johns.
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.


Test patch:
diff --git a/src/rez/data/tests/extensions/bar/__init__.py b/src/rez/data/tests/extensions/bar/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/rez/data/tests/extensions/bar/rezplugins/__init__.py b/src/rez/data/tests/extensions/bar/rezplugins/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/rez/data/tests/extensions/bar/rezplugins/package_repository/__init__.py b/src/rez/data/tests/extensions/bar/rezplugins/package_repository/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/rez/data/tests/extensions/bar/rezplugins/package_repository/memory.py b/src/rez/data/tests/extensions/bar/rezplugins/package_repository/memory.py
new file mode 100644
index 00000000..4a0a7243
--- /dev/null
+++ b/src/rez/data/tests/extensions/bar/rezplugins/package_repository/memory.py
@@ -0,0 +1,13 @@
+
+from rez.package_repository import PackageRepository
+
+
+class MemoryPackageRepository(PackageRepository):
+    @classmethod
+    def name(cls):
+        return "memory"
+    on_test = "bar"
+
+
+def register_plugin():
+    return MemoryPackageRepository
diff --git a/src/rez/data/tests/extensions/foo/__init__.py b/src/rez/data/tests/extensions/foo/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/rez/data/tests/extensions/foo/rezplugins/__init__.py b/src/rez/data/tests/extensions/foo/rezplugins/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/rez/data/tests/extensions/foo/rezplugins/package_repository/__init__.py b/src/rez/data/tests/extensions/foo/rezplugins/package_repository/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/rez/data/tests/extensions/foo/rezplugins/package_repository/cloud.py b/src/rez/data/tests/extensions/foo/rezplugins/package_repository/cloud.py
new file mode 100644
index 00000000..f5729a18
--- /dev/null
+++ b/src/rez/data/tests/extensions/foo/rezplugins/package_repository/cloud.py
@@ -0,0 +1,12 @@
+
+from rez.package_repository import PackageRepository
+
+
+class CloudPackageRepository(PackageRepository):
+    @classmethod
+    def name(cls):
+        return "cloud"
+
+
+def register_plugin():
+    return CloudPackageRepository
diff --git a/src/rez/data/tests/extensions/non-mod/rezplugins/__init__.py b/src/rez/data/tests/extensions/non-mod/rezplugins/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/rez/data/tests/extensions/non-mod/rezplugins/package_repository/__init__.py b/src/rez/data/tests/extensions/non-mod/rezplugins/package_repository/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/rez/data/tests/extensions/non-mod/rezplugins/package_repository/memory.py b/src/rez/data/tests/extensions/non-mod/rezplugins/package_repository/memory.py
new file mode 100644
index 00000000..26308100
--- /dev/null
+++ b/src/rez/data/tests/extensions/non-mod/rezplugins/package_repository/memory.py
@@ -0,0 +1,13 @@
+
+from rez.package_repository import PackageRepository
+
+
+class MemoryPackageRepository(PackageRepository):
+    @classmethod
+    def name(cls):
+        return "memory"
+    on_test = "non-mod"
+
+
+def register_plugin():
+    return MemoryPackageRepository
diff --git a/src/rez/tests/test_completion.py b/src/rez/tests/test_completion.py
index 1ee88dd6..d99f82d8 100644
--- a/src/rez/tests/test_completion.py
+++ b/src/rez/tests/test_completion.py
@@ -28,6 +28,7 @@ class TestCompletion(TestBase):
         _eq("plugin", ["plugins",
                        "plugin_path"])
         _eq("plugins", ["plugins",
+                        "plugins.command",
                         "plugins.package_repository",
                         "plugins.build_process",
                         "plugins.build_system",
diff --git a/src/rez/tests/test_plugin_manager.py b/src/rez/tests/test_plugin_manager.py
new file mode 100644
index 00000000..9ae6d6ce
--- /dev/null
+++ b/src/rez/tests/test_plugin_manager.py
@@ -0,0 +1,102 @@
+"""
+test rezplugins manager behaviors
+"""
+from rez.tests.util import TestBase, TempdirMixin, restore_sys_path
+from rez.plugin_managers import plugin_manager, uncache_rezplugins_module_paths
+from rez.package_repository import package_repository_manager
+import sys
+import unittest
+
+
+class TestPluginManagers(TestBase, TempdirMixin):
+    def __init__(self, *nargs, **kwargs):
+        TestBase.__init__(self, *nargs, **kwargs)
+        self._reset_plugin_manager()
+
+    @classmethod
+    def _reset_plugin_manager(cls):
+        # for resetting package_repository type plugins
+        package_repository_manager.clear_caches()
+        package_repository_manager.pool.resource_classes.clear()
+        # for resetting new-style plugins
+        uncache_rezplugins_module_paths()
+
+        plugin_types = []
+        for singleton in plugin_manager._plugin_types.values():
+            plugin_types.append(singleton.instance_class)
+        plugin_manager._plugin_types.clear()
+
+        for plugin_type in plugin_types:
+            plugin_manager.register_plugin_type(plugin_type)
+
+        for key in list(sys.modules.keys()):
+            if key.startswith("rezplugins."):
+                del sys.modules[key]
+
+    @classmethod
+    def setUpClass(cls):
+        cls.settings = {"debug_plugins": True}
+
+    @classmethod
+    def tearDownClass(cls):
+        cls._reset_plugin_manager()
+
+    def setUp(self):
+        TestBase.setUp(self)
+        self._reset_plugin_manager()
+
+    def test_old_loading_style(self):
+        """Test loading rez plugin from plugin_path"""
+        self.update_settings(dict(
+            plugin_path=[self.data_path("extensions", "foo")]
+        ))
+
+        cloud_cls = plugin_manager.get_plugin_class(
+            "package_repository", "cloud")
+        self.assertEqual(cloud_cls.name(), "cloud")
+
+    def test_new_loading_style(self):
+        """Test loading rez plugin from python modules"""
+        with restore_sys_path():
+            sys.path.append(self.data_path("extensions"))
+
+            cloud_cls = plugin_manager.get_plugin_class(
+                "package_repository", "cloud")
+            self.assertEqual(cloud_cls.name(), "cloud")
+
+    def test_plugin_override_1(self):
+        """Test plugin from plugin_path can override the default"""
+        self.update_settings(dict(
+            plugin_path=[self.data_path("extensions", "non-mod")]
+        ))
+
+        mem_cls = plugin_manager.get_plugin_class(
+            "package_repository", "memory")
+        self.assertEqual("non-mod", mem_cls.on_test)
+
+    def test_plugin_override_2(self):
+        """Test plugin from python modules can override the default"""
+        with restore_sys_path():
+            sys.path.append(self.data_path("extensions"))
+
+            mem_cls = plugin_manager.get_plugin_class(
+                "package_repository", "memory")
+            self.assertEqual("bar", mem_cls.on_test)
+
+    def test_plugin_override_3(self):
+        """Test plugin from python modules can override plugin_path"""
+        with restore_sys_path():
+            # setup new
+            sys.path.append(self.data_path("extensions"))
+            # setup old
+            self.update_settings(dict(
+                plugin_path=[self.data_path("extensions", "non-mod")]
+            ))
+
+            mem_cls = plugin_manager.get_plugin_class(
+                "package_repository", "memory")
+            self.assertEqual("bar", mem_cls.on_test)
+
+
+if __name__ == '__main__':
+    unittest.main()