Repository: facebookresearch/hydra
Pull request #256 (2019-10-23 00:43:03)

Example Launcher plugin

omry: 1. moved example plugins into plugins/examples
2. new example Launcher plugin

Patch:
diff --git a/noxfile.py b/noxfile.py
index 54381512a5..6a3c9b5e08 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -41,14 +41,31 @@ def run_pytest(session, directory="."):
 
 
 def plugin_names():
-    return sorted(os.listdir(os.path.join(BASE, "plugins")))
+    return [x["name"] for x in list_plugins()]
+
+
+def list_plugins():
+    example_plugins = [
+        {"name": x, "path": "examples/{}".format(x)}
+        for x in sorted(os.listdir(os.path.join(BASE, "plugins/examples")))
+    ]
+    plugins = [
+        {"name": x, "path": x}
+        for x in sorted(os.listdir(os.path.join(BASE, "plugins")))
+        if x != "examples"
+    ]
+    return plugins + example_plugins
 
 
 def get_all_plugins():
     return [
-        (plugin, "hydra_plugins." + plugin)
-        for plugin in plugin_names()
-        if plugin in PLUGINS or PLUGINS == ["ALL"]
+        {
+            "name": plugin["name"],
+            "path": plugin["path"],
+            "module": "hydra_plugins." + plugin["name"],
+        }
+        for plugin in list_plugins()
+        if plugin["name"] in PLUGINS or PLUGINS == ["ALL"]
     ]
 
 
@@ -83,7 +100,7 @@ def get_setup_python_versions(session, setup_py):
 
 def get_plugin_python_version(session, plugin):
     return get_setup_python_versions(
-        session, os.path.join(BASE, "plugins", plugin, "setup.py")
+        session, os.path.join(BASE, "plugins", plugin["path"], "setup.py")
     )
 
 
@@ -93,7 +110,7 @@ def get_plugin_python_version(session, plugin):
     PLUGINS_INSTALL_COMMANDS,
     ids=[" ".join(x) for x in PLUGINS_INSTALL_COMMANDS],
 )
-@nox.parametrize("plugin", plugin_names(), ids=plugin_names())
+@nox.parametrize("plugin", list_plugins(), ids=plugin_names())
 def test_plugin(session, plugin, install_cmd):
     session.install("--upgrade", "setuptools", "pip")
     # Verify this plugin supports the python we are testing on, skip otherwise
@@ -109,21 +126,21 @@ def test_plugin(session, plugin, install_cmd):
     plugin_enabled = {}
     all_plugins = get_all_plugins()
     # Install all plugins in session
-    for a_plugin in all_plugins:
-        cmd = list(install_cmd) + [os.path.join("plugins", a_plugin[0])]
-        pythons = get_plugin_python_version(session, a_plugin[0])
-        plugin_enabled[plugin] = session.python in pythons
-        if plugin_enabled[plugin]:
+    for plugin in all_plugins:
+        cmd = list(install_cmd) + [os.path.join("plugins", plugin["path"])]
+        pythons = get_plugin_python_version(session, plugin)
+        plugin_enabled[plugin["path"]] = session.python in pythons
+        if plugin_enabled[plugin["path"]]:
             session.run(*cmd, silent=True)
 
     # Test that we can import Hydra
     session.run("python", "-c", "from hydra import main", silent=True)
 
     # Test that we can import all installed plugins
-    for a_plugin in all_plugins:
+    for plugin in all_plugins:
         # install all other plugins that are compatible with the current Python version
-        if plugin_enabled[plugin]:
-            session.run("python", "-c", "import {}".format(a_plugin[1]))
+        if plugin_enabled[plugin["path"]]:
+            session.run("python", "-c", "import {}".format(plugin["module"]))
 
     install_pytest(session)
 
@@ -131,7 +148,7 @@ def test_plugin(session, plugin, install_cmd):
     run_pytest(session, "tests")
 
     # Run tests for current plugin
-    session.chdir(os.path.join(BASE, "plugins", plugin))
+    session.chdir(os.path.join(BASE, "plugins", plugin["path"]))
     run_pytest(session)
 
 
@@ -144,12 +161,12 @@ def coverage(session):
     # Install all plugins in session
     for plugin in get_all_plugins():
         session.run(
-            "pip", "install", "-e", os.path.join("plugins", plugin[0]), silent=True
+            "pip", "install", "-e", os.path.join("plugins", plugin["path"]), silent=True
         )
 
     session.run("coverage", "erase")
     session.run("coverage", "run", "--append", "-m", "pytest", silent=True)
-    for plugin in plugin_names():
+    for plugin in list_plugins():
         plugin_python_versions = get_plugin_python_version(session, plugin)
         if session.python not in plugin_python_versions:
             continue
@@ -160,7 +177,7 @@ def coverage(session):
             "--append",
             "-m",
             "pytest",
-            os.path.join("plugins", plugin),
+            os.path.join("plugins", plugin["path"]),
             silent=True,
         )
 
diff --git a/plugins/examples/example_launcher/MANIFEST.in b/plugins/examples/example_launcher/MANIFEST.in
new file mode 100644
index 0000000000..df1000cc71
--- /dev/null
+++ b/plugins/examples/example_launcher/MANIFEST.in
@@ -0,0 +1,3 @@
+global-exclude *.pyc
+global-exclude __pycache__
+recursive-include hydra_plugins/* *.yaml
diff --git a/plugins/examples/example_launcher/README.md b/plugins/examples/example_launcher/README.md
new file mode 100644
index 0000000000..a8e13b8bf2
--- /dev/null
+++ b/plugins/examples/example_launcher/README.md
@@ -0,0 +1,35 @@
+# Hydra example Launcher plugin
+
+This plugin provides an example for how to write a custom Launcher for Hydra.
+The configuration for this launcher is in packages with the plugin:
+
+`hydra_plugins/example_launcher/conf/hydra/launcher/example.yaml`:
+```yaml
+hydra:
+  launcher:
+    class: hydra_plugins.example_launcher.ExampleLauncher
+    params:
+      foo: 10
+      bar: abcde
+```
+The example application is overriding the Launcher used by Hydra.
+When the launcher is initialized, its "using" the foo and bar parameters.
+
+Output of the example application:
+```text
+$ python example/my_app.py  --multirun db=postgresql,mysql
+[2019-10-22 19:45:05,060] - Example Launcher(foo=10, bar=abcde) is launching 2 jobs locally
+[2019-10-22 19:45:05,060] - Sweep output dir : multirun/2019-10-22/19-45-05
+[2019-10-22 19:45:05,060] -     #0 : db=postgresql
+db:
+  driver: postgresql
+  pass: drowssap
+  timeout: 10
+  user: postgre_user
+
+[2019-10-22 19:45:05,135] -     #1 : db=mysql
+db:
+  driver: mysql
+  pass: secret
+  user: omry
+```
\ No newline at end of file
diff --git a/plugins/examples/example_launcher/example/conf/config.yaml b/plugins/examples/example_launcher/example/conf/config.yaml
new file mode 100644
index 0000000000..a5c76f7615
--- /dev/null
+++ b/plugins/examples/example_launcher/example/conf/config.yaml
@@ -0,0 +1,3 @@
+defaults:
+  - hydra/launcher: example
+  - db: mysql
\ No newline at end of file
diff --git a/plugins/examples/example_launcher/example/conf/db/mysql.yaml b/plugins/examples/example_launcher/example/conf/db/mysql.yaml
new file mode 100644
index 0000000000..dc46278122
--- /dev/null
+++ b/plugins/examples/example_launcher/example/conf/db/mysql.yaml
@@ -0,0 +1,4 @@
+db:
+  driver: mysql
+  user: omry
+  pass: secret
diff --git a/plugins/examples/example_launcher/example/conf/db/postgresql.yaml b/plugins/examples/example_launcher/example/conf/db/postgresql.yaml
new file mode 100644
index 0000000000..862073c5d8
--- /dev/null
+++ b/plugins/examples/example_launcher/example/conf/db/postgresql.yaml
@@ -0,0 +1,5 @@
+db:
+  driver: postgresql
+  user: postgre_user
+  pass: drowssap
+  timeout: 10
diff --git a/plugins/examples/example_launcher/example/my_app.py b/plugins/examples/example_launcher/example/my_app.py
new file mode 100644
index 0000000000..5b48a4e304
--- /dev/null
+++ b/plugins/examples/example_launcher/example/my_app.py
@@ -0,0 +1,11 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import hydra
+
+
+@hydra.main(config_path="conf/config.yaml")
+def my_app(cfg):
+    print(cfg.pretty())
+
+
+if __name__ == "__main__":
+    my_app()
diff --git a/plugins/example_plugin/hydra_plugins/__init__.py b/plugins/examples/example_launcher/hydra_plugins/__init__.py
similarity index 100%
rename from plugins/example_plugin/hydra_plugins/__init__.py
rename to plugins/examples/example_launcher/hydra_plugins/__init__.py
diff --git a/plugins/examples/example_launcher/hydra_plugins/example_launcher/__init__.py b/plugins/examples/example_launcher/hydra_plugins/example_launcher/__init__.py
new file mode 100644
index 0000000000..9f4ef2ee7d
--- /dev/null
+++ b/plugins/examples/example_launcher/hydra_plugins/example_launcher/__init__.py
@@ -0,0 +1,4 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from .example_launcher import ExampleLauncher
+
+__all__ = ["ExampleLauncher"]
diff --git a/plugins/example_plugin/tests/__init__.py b/plugins/examples/example_launcher/hydra_plugins/example_launcher/conf/__init__.py
similarity index 100%
rename from plugins/example_plugin/tests/__init__.py
rename to plugins/examples/example_launcher/hydra_plugins/example_launcher/conf/__init__.py
diff --git a/plugins/examples/example_launcher/hydra_plugins/example_launcher/conf/hydra/launcher/example.yaml b/plugins/examples/example_launcher/hydra_plugins/example_launcher/conf/hydra/launcher/example.yaml
new file mode 100644
index 0000000000..6d0e4a8639
--- /dev/null
+++ b/plugins/examples/example_launcher/hydra_plugins/example_launcher/conf/hydra/launcher/example.yaml
@@ -0,0 +1,6 @@
+hydra:
+  launcher:
+    class: hydra_plugins.example_launcher.ExampleLauncher
+    params:
+      foo: 10
+      bar: abcde
\ No newline at end of file
diff --git a/plugins/examples/example_launcher/hydra_plugins/example_launcher/example_launcher.py b/plugins/examples/example_launcher/hydra_plugins/example_launcher/example_launcher.py
new file mode 100644
index 0000000000..f3e018e35a
--- /dev/null
+++ b/plugins/examples/example_launcher/hydra_plugins/example_launcher/example_launcher.py
@@ -0,0 +1,91 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import logging
+
+from omegaconf import open_dict
+
+from hydra._internal.config_search_path import ConfigSearchPath
+from hydra._internal.pathlib import Path
+from hydra.plugins import Launcher
+from hydra.plugins import SearchPathPlugin
+from hydra.plugins.common.utils import (
+    configure_log,
+    get_overrides_dirname,
+    filter_overrides,
+    run_job,
+    setup_globals,
+    HydraConfig,
+)
+
+log = logging.getLogger(__name__)
+
+
+class ExampleLauncherSearchPathPlugin(SearchPathPlugin):
+    """
+    This plugin is allowing configuration files provided by the ExampleLauncher plugin to be discovered
+    and used once the ExampleLauncher plugin is installed
+    """
+
+    def manipulate_search_path(self, search_path):
+        assert isinstance(search_path, ConfigSearchPath)
+        # Appends the search path for this plugin to the end of the search path
+        search_path.append(
+            "hydra-example-launcher", "pkg://hydra_plugins.example_launcher.conf"
+        )
+
+
+class ExampleLauncher(Launcher):
+    def __init__(self, foo, bar):
+        self.config = None
+        self.config_loader = None
+        self.task_function = None
+        # foo and var are coming from the the plugin's configuration
+        self.foo = foo
+        self.bar = bar
+
+    def setup(self, config, config_loader, task_function):
+        self.config = config
+        self.config_loader = config_loader
+        self.task_function = task_function
+
+    def launch(self, job_overrides):
+        """
+        :param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
+        :return: an array of return values from run_job with indexes corresponding to the input list indexes.
+        """
+        setup_globals()
+        configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose)
+        sweep_dir = Path(str(self.config.hydra.sweep.dir))
+        sweep_dir.mkdir(parents=True, exist_ok=True)
+        log.info(
+            "Example Launcher(foo={}, bar={}) is launching {} jobs locally".format(
+                self.foo, self.bar, len(job_overrides)
+            )
+        )
+        log.info("Sweep output dir : {}".format(sweep_dir))
+        runs = []
+
+        for idx, overrides in enumerate(job_overrides):
+            log.info("\t#{} : {}".format(idx, " ".join(filter_overrides(overrides))))
+            sweep_config = self.config_loader.load_sweep_config(
+                self.config, list(overrides)
+            )
+            with open_dict(sweep_config):
+                # This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
+                # In that case, it will not be available here because we are still in the main process.
+                # but instead should be populated remotely before calling the task_function.
+                sweep_config.hydra.job.id = "job_id_for_{}".format(idx)
+                sweep_config.hydra.job.num = idx
+                sweep_config.hydra.job.override_dirname = get_overrides_dirname(
+                    sweep_config.hydra.overrides.task
+                )
+            HydraConfig().set_config(sweep_config)
+
+            ret = run_job(
+                config=sweep_config,
+                task_function=self.task_function,
+                job_dir_key="hydra.sweep.dir",
+                job_subdir_key="hydra.sweep.subdir",
+            )
+            runs.append(ret)
+            configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose)
+        return runs
diff --git a/plugins/examples/example_launcher/setup.py b/plugins/examples/example_launcher/setup.py
new file mode 100644
index 0000000000..64e2d60d19
--- /dev/null
+++ b/plugins/examples/example_launcher/setup.py
@@ -0,0 +1,31 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from setuptools import setup, find_packages
+
+with open("README.md", "r") as fh:
+    LONG_DESC = fh.read()
+    setup(
+        name="hydra-example-launcher",
+        version="0.1.0",
+        author="Omry Yadan",
+        author_email="omry@fb.com",
+        description="Example Hydra Launcher plugin",
+        long_description=LONG_DESC,
+        long_description_content_type="text/markdown",
+        url="https://github.com/facebookresearch/hydra/",
+        packages=find_packages(exclude=["tests", "example"]),
+        classifiers=[
+            # Feel free to choose another license
+            "License :: OSI Approved :: MIT License",
+            # Python versions are used by the noxfile in Hydra to determine which
+            # python versions to test this plugin with
+            "Programming Language :: Python :: 2.7",
+            "Programming Language :: Python :: 3.6",
+            "Programming Language :: Python :: 3.7",
+            "Operating System :: OS Independent",
+        ],
+        install_requires=["hydra-core"],
+        # If this plugin is providing configuration files, be sure to include them in the package.
+        # See MANIFEST.in.
+        # For configurations to be discoverable at runtime, they should also be added to the search path.
+        include_package_data=True,
+    )
diff --git a/plugins/examples/example_plugin/MANIFEST.in b/plugins/examples/example_plugin/MANIFEST.in
new file mode 100644
index 0000000000..df1000cc71
--- /dev/null
+++ b/plugins/examples/example_plugin/MANIFEST.in
@@ -0,0 +1,3 @@
+global-exclude *.pyc
+global-exclude __pycache__
+recursive-include hydra_plugins/* *.yaml
diff --git a/plugins/example_plugin/README.md b/plugins/examples/example_plugin/README.md
similarity index 69%
rename from plugins/example_plugin/README.md
rename to plugins/examples/example_plugin/README.md
index fa82c3ec87..904b9f3bb2 100644
--- a/plugins/example_plugin/README.md
+++ b/plugins/examples/example_plugin/README.md
@@ -1,5 +1,5 @@
 # Hydra example plugin
-Use this as the template for future plugins.
+Use this as the template for generic Hydra plugins.
 
 Core plugins (Plugins that are developed jointly with Hydra) lives in the plugins subdirectory inside the Hydra repo.
 Third party plugins can live anywhere (public or private Github repos, Organizations in-house SCM etc)
\ No newline at end of file
diff --git a/plugins/examples/example_plugin/hydra_plugins/__init__.py b/plugins/examples/example_plugin/hydra_plugins/__init__.py
new file mode 100644
index 0000000000..dc1c6c1ca8
--- /dev/null
+++ b/plugins/examples/example_plugin/hydra_plugins/__init__.py
@@ -0,0 +1,2 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)
diff --git a/plugins/example_plugin/hydra_plugins/example_plugin/__init__.py b/plugins/examples/example_plugin/hydra_plugins/example_plugin/__init__.py
similarity index 100%
rename from plugins/example_plugin/hydra_plugins/example_plugin/__init__.py
rename to plugins/examples/example_plugin/hydra_plugins/example_plugin/__init__.py
diff --git a/plugins/example_plugin/hydra_plugins/example_plugin/example_plugin.py b/plugins/examples/example_plugin/hydra_plugins/example_plugin/example_plugin.py
similarity index 100%
rename from plugins/example_plugin/hydra_plugins/example_plugin/example_plugin.py
rename to plugins/examples/example_plugin/hydra_plugins/example_plugin/example_plugin.py
diff --git a/plugins/example_plugin/setup.py b/plugins/examples/example_plugin/setup.py
similarity index 66%
rename from plugins/example_plugin/setup.py
rename to plugins/examples/example_plugin/setup.py
index 9e350e3606..29c658a6f1 100644
--- a/plugins/example_plugin/setup.py
+++ b/plugins/examples/example_plugin/setup.py
@@ -8,12 +8,14 @@ with open("README.md", "r") as fh:
         version="0.1.0",
         author="Omry Yadan",
         author_email="omry@fb.com",
-        description="Example plugin for hydra",
+        description="Example Hydra plugin",
         long_description=LONG_DESC,
         long_description_content_type="text/markdown",
         url="https://github.com/facebookresearch/hydra/",
         packages=find_packages(exclude=["tests"]),
         classifiers=[
+            # Feel free to choose another license
+            "License :: OSI Approved :: MIT License",
             # Python versions are used by the noxfile in Hydra to determine which
             # python versions to test this plugin with
             "Programming Language :: Python :: 2.7",
@@ -21,5 +23,9 @@ with open("README.md", "r") as fh:
             "Programming Language :: Python :: 3.7",
             "Operating System :: OS Independent",
         ],
-        install_requires=[],
+        install_requires=["hydra-core"],
+        # If this plugin is providing configuraiton files, be sure to include them in the package.
+        # See MANIFEST.in.
+        # For configurations to be discoverable at runtime, they should also be added to the search path.
+        include_package_data=True,
     )


Test patch:
diff --git a/plugins/examples/example_launcher/tests/__init__.py b/plugins/examples/example_launcher/tests/__init__.py
new file mode 100644
index 0000000000..168f9979a4
--- /dev/null
+++ b/plugins/examples/example_launcher/tests/__init__.py
@@ -0,0 +1,1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
diff --git a/plugins/examples/example_launcher/tests/test_example_launcher_plugin.py b/plugins/examples/example_launcher/tests/test_example_launcher_plugin.py
new file mode 100644
index 0000000000..1a4f1ca674
--- /dev/null
+++ b/plugins/examples/example_launcher/tests/test_example_launcher_plugin.py
@@ -0,0 +1,30 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+import pytest
+
+from hydra.plugins import Launcher
+from hydra._internal.plugins import Plugins
+from hydra.test_utils.launcher_common_tests import LauncherTestSuite
+from hydra_plugins.example_launcher import ExampleLauncher
+
+# This has to be included here for the LauncherTestSuite to work.
+# noinspection PyUnresolvedReferences
+from hydra.test_utils.test_utils import sweep_runner  # noqa: F401
+
+
+def test_discovery():
+    """
+    Tests that this plugin can be discovered via the plugins subsystem when looking for Launchers
+    :return:
+    """
+    assert ExampleLauncher.__name__ in [x.__name__ for x in Plugins.discover(Launcher)]
+
+
+@pytest.mark.parametrize("launcher_name, overrides", [("example", [])])
+class TestExampleLauncher(LauncherTestSuite):
+    """
+    Run the Launcher test suite on this launcher.
+    Note that hydra/launcher/example.yaml should be provided by this launcher.
+    """
+
+    pass
diff --git a/plugins/examples/example_plugin/tests/__init__.py b/plugins/examples/example_plugin/tests/__init__.py
new file mode 100644
index 0000000000..168f9979a4
--- /dev/null
+++ b/plugins/examples/example_plugin/tests/__init__.py
@@ -0,0 +1,1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved