Repository: ray-project/ray

Issue:
[tune] shim instantiation of search algorithms
### Describe your feature request

allow creating a search algorithm for `tune.run` given:
- (maybe; see option 1 below) a string defining the search algorithm to use
- (maybe; see option 2 below) the search space configuration

### API
there will be two ways to use shim instantiation (option 3 from the design doc):

#### Option 1
(notice how the search algorithm definition doesn't require redefining the search configuration when initializing `HyperOptSearch`)

> ```python
> from ray import tune
> from ray.tune.param import Float, Integer, Categorical
> from ray.tune.param import Grid, Uniform, LogUniform, Normal
> 
> space = {
>     "lr": Float(1e-4, 1e-1).LogUniform(),
>     "num_epochs": Integer(5, 10).Uniform(),
>     "temperature": Float().Normal(1, 0.1),
>     "batch_size": Categorical([32, 64]).Grid()
> }
> 
> tune.run(
>     trainable,
>     space,
>     search_alg=HyperOptSearch(metric="mean_loss"))
> ```

#### Option 2

> ```python
> from ray import tune
> from ray.tune.param import Float, Integer, Categorical
> from ray.tune.param import Grid, Uniform, LogUniform, Normal
> 
> space = {
>     "lr": Float(1e-4, 1e-1).LogUniform(),
>     "num_epochs": Integer(5, 10).Uniform(),
>     "temperature": Float().Normal(1, 0.1),
>     "batch_size": Categorical([32, 64]).Grid()
> }
> 
> search_alg = tune.create_searcher("HyperOpt", space,
>     metric="mean_loss")
> 
> tune.run(
>     trainable,
>     search_alg=search_alg)
> ```

---

related: #9969, #10401, #10444

CC @richardliaw @krfricke 



Discussion:
```python
from ray import tune
from ray.tune.param import Float, Integer, Categorical
from ray.tune.param import Grid, Uniform, LogUniform, Normal

space = {
    "lr": Float(1e-4, 1e-1).LogUniform(),
    "num_epochs": Integer(5, 10).Uniform(),
    "temperature": Float().Normal(1, 0.1),
    "batch_size": Categorical([32, 64]).Grid()
}

###############################
# Shim API - Option 3
################################
tune.run(
    trainable,
    space,
    search_alg=tune.create_searcher("HyperOpt", metric="mean_loss"))

# This should also work.
# tune.run(
#    trainable,
#    space,
#    search_alg=HyperOptSearch(metric="mean_loss"))
```
untested; we should use an if-statement instead of `getattr`:

```python
def create_searcher(search_alg: str, **kwargs):
    from ray.tune import suggest
    return getattr(suggest, f'{search_alg.lower()}.{search_alg}Search')(None, **kwargs)
    # in HyperOptSearch.__init__ we check if space is None, and if so, set HyperOptSearch.domain to None
    # in tune.run, we check if `HyperOptSearch.domain` is None, and if so, we set assign the proper config based on the config kwarg using PR #10444
```
instead of this getattr, can you just create a dictionary?

```python
def import_hyperopt_search():
    from ray.tune.suggest import HyperOptSearch
    return HyperOptSearch

SEARCH_ALG_IMPORT = {
	"hyperopt": import_hyperopt_search,
    "dragonfly": import_dragonfly_search,
}
```

This avoids this dependency on naming convention.
```python
def create_searcher(search_alg, **kwargs):
    # TODO: docstring
    def _import_ax_search():
        from ray.tune.suggest.ax import AxSearch
        return AxSearch

    def _import_dragonfly_search():
        from ray.tune.suggest.dragonfly import DragonflySearch
        return DragonflySearch

    def _import_skopt_search():
        from ray.tune.suggest.skopt import SkOptSearch
        return SkOptSearch

    def _import_hyperopt_search():
        from ray.tune.suggest import HyperOptSearch
        return HyperOptSearch

    def _import_bayesopt_search():
        from ray.tune.suggest.bayesopt import BayesOptSearch
        return BayesOptSearch

    def _import_bohb_search():
        from ray.tune.suggest.bohb import TuneBOHB
        return TuneBOHB

    def _import_nevergrad_search():
        from ray.tune.suggest.nevergrad import NevergradSearch
        return NevergradSearch

    def _import_optuna_search():
        from ray.tune.suggest.optuna import OptunaSearch
        return OptunaSearch

    def _import_zoopt_search():
        from ray.tune.suggest.zoopt import ZOOptSearch
        return ZOOptSearch

    def _import_sigopt_search():
        from ray.tune.suggest.sigopt import SigOptSearch
        return SigOptSearch

    SEARCH_ALG_IMPORT = {
        "ax": _import_ax_search,
        "dragonfly": _import_dragonfly_search,
        "skopt": _import_skopt_search,
        "hyperopt": _import_hyperopt_search,
        "bayesopt": _import_bayesopt_search,
        "bohb": _import_bohb_search,
        "nevergrad": _import_nevergrad_search,
        "optuna": _import_optuna_search,
        "zoopt": _import_zoopt_search,
        "sigopt": _import_sigopt_search,
    }
    return SEARCH_ALG_IMPORT[search_alg](**kwargs)
```
Look good to me! Consider moving some of the default arguments (i.e., metric, mode) also into the wrapper so that it's explicit and captured in the docstring.
could you push a PR @sumanthratna and tag me when you do? Thanks!

Resolving patch:
diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py
--- a/python/ray/tune/__init__.py
+++ b/python/ray/tune/__init__.py
@@ -15,6 +15,8 @@
 from ray.tune.sample import (function, sample_from, uniform, quniform, choice,
                              randint, qrandint, randn, qrandn, loguniform,
                              qloguniform)
+from ray.tune.suggest import create_searcher
+from ray.tune.schedulers import create_scheduler
 
 __all__ = [
     "Trainable", "DurableTrainable", "TuneError", "grid_search",
@@ -24,5 +26,5 @@
     "loguniform", "qloguniform", "ExperimentAnalysis", "Analysis",
     "CLIReporter", "JupyterNotebookReporter", "ProgressReporter", "report",
     "get_trial_dir", "get_trial_name", "get_trial_id", "make_checkpoint_dir",
-    "save_checkpoint", "checkpoint_dir"
+    "save_checkpoint", "checkpoint_dir", "create_searcher", "create_scheduler"
 ]
diff --git a/python/ray/tune/schedulers/__init__.py b/python/ray/tune/schedulers/__init__.py
--- a/python/ray/tune/schedulers/__init__.py
+++ b/python/ray/tune/schedulers/__init__.py
@@ -7,6 +7,69 @@
 from ray.tune.schedulers.pbt import (PopulationBasedTraining,
                                      PopulationBasedTrainingReplay)
 
+
+def create_scheduler(
+        scheduler,
+        metric="episode_reward_mean",
+        mode="max",
+        **kwargs,
+):
+    """Instantiate a scheduler based on the given string.
+
+    This is useful for swapping between different schedulers.
+
+    Args:
+        scheduler (str): The scheduler to use.
+        metric (str): The training result objective value attribute. Stopping
+            procedures will use this attribute.
+        mode (str): One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        **kwargs: Additional parameters.
+            These keyword arguments will be passed to the initialization
+            function of the chosen class.
+    Returns:
+        ray.tune.schedulers.trial_scheduler.TrialScheduler: The scheduler.
+    Example:
+        >>> scheduler = tune.create_scheduler('pbt')
+    """
+
+    def _import_async_hyperband_scheduler():
+        from ray.tune.schedulers import AsyncHyperBandScheduler
+        return AsyncHyperBandScheduler
+
+    def _import_median_stopping_rule_scheduler():
+        from ray.tune.schedulers import MedianStoppingRule
+        return MedianStoppingRule
+
+    def _import_hyperband_scheduler():
+        from ray.tune.schedulers import HyperBandScheduler
+        return HyperBandScheduler
+
+    def _import_hb_bohb_scheduler():
+        from ray.tune.schedulers import HyperBandForBOHB
+        return HyperBandForBOHB
+
+    def _import_pbt_search():
+        from ray.tune.schedulers import PopulationBasedTraining
+        return PopulationBasedTraining
+
+    SCHEDULER_IMPORT = {
+        "async_hyperband": _import_async_hyperband_scheduler,
+        "median_stopping_rule": _import_median_stopping_rule_scheduler,
+        "hyperband": _import_hyperband_scheduler,
+        "hb_bohb": _import_hb_bohb_scheduler,
+        "pbt": _import_pbt_search,
+    }
+    scheduler = scheduler.lower()
+    if scheduler not in SCHEDULER_IMPORT:
+        raise ValueError(
+            f"Search alg must be one of {list(SCHEDULER_IMPORT)}. "
+            f"Got: {scheduler}")
+
+    SchedulerClass = SCHEDULER_IMPORT[scheduler]()
+    return SchedulerClass(metric=metric, mode=mode, **kwargs)
+
+
 __all__ = [
     "TrialScheduler", "HyperBandScheduler", "AsyncHyperBandScheduler",
     "ASHAScheduler", "MedianStoppingRule", "FIFOScheduler",
diff --git a/python/ray/tune/suggest/__init__.py b/python/ray/tune/suggest/__init__.py
--- a/python/ray/tune/suggest/__init__.py
+++ b/python/ray/tune/suggest/__init__.py
@@ -5,6 +5,94 @@
 from ray.tune.suggest.variant_generator import grid_search
 from ray.tune.suggest.repeater import Repeater
 
+
+def create_searcher(
+        search_alg,
+        metric="episode_reward_mean",
+        mode="max",
+        **kwargs,
+):
+    """Instantiate a search algorithm based on the given string.
+
+    This is useful for swapping between different search algorithms.
+
+    Args:
+        search_alg (str): The search algorithm to use.
+        metric (str): The training result objective value attribute. Stopping
+            procedures will use this attribute.
+        mode (str): One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        **kwargs: Additional parameters.
+            These keyword arguments will be passed to the initialization
+            function of the chosen class.
+    Returns:
+        ray.tune.suggest.Searcher: The search algorithm.
+    Example:
+        >>> search_alg = tune.create_searcher('ax')
+    """
+
+    def _import_ax_search():
+        from ray.tune.suggest.ax import AxSearch
+        return AxSearch
+
+    def _import_dragonfly_search():
+        from ray.tune.suggest.dragonfly import DragonflySearch
+        return DragonflySearch
+
+    def _import_skopt_search():
+        from ray.tune.suggest.skopt import SkOptSearch
+        return SkOptSearch
+
+    def _import_hyperopt_search():
+        from ray.tune.suggest.hyperopt import HyperOptSearch
+        return HyperOptSearch
+
+    def _import_bayesopt_search():
+        from ray.tune.suggest.bayesopt import BayesOptSearch
+        return BayesOptSearch
+
+    def _import_bohb_search():
+        from ray.tune.suggest.bohb import TuneBOHB
+        return TuneBOHB
+
+    def _import_nevergrad_search():
+        from ray.tune.suggest.nevergrad import NevergradSearch
+        return NevergradSearch
+
+    def _import_optuna_search():
+        from ray.tune.suggest.optuna import OptunaSearch
+        return OptunaSearch
+
+    def _import_zoopt_search():
+        from ray.tune.suggest.zoopt import ZOOptSearch
+        return ZOOptSearch
+
+    def _import_sigopt_search():
+        from ray.tune.suggest.sigopt import SigOptSearch
+        return SigOptSearch
+
+    SEARCH_ALG_IMPORT = {
+        "ax": _import_ax_search,
+        "dragonfly": _import_dragonfly_search,
+        "skopt": _import_skopt_search,
+        "hyperopt": _import_hyperopt_search,
+        "bayesopt": _import_bayesopt_search,
+        "bohb": _import_bohb_search,
+        "nevergrad": _import_nevergrad_search,
+        "optuna": _import_optuna_search,
+        "zoopt": _import_zoopt_search,
+        "sigopt": _import_sigopt_search,
+    }
+    search_alg = search_alg.lower()
+    if search_alg not in SEARCH_ALG_IMPORT:
+        raise ValueError(
+            f"Search alg must be one of {list(SEARCH_ALG_IMPORT)}. "
+            f"Got: {search_alg}")
+
+    SearcherClass = SEARCH_ALG_IMPORT[search_alg]()
+    return SearcherClass(metric=metric, mode=mode, **kwargs)
+
+
 __all__ = [
     "SearchAlgorithm", "Searcher", "BasicVariantGenerator", "SearchGenerator",
     "grid_search", "Repeater", "ConcurrencyLimiter"


Test patch:
diff --git a/python/ray/tune/tests/test_api.py b/python/ray/tune/tests/test_api.py
--- a/python/ray/tune/tests/test_api.py
+++ b/python/ray/tune/tests/test_api.py
@@ -14,7 +14,8 @@
 from ray.tune import (DurableTrainable, Trainable, TuneError, Stopper,
                       EarlyStopping)
 from ray.tune import register_env, register_trainable, run_experiments
-from ray.tune.schedulers import TrialScheduler, FIFOScheduler
+from ray.tune.schedulers import (TrialScheduler, FIFOScheduler,
+                                 AsyncHyperBandScheduler)
 from ray.tune.trial import Trial
 from ray.tune.result import (TIMESTEPS_TOTAL, DONE, HOSTNAME, NODE_IP, PID,
                              EPISODES_TOTAL, TRAINING_ITERATION,
@@ -24,6 +25,8 @@
 from ray.tune.experiment import Experiment
 from ray.tune.resources import Resources
 from ray.tune.suggest import grid_search
+from ray.tune.suggest.hyperopt import HyperOptSearch
+from ray.tune.suggest.ax import AxSearch
 from ray.tune.suggest._mock import _MockSuggestionAlgorithm
 from ray.tune.utils import (flatten_dict, get_pinned_object,
                             pin_in_object_store)
@@ -1105,6 +1108,30 @@ def train(config, reporter):
             self.assertIn("LOG_STDERR", content)
 
 
+class ShimCreationTest(unittest.TestCase):
+    def testCreateScheduler(self):
+        kwargs = {"metric": "metric_foo", "mode": "min"}
+
+        scheduler = "async_hyperband"
+        shim_scheduler = tune.create_scheduler(scheduler, **kwargs)
+        real_scheduler = AsyncHyperBandScheduler(**kwargs)
+        assert type(shim_scheduler) is type(real_scheduler)
+
+    def testCreateSearcher(self):
+        kwargs = {"metric": "metric_foo", "mode": "min"}
+
+        searcher_ax = "ax"
+        shim_searcher_ax = tune.create_searcher(searcher_ax, **kwargs)
+        real_searcher_ax = AxSearch(space=[], **kwargs)
+        assert type(shim_searcher_ax) is type(real_searcher_ax)
+
+        searcher_hyperopt = "hyperopt"
+        shim_searcher_hyperopt = tune.create_searcher(searcher_hyperopt,
+                                                      **kwargs)
+        real_searcher_hyperopt = HyperOptSearch({}, **kwargs)
+        assert type(shim_searcher_hyperopt) is type(real_searcher_hyperopt)
+
+
 if __name__ == "__main__":
     import pytest
     import sys