Repository: facebookresearch/nevergrad

Issue:
Interfacing Nevergrad with more Bayesian optimization libraries ?
We have an interface with a single Bayesian optimization library.
We have no parallel version, or no version for the noisy case.
We might either
- explore a better use of the existing bayesian optimization library
- implement an interface towards another bayesian optimization library

The example of SPSA is one of the simplest examples of implementation of an algorithm, so this could be used as a model regarding how to include a new algorithm: https://github.com/facebookresearch/nevergrad/blob/2ef9cb8cc3e1b769745f05dc1c8689386459e88c/nevergrad/optimization/optimizerlib.py#L686

(_internal_ask should provide the next point to be tested, this is the key part of implementing an optimization algorithm)


Discussion:
e.g. Turbo ?
Hi @teytaud. I am interested to add interface for two widely-used bayesian optimization library (for hyper-parameters tuning): [SMAC](https://github.com/automl/SMAC3) and [Hyperopt](https://github.com/hyperopt/hyperopt). 
It may also be interesting for #618. What do you think ?
Each of those (SMAC, HyperOpt, and #618) is super useful, and to the best of my knowledge there is nobody on it!
Ok, I will take up this issue.

Resolving patch:
diff --git a/nevergrad/benchmark/experiments.py b/nevergrad/benchmark/experiments.py
--- a/nevergrad/benchmark/experiments.py
+++ b/nevergrad/benchmark/experiments.py
@@ -81,7 +81,7 @@ def mltuning(seed: tp.Optional[int] = None, overfitter: bool = False, seq: bool
 
     # First, a few functions with constraints.
     optims = ["Shiwa", "DE", "DiscreteOnePlusOne", "PortfolioDiscreteOnePlusOne", "CMA", "MetaRecentering",
-              "DoubleFastGADiscreteOnePlusOne", "PSO", "BO", "MetaTuneRecentering"]
+              "DoubleFastGADiscreteOnePlusOne", "PSO", "BO", "MetaTuneRecentering", "HyperOpt"]
     if default_optims is not None:
         optims = default_optims
     for dimension in [None, 1, 2, 3]:
diff --git a/nevergrad/optimization/externalbo.py b/nevergrad/optimization/externalbo.py
new file mode 100644
--- /dev/null
+++ b/nevergrad/optimization/externalbo.py
@@ -0,0 +1,235 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import numpy as np
+from . import base
+from .base import IntOrParameter
+import nevergrad.common.typing as tp
+from nevergrad.parametrization import transforms
+from nevergrad.parametrization import parameter as p
+
+import hyperopt # type: ignore
+from hyperopt import hp, Trials, Domain, tpe # type: ignore
+
+def _hp_parametrization_to_dict(x, **kwargs):
+    if isinstance(x, p.Instrumentation):
+        x_dict = kwargs.get("default", {})
+        for idx_param in range(len(x[0].value)):
+            x_dict.update(_hp_parametrization_to_dict(x[0][idx_param], name=str(idx_param)))
+        for name in x[1].value.keys():
+            x_dict.update(_hp_parametrization_to_dict(x[1][name], name=name))
+        return x_dict
+    elif isinstance(x, (p.Log, p.Scalar)):
+        return {kwargs["name"]: [x.value]}
+    elif isinstance(x, p.Choice):
+        x_dict = {}
+        for i in range(len(x.choices)):
+            if x.value == x.choices[i].value:
+                x_dict[kwargs["name"]] = [i]
+                if isinstance(x.choices[i], (p.Log, p.Scalar, p.Choice)):
+                    x_dict[kwargs["name"] + f"__{i}"] = [x.choices[i].value]
+                elif isinstance(x.choices[i], p.Instrumentation):
+                    x_dict.update(_hp_parametrization_to_dict(x.choices[i]))
+        return x_dict
+
+
+def _hp_dict_to_parametrization(x):
+    if isinstance(x, dict) and ("args" in x) and ("kwargs" in x):
+        x["args"] = tuple([_hp_dict_to_parametrization(x["args"][str(i)]) for i in range(len(x["args"]))])
+        x["kwargs"] = {k: _hp_dict_to_parametrization(v) for k, v in x["kwargs"].items()}
+        return (x["args"], x["kwargs"])
+    return x
+
+
+def _get_search_space(param_name, param):
+    if isinstance(param, p.Instrumentation):
+        space = {}
+        space["args"] = {
+                str(idx_param): _get_search_space(str(idx_param), param[0][idx_param]) # type: ignore
+                for idx_param in range(len(param[0].value))
+        }
+        space["kwargs"] = {
+                param_name: _get_search_space(param_name, param[1][param_name]) # type: ignore
+                for param_name in param[1].value.keys()
+        }
+        return space
+
+    elif isinstance(param, (p.Log, p.Scalar)):
+        if (param.bounds[0][0] is None) or (param.bounds[1][0] is None):
+            if isinstance(param, p.Scalar) and not param.integer:
+                return hp.lognormal(label=param_name, mu=0, sigma=1)
+            raise ValueError(f"Scalar {param_name} not bounded.")
+        elif isinstance(param, p.Log):
+            return hp.loguniform(label=param_name,
+                                 low=np.log(param.bounds[0][0]),
+                                 high=np.log(param.bounds[1][0]))
+        elif isinstance(param, p.Scalar):
+            if param.integer:
+                return hp.randint(label=param_name,
+                                  low=int(param.bounds[0][0]),
+                                  high=int(param.bounds[1][0]))
+            else:
+                return hp.uniform(label=param_name,
+                                  low=param.bounds[0][0],
+                                  high=param.bounds[1][0])
+
+    elif isinstance(param, p.Choice):
+        list_types = [type(param.choices[i]) for i in range(len(param.choices))
+                      if not isinstance(param.choices[i], (p.Instrumentation, p.Constant))]
+
+        if len(list_types) != len(set(list_types)): raise NotImplementedError
+        return hp.choice(param_name,
+                         [_get_search_space(param_name + "__" + str(i), param.choices[i])
+                          for i in range(len(param.choices))])
+
+    elif isinstance(param, p.Constant):
+        return param.value
+
+    # Hyperopt do not support array
+    raise NotImplementedError
+
+
+class _HyperOpt(base.Optimizer):
+    # pylint: disable=too-many-instance-attributes
+    def __init__(self, parametrization: IntOrParameter,
+                 budget: tp.Optional[int] = None,
+                 num_workers: int = 1,
+                 *,
+                 prior_weight: float = 1.0,
+                 n_startup_jobs: int = 20,
+                 n_EI_candidates: int = 24,
+                 gamma: float = 0.25,
+                 verbose: bool = False) -> None:
+        super().__init__(parametrization, budget=budget, num_workers=num_workers)
+        try:
+            # try to convert parametrization to hyperopt search space
+            if not isinstance(self.parametrization, p.Instrumentation): raise NotImplementedError
+            self.space = _get_search_space(self.parametrization.name, self.parametrization)
+            self._transform = None
+        except NotImplementedError:
+            self._transform = transforms.ArctanBound(0, 1)
+            self.space = {f"x_{i}": hp.uniform(f"x_{i}", 0, 1) for i in range(self.dimension)}
+
+        self.trials = Trials()
+        self.domain = Domain(fn=None, expr=self.space, pass_expr_memo_ctrl=False)
+        self.tpe_args = {
+            "prior_weight": prior_weight,
+            "n_startup_jobs": n_startup_jobs,
+            "n_EI_candidates": n_EI_candidates,
+            "gamma": gamma,
+            "verbose": verbose
+        }
+
+    def _internal_ask_candidate(self) -> p.Parameter:
+        # Inspired from FMinIter class (hyperopt)
+        next_id = self.trials.new_trial_ids(1)
+        new_trial = tpe.suggest(next_id, self.domain, self.trials,
+                                self._rng.randint(2 ** 31 - 1), **self.tpe_args)[0]
+        self.trials.insert_trial_doc(new_trial)
+        self.trials.refresh()
+
+        candidate = self.parametrization.spawn_child()
+
+        if self._transform:
+            data = np.array([new_trial["misc"]["vals"][f"x_{i}"][0] for i in range(self.dimension)])
+            candidate = candidate.set_standardized_data(self._transform.backward(data))
+
+            # For consistency, we need to update hyperopt history
+            # when standardized data is changed
+            if any(data != self._transform.forward(candidate.get_standardized_data(reference=self.parametrization))):
+                for it, val in enumerate(self._transform.forward(candidate.get_standardized_data(reference=self.parametrization))):
+                    self.trials._dynamic_trials[next_id[0]]["misc"]["vals"][f"x_{it}"][0] = val
+        else:
+            spec = hyperopt.base.spec_from_misc(new_trial["misc"])
+            config = hyperopt.space_eval(self.space, spec)
+            candidate.value = _hp_dict_to_parametrization(config)
+
+        candidate._meta["trial_id"] = new_trial["tid"]
+        return candidate
+
+    def _internal_tell_candidate(self, candidate: p.Parameter, loss: float) -> None:
+        result = {"loss": loss, "status": "ok"}
+        assert "trial_id" in candidate._meta
+        tid = candidate._meta["trial_id"]
+        assert self.trials._dynamic_trials[tid]["state"] == hyperopt.JOB_STATE_NEW
+
+        now = hyperopt.utils.coarse_utcnow()
+        self.trials._dynamic_trials[tid]["book_time"] = now
+        self.trials._dynamic_trials[tid]["refresh_time"] = now
+        self.trials._dynamic_trials[tid]["state"] = hyperopt.JOB_STATE_DONE
+        self.trials._dynamic_trials[tid]["result"] = result
+        self.trials._dynamic_trials[tid]["refresh_time"] = hyperopt.utils.coarse_utcnow()
+        self.trials.refresh()
+
+    def _internal_tell_not_asked(self, candidate: p.Parameter, loss: float) -> None:
+        next_id = self.trials.new_trial_ids(1)
+        new_trial = hyperopt.rand.suggest(next_id,
+                                          self.domain,
+                                          self.trials, self._rng.randint(2 ** 31 - 1))
+        self.trials.insert_trial_docs(new_trial)
+        self.trials.refresh()
+        tid = next_id[0]
+
+        if self._transform:
+            data = candidate.get_standardized_data(reference=self.parametrization)
+            data = self._transform.forward(data)
+            self.trials._dynamic_trials[tid]["misc"]["vals"] = {f"x_{i}": [data[i]] for i in range(len(data))}
+        else:
+            null_config: dict = {k: [] for k in self.trials._dynamic_trials[tid]["misc"]["vals"].keys()}
+            new_vals: dict = _hp_parametrization_to_dict(candidate, default=null_config)
+            self.trials._dynamic_trials[tid]["misc"]["vals"] = new_vals
+
+        self.trials.refresh()
+        candidate._meta["trial_id"] = tid
+        self._internal_tell_candidate(candidate, loss)
+
+
+class ParametrizedHyperOpt(base.ConfiguredOptimizer):
+    # pylint: disable=too-many-instance-attributes
+    """Hyperopt: Distributed Asynchronous Hyper-parameter Optimization.
+    This class is a wrapper over the `hyperopt <https://github.com/hyperopt/hyperopt>`_ package.
+
+    Parameters
+    ----------
+    parametrization: int or Parameter
+        Parametrization object
+    budget: int
+        Number of iterations
+    num_workers: int
+        Number of workers
+    prior_weight: float (default 1.0)
+        Smoothing factor to avoid having zero probabilities
+    n_startup_jobs: int (default 20)
+        Number of random uniform suggestions at initialization
+    n_EI_candidates: int (default 24)
+        Number of generated candidates during EI maximization
+    gamma: float (default 0.25)
+        Threshold to split between l(x) and g(x), see eq. 2 in
+
+    verbose: bool (default False)
+        Hyperopt algorithm verbosity
+
+    Note
+    ----
+    HyperOpt is described in Bergstra, James S., et al.
+    "Algorithms for hyper-parameter optimization."
+    Advances in neural information processing systems. 2011
+    """
+
+    no_parallelization = False
+
+    # pylint: disable=unused-argument
+    def __init__(
+        self,
+        *,
+        prior_weight: float = 1.0,
+        n_startup_jobs: int = 20,
+        n_EI_candidates: int = 24,
+        gamma: float = 0.25,
+        verbose: bool = False
+    ) -> None:
+        super().__init__(_HyperOpt, locals())
+
+HyperOpt = ParametrizedHyperOpt().set_name("HyperOpt", register=True)
diff --git a/nevergrad/optimization/optimizerlib.py b/nevergrad/optimization/optimizerlib.py
--- a/nevergrad/optimization/optimizerlib.py
+++ b/nevergrad/optimization/optimizerlib.py
@@ -31,6 +31,11 @@
 from .oneshot import *  # noqa: F403
 from .recastlib import *  # noqa: F403
 
+try:
+    from .externalbo import HyperOpt # pylint: disable=unused-import
+except ModuleNotFoundError:
+    pass
+
 # run with LOGLEVEL=DEBUG for more debug information
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 logger = logging.getLogger(__name__)


Test patch:
diff --git a/nevergrad/optimization/test_externalbo.py b/nevergrad/optimization/test_externalbo.py
new file mode 100644
--- /dev/null
+++ b/nevergrad/optimization/test_externalbo.py
@@ -0,0 +1,118 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import pytest
+import numpy as np
+import nevergrad as ng
+from .optimizerlib import registry
+from .externalbo import _hp_parametrization_to_dict, _hp_dict_to_parametrization
+
+@pytest.mark.parametrize(  # type: ignore
+    "parametrization,has_transform",
+    [(ng.p.Choice(list(range(10))), True),
+    (ng.p.Scalar(lower=0, upper=1), True),
+    (ng.p.Scalar(lower=0, upper=10).set_integer_casting(), True),
+    (ng.p.Log(lower=1e-3, upper=1e3), True),
+    (ng.p.Array(init=np.zeros(10)), True),
+    (ng.p.Instrumentation(
+                        ng.p.Scalar(lower=0, upper=1),
+                        a=ng.p.Choice(list(range(10)))
+    ), False),
+    (ng.p.Instrumentation(a=ng.p.Choice([
+                         ng.p.Scalar(lower=0, upper=1),
+                         ng.p.Scalar(lower=100, upper=1000)
+    ])), True),
+    (ng.p.Instrumentation(a=ng.p.Choice([
+                         ng.p.Choice(list(range(10))),
+                         ng.p.Scalar(lower=0, upper=1),
+    ])), False),
+    (ng.p.Instrumentation(a=ng.p.Choice([
+                             ng.p.Instrumentation(b=ng.p.Choice(list(range(10))),
+                                                  c=ng.p.Log(lower=1e-3, upper=1e3)),
+                             ng.p.Instrumentation(d=ng.p.Scalar(lower=0, upper=1),
+                                                  e=ng.p.Log(lower=1e-3, upper=1e3)),
+    ])), False)
+    ]
+)
+def test_hyperopt(parametrization, has_transform) -> None:
+    optim1 = registry["HyperOpt"](parametrization=parametrization, budget=5)
+    optim2 = registry["HyperOpt"](parametrization=parametrization.copy(), budget=5)
+    for it in range(4):
+        cand = optim1.ask()
+        optim1.tell(cand, 0) # Tell asked
+        del cand._meta["trial_id"]
+        optim2.tell(cand, 0) # Tell not asked
+        assert optim1.trials._dynamic_trials[it]["misc"]["vals"] == optim2.trials._dynamic_trials[it]["misc"]["vals"] # type: ignore
+
+    assert optim1.trials.new_trial_ids(1) == optim2.trials.new_trial_ids(1) # type: ignore
+    assert optim1.trials.new_trial_ids(1)[0] == (it + 2) # type: ignore
+    assert (optim1._transform is not None) == has_transform # type: ignore
+
+    # Test parallelization
+    opt = registry["HyperOpt"](parametrization=parametrization, budget=30, num_workers=5)
+    for k in range(40):
+        cand = opt.ask()
+        if not k:
+            opt.tell(cand, 1)
+
+
+@pytest.mark.parametrize(  # type: ignore
+    "parametrization,values",
+    [(ng.p.Instrumentation(a=ng.p.Choice([
+         ng.p.Choice(list(range(10))),
+         ng.p.Scalar(lower=0, upper=1)])),
+        [
+            (
+                ((), {'a': 0.5}),
+                {'a': [1], 'a__1': [0.5]},
+                {"args": {}, "kwargs": {"a": 0.5}}
+            ),
+            (
+                ((), {'a': 1}),
+                {'a': [0], 'a__0': [1]},
+                {"args": {}, "kwargs": {"a": 1}}
+            )
+        ]
+    ),
+    (ng.p.Instrumentation(
+        ng.p.Scalar(lower=0, upper=1),
+        a=ng.p.Choice(list(range(10)))),
+        [
+            (
+                ((0.5,), {'a': 3}),
+                {'0': [0.5], 'a': [3]},
+                {"args": {'0': 0.5}, "kwargs": {"a": 3}}
+            ),
+            (
+                ((0.99,), {'a': 0}),
+                {'0': [0.99], 'a': [0]},
+                {"args": {'0': 0.99}, "kwargs": {"a": 0}}
+            ),
+        ]
+    ),
+    (ng.p.Instrumentation(a=ng.p.Choice([
+         ng.p.Instrumentation(b=ng.p.Choice(list(range(10))),
+                              c=ng.p.Log(lower=1e-3, upper=1e3)),
+         ng.p.Instrumentation(d=ng.p.Scalar(lower=0, upper=1),
+                              e=ng.p.Log(lower=1e-3, upper=1e3))])),
+        [
+            (
+                ((), {'a': ((), {'d': 0.5, 'e': 1.0})}),
+                {'a': [1], 'd': [0.5], 'e': [1.0]},
+                {"args": {}, "kwargs": {"a": {"args": {}, "kwargs": {"d": 0.5, "e": 1.0}}}}
+            ),
+            (
+                ((), {'a': ((), {'b': 0, 'c': 0.014})}),
+                {'a': [0], 'b': [0], 'c': [0.014]},
+                {"args": {}, "kwargs": {"a": {"args": {}, "kwargs": {"b": 0, "c": 0.014}}}}
+            )
+        ]
+    )]
+)
+def test_hyperopt_helpers(parametrization, values):
+    for val, dict_val, hyperopt_val in values:
+        parametrization.value = val
+        assert _hp_parametrization_to_dict(parametrization) == dict_val
+        assert _hp_dict_to_parametrization(hyperopt_val) == parametrization.value