Repository: facebookresearch/nevergrad

Issue:
Add a list of optimization algorithms to the doc
Thanks for all the hard work on nevergrad, it's very useful.

It would be nice to have some documentation on the currently supported optimizers. The comments in [optimizerlib](https://github.com/facebookresearch/nevergrad/blob/master/nevergrad/optimization/optimizerlib.py) already contain some details - maybe would it be possible to automatically generate some documentation out of it (using some tools from [this list](https://wiki.python.org/moin/DocumentationTools) ?).

Ideally some documentation could also be added to the optimizers from [recastlib](https://github.com/facebookresearch/nevergrad/blob/master/nevergrad/optimization/recastlib.py).

Discussion:

Patch:
diff --git a/README.md b/README.md
index 985979ea..8e1c6197 100644
--- a/README.md
+++ b/README.md
@@ -85,6 +85,35 @@ recommendation = optimizer.provide_recommendation()
 
 Please make sure that your function returns a float, and that you indeed want to perform minimization and not maximization ;)
 
+### Choosing an optimizer
+
+**You can print the full list of optimizers** with:
+```
+from nevergrad.optimization import registry
+print(sorted(registry.keys()))
+```
+
+All algorithms have strenghts and weaknesses. Questionable rules of thumb could be:
+- `TwoPointsDE` is excellent in many cases, including very high `num_workers`.
+- `DiscretePortfolioOnePlusOne` is excellent in discrete settings of mixed settings when high precision on parameters is not relevant; it's possibly a good choice for hyperparameter choice.
+- `OnePlusOne` is a simple robust method for continuous parameters with `num_workers` < 8.
+- `CMA` is excellent for control (e.g. neurocontrol) when the environment is not very noisy (num_workers ~50 ok) and when the budget is large (e.g. 1000 x the dimension).
+- `TBPSA` is excellent for problems corrupted by noise, in particular overparametrized (neural) ones; very high `num_workers` ok).
+- `PSO` is excellent in terms of robustness, high `num_workers` ok.
+- `ScrHammersleySearchPlusMiddlePoint` is excellent for super parallel cases (fully one-shot, i.e. `num_workers` = budget included) or for very multimodal cases (such as some of our MLDA problems); don't use softmax with this optimizer.
+- `RandomSearch` is the classical random search baseline; don't use softmax with this optimizer.
+
+**Optimizing machine learning hyperparameters**:
+
+When optimizing hyperparameters as e.g. in machine learning. If you don't know what to do, 
+- use `SoftmaxCategorical` for discrete variables (see below). 
+- use `TwoPointsDE` with `num_workers` equal to the number of workers available to you.
+See https://github.com/facebookresearch/nevergrad/blob/master/docs/machinelearning.md for more.
+
+Or if you want something more aimed at robustly outperforming random search:
+- use `OrderedDiscrete` for discrete variables, taking care that the default value is in the middle.
+- Use `ScrHammersleySearchPlusMiddlePoint` (`PlusMiddlePoint` only if you have continuous parameters or good default values for discrete parameters).
+
 ## Benchmarks
 
 The benchmark tools aim at providing a way to evaluate optimizers on a large range of settings. They provide a way to run the optimizers on all settings and record the results, as well as ways to plot the results of the experiments.
diff --git a/nevergrad/benchmark/execution.py b/nevergrad/benchmark/execution.py
new file mode 100644
index 00000000..63852189
--- /dev/null
+++ b/nevergrad/benchmark/execution.py
@@ -0,0 +1,77 @@
+import heapq
+from typing import List, Callable, Any, NamedTuple
+from ..functions.base import PostponedObject  # this object only serves to provide delays that the executor must use to order jobs
+
+
+class MockedSteadyJob:
+    """Job returned by the MockedSteadyExecutor, with the usual
+    "done()" and "result()" methods.
+    """
+
+    def __init__(self, value: float, executor: "MockedSteadyExecutor") -> None:
+        self._value = value
+        self._done = False
+        self._is_read = False
+        self._executor = executor
+
+    def done(self) -> bool:
+        return self._done
+
+    def result(self) -> float:
+        """Return the result if "done()" is true, and raises
+        a RuntimeError otherwise.
+        """
+        if not self._done:
+            raise RuntimeError("Asking result which is not ready")
+        self._is_read = True
+        self._executor.update_queue()
+        return self._value
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}<value={self._value}, done={self._done}, read={self._is_read}>)'
+
+
+class OrderedJobs(NamedTuple):
+    """Handle for sorting jobs by release_time (or submission order in case of tie)
+    """
+    release_time: float
+    order: int
+    job: MockedSteadyJob
+
+
+class MockedSteadyExecutor:
+    """Executor that mocks a steady state, by only providing 1 job at a time which is done() while
+    not having been "read" (i.e. "result()" method was not executed).
+    This ensures we control the order of update of the optimizer for benchmarking.
+
+    Additionally, "delays" can be provided by the function so that jobs are not "done()" by order of
+    submission. To this end, callables must implement a "computation_time" method.
+    """
+
+    def __init__(self) -> None:
+        self.priority_queue: List[OrderedJobs] = []
+        self._order = 0
+        self._time = 0.
+
+    def submit(self, function: Callable, *args: Any, **kwargs: Any) -> MockedSteadyJob:
+        if self.priority_queue:  # new job may come before the current "next" job
+            self.priority_queue[0].job._done = False
+        value = function(*args, **kwargs)
+        job = MockedSteadyJob(value, self)
+        # compute the delay and add to queue
+        delay = 0.
+        if isinstance(function, PostponedObject):
+            delay = max(0, function.get_postponing_delay((args, kwargs), value))
+        heapq.heappush(self.priority_queue, OrderedJobs(self._time + delay, self._order, job))
+        # update order and "next" finished job
+        self._order += 1
+        self.priority_queue[0].job._done = True
+        return job
+
+    def update_queue(self) -> None:
+        """Called whenever a result is read, so as to activate the next result in line
+        """
+        while self.priority_queue and self.priority_queue[0].job._is_read:
+            self._time = heapq.heappop(self.priority_queue).release_time
+            if self.priority_queue:
+                self.priority_queue[0].job._done = True
diff --git a/nevergrad/benchmark/xpbase.py b/nevergrad/benchmark/xpbase.py
index 55662c9a..51346d4c 100644
--- a/nevergrad/benchmark/xpbase.py
+++ b/nevergrad/benchmark/xpbase.py
@@ -8,18 +8,19 @@ import time
 import random
 import warnings
 import traceback
-from typing import Dict, Union, Callable, Any, Optional, Iterator
+from typing import Dict, Union, Callable, Any, Optional, Iterator, Tuple
 import numpy as np
 from ..common import decorators
 from ..functions import BaseFunction
 from ..optimization import base
 from ..optimization.optimizerlib import registry as optimizer_registry
+from . import execution
 
 
 registry = decorators.Registry()
 
 
-class CallCounter:
+class CallCounter(execution.PostponedObject):
     """Simple wrapper which counts the number
     of calls to a function.
 
@@ -37,6 +38,60 @@ class CallCounter:
         self.num_calls += 1
         return self.func(*args, **kwargs)
 
+    def get_postponing_delay(self, arguments: Tuple[Tuple[Any, ...], Dict[str, Any]], value: float) -> float:
+        """Propagate subfunction delay
+        """
+        if isinstance(self.func, execution.PostponedObject):
+            return self.func.get_postponing_delay(arguments, value)
+        return 0
+
+
+class OptimizerSettings:
+    """Handle for optimizer settings (name, num_workers etc)
+    Optimizers can be instanciated through this class, providing the optimization space dimension.
+
+    Note
+    ----
+    Eventually, this class should be moved to be directly used for defining experiments.
+    """
+
+    def __init__(self, name: str, budget: int, num_workers: int = 1, batch_mode: bool = True) -> None:
+        self._setting_names = [x for x in locals() if x != "self"]
+        assert name in optimizer_registry, f"{name} is not registered"
+        self.name = name
+        self.budget = budget
+        self.num_workers = num_workers
+        self.batch_mode = batch_mode
+
+    def __repr__(self) -> str:
+        return f"Experiment: {self.name}<budget={self.budget}, num_workers={self.num_workers}, batch_mode={self.batch_mode}>"
+
+    @property
+    def is_incoherent(self) -> bool:
+        """Flags settings which are known to be impossible to process.
+        Currently, this means we flag:
+        - no_parallelization optimizers for num_workers > 1
+        """
+        # flag no_parallelization when num_workers greater than 1
+        optimizer = optimizer_registry[self.name]
+        return optimizer.no_parallelization and bool(self.num_workers > 1)
+
+    def instanciate(self, dimension: int) -> base.Optimizer:
+        """Instanciate an optimizer, providing the optimization space dimension
+        """
+        optim: base.Optimizer = optimizer_registry[self.name](dimension=dimension, budget=self.budget, num_workers=self.num_workers)
+        return optim
+
+    def get_description(self) -> Dict[str, Any]:
+        """Returns a dictionary describing the optimizer settings
+        """
+        return {x if x != "name" else "optimizer_name": getattr(self, x) for x in self._setting_names}
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, self.__class__):
+            return all(getattr(self, attr) == getattr(other, attr) for attr in self._setting_names)
+        return False
+
 
 def create_seed_generator(seed: Optional[int]) -> Iterator[Optional[int]]:
     """Create a stream of seeds, independent from the standard random stream.
@@ -74,18 +129,16 @@ class Experiment:
     """
 
     # pylint: disable=too-many-arguments
-    def __init__(self, function: BaseFunction, optimizer_name: str, budget: int, num_workers: int = 1, seed: Optional[int] = None) -> None:
+    def __init__(self, function: BaseFunction, optimizer_name: str, budget: int, num_workers: int = 1,
+                 batch_mode: bool = True, seed: Optional[int] = None) -> None:
         assert isinstance(function, BaseFunction), "All experiment functions should derive from BaseFunction"
         self.function = function
         self.seed = seed  # depending on the inner workings of the function, the experiment may not be repeatable
-        assert optimizer_name in optimizer_registry, f"{optimizer_name} is not registered"
-        self._optimizer_parameters = {"optimizer_name": optimizer_name, "num_workers": num_workers, "budget": budget}
+        self.optimsettings = OptimizerSettings(name=optimizer_name, num_workers=num_workers, budget=budget, batch_mode=batch_mode)
         self.result = {"loss": np.nan, "elapsed_budget": np.nan, "elapsed_time": np.nan, "error": ""}
 
     def __repr__(self) -> str:
-        budget, num_workers, optimizer_name = [self._optimizer_parameters[x] for x in ["budget", "num_workers", "optimizer_name"]]
-        dim = self.function.dimension
-        return f"Experiment: {optimizer_name}(dimension={dim}, budget={budget}, num_workers={num_workers}) on {self.function}"
+        return f"Experiment: {self.optimsettings} (dim={self.function.dimension}) on {self.function}"
 
     @property
     def is_incoherent(self) -> bool:
@@ -93,9 +146,7 @@ class Experiment:
         Currently, this means we flag:
         - no_parallelization optimizers for num_workers > 1
         """
-        # flag no_parallelization when num_workers greater than 1
-        optimizer = optimizer_registry[self._optimizer_parameters["optimizer_name"]]
-        return optimizer.no_parallelization and bool(self._optimizer_parameters["num_workers"] > 1)  # type: ignore
+        return self.optimsettings.is_incoherent
 
     def run(self) -> Dict[str, Any]:
         """Run an experiment with the provided settings
@@ -120,7 +171,7 @@ class Experiment:
             print("\n", file=sys.stderr)
         return self.get_description()
 
-    def _log_results(self, t0: float, num_calls: int, recommendation: float) -> None:
+    def _log_results(self, t0: float, num_calls: int, recommendation: base.ArrayLike) -> None:
         """Internal method for logging results before handling the error
         """
         num_eval = 100  # evaluations of the cost function on the recommendation
@@ -129,9 +180,8 @@ class Experiment:
         t_recommendation = self.function.transform(recommendation)
         self.result["loss"] = sum(self.function.oracle_call(t_recommendation) for _ in range(num_eval)) / num_eval
         self.result["elapsed_budget"] = num_calls
-        if num_calls > self._optimizer_parameters["budget"]:  # type: ignore
-            optim_name = self._optimizer_parameters["optimizer_name"]
-            raise RuntimeError(f"Too much elapsed budget {num_calls} for {optim_name} on {self.function}")
+        if num_calls > self.optimsettings.budget:
+            raise RuntimeError(f"Too much elapsed budget {num_calls} for {self.optimsettings.name} on {self.function}")
 
     def _run_with_error(self, callbacks: Optional[Dict[str, base._OptimCallBack]] = None) -> None:
         """Run an experiment with the provided artificial function and optimizer
@@ -145,20 +195,21 @@ class Experiment:
         if self.seed is not None:
             np.random.seed(self.seed)
             random.seed(self.seed)
-        budget, num_workers, optimizer_name = [self._optimizer_parameters[x] for x in ["budget", "num_workers", "optimizer_name"]]
         # optimizer instantiation can be slow and is done only here to make xp iterators very fast
-        optimizer = optimizer_registry[optimizer_name](dimension=self.function.dimension, budget=budget, num_workers=num_workers)
+        optimizer = self.optimsettings.instanciate(dimension=self.function.dimension)
         if callbacks is not None:
             for name, func in callbacks.items():
                 optimizer.register_callback(name, func)
         assert optimizer.budget is not None, "A budget must be provided"
-        assert optimizer.dimension == self.function.dimension
         t0 = time.time()
         counter = CallCounter(self.function)
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", category=base.InefficientSettingsWarning)  # benchmark do not need to be efficient
+            # use default executor for batch mode (sequential, but mock for steady state
+            # ("production" steady state is a not strictly steady state + does not handle mocked delays)
+            executor: Optional[execution.MockedSteadyExecutor] = None if self.optimsettings.batch_mode else execution.MockedSteadyExecutor()
             try:
-                recommendation = optimizer.optimize(counter, batch_mode=True)
+                recommendation = optimizer.optimize(counter, batch_mode=self.optimsettings.batch_mode, executor=executor)
             except Exception as e:  # pylint: disable=broad-except
                 recommendation = optimizer.provide_recommendation()  # get the recommendation anyway
                 self._log_results(t0, counter.num_calls, recommendation)
@@ -171,10 +222,10 @@ class Experiment:
         """
         summary = dict(self.result, seed=-1 if self.seed is None else self.seed)
         summary.update(self.function.descriptors)
-        summary.update(self._optimizer_parameters)
+        summary.update(self.optimsettings.get_description())
         return summary
 
     def __eq__(self, other: Any) -> bool:
         if not isinstance(other, Experiment):
             return False
-        return self.function == other.function and self._optimizer_parameters == other._optimizer_parameters
+        return self.function == other.function and self.optimsettings == other.optimsettings
diff --git a/nevergrad/common/decorators.py b/nevergrad/common/decorators.py
index d5d0f416..5b459b61 100644
--- a/nevergrad/common/decorators.py
+++ b/nevergrad/common/decorators.py
@@ -20,7 +20,7 @@ class Registry(dict):
         The info variable can be filled up using the register_with_info
         decorator instead of this one.
         """
-        name = obj.__name__
+        name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__
         if name in self:
             raise RuntimeError(f'Encountered a name collision "{name}"')
         self[name] = obj
diff --git a/nevergrad/functions/base.py b/nevergrad/functions/base.py
index 5e90df3f..9497dc1a 100644
--- a/nevergrad/functions/base.py
+++ b/nevergrad/functions/base.py
@@ -4,7 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 
 import abc
-from typing import Dict, Any, Callable, Optional
+from typing import Dict, Any, Callable, Optional, Tuple
 import numpy as np
 
 
@@ -176,3 +176,18 @@ class ArtificiallyNoisyBaseFunction(BaseFunction):  # pylint: disable=abstract-m
                     noise_level *= (1. + x_transf.ravel()[0]*100.)
                 noise = noise_level * np.random.normal(0, 1) * (self.oracle_call(side_point) - fx)
         return fx + noise
+
+
+class PostponedObject(abc.ABC):
+    """Abstract class to inherit in order to notify the steady state benchmark executor that
+    the function implements a delay. This delay will be used while benchmarking to provide the
+    evaluation in a varying order.
+    The main aim of this class is to make sure there is no typo in the name of the special function.
+
+    See benchmark/execution.py for more details. This object is implemented here to avoid circular
+    imports.
+    """
+
+    @abc.abstractmethod
+    def get_postponing_delay(self, arguments: Tuple[Tuple[Any, ...], Dict[str, Any]], value: float) -> float:
+        raise NotImplementedError
diff --git a/nevergrad/functions/corefuncs.py b/nevergrad/functions/corefuncs.py
index 395534c0..975c4e4d 100644
--- a/nevergrad/functions/corefuncs.py
+++ b/nevergrad/functions/corefuncs.py
@@ -3,8 +3,10 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-import numpy as np
 import time
+from typing import Dict, Any, Tuple
+import numpy as np
+from .base import PostponedObject
 from ..instrumentation import discretization
 from ..common.decorators import Registry
 
@@ -46,6 +48,19 @@ def delayedsphere(x: np.ndarray) -> float:
     return float(np.sum(x**2))
 
 
+class DelayedSphere(PostponedObject):
+
+    def __call__(self, x: np.ndarray) -> float:
+        return float(np.sum(x**2))
+
+    def get_postponing_delay(self, arguments: Tuple[Tuple[Any, ...], Dict[str, Any]], value: float) -> float:
+        x = arguments[0][0]
+        return abs(1./x[0]) / 1000. if x[0] != 0. else 0.
+
+
+registry.register(DelayedSphere())
+
+
 @registry.register
 def sphere(x: np.ndarray) -> float:
     return float(np.sum(x**2))
diff --git a/nevergrad/functions/functionlib.py b/nevergrad/functions/functionlib.py
index 61a38ea4..759aa0d5 100644
--- a/nevergrad/functions/functionlib.py
+++ b/nevergrad/functions/functionlib.py
@@ -4,15 +4,16 @@
 # LICENSE file in the root directory of this source tree.
 
 import hashlib
-from typing import List
+from typing import List, Tuple, Any, Dict
 import numpy as np
 from . import utils
 from . import corefuncs
 from .base import ArtificiallyNoisyBaseFunction
+from .base import PostponedObject
 from ..common import tools
 
 
-class ArtificialFunction(ArtificiallyNoisyBaseFunction):
+class ArtificialFunction(ArtificiallyNoisyBaseFunction, PostponedObject):
     """Artificial function object. This allows the creation of functions with different
     dimension and structure to be used for benchmarking in many different settings.
 
@@ -83,6 +84,7 @@ class ArtificialFunction(ArtificiallyNoisyBaseFunction):
         # record necessary info and prepare transforms
         dimension = block_dimension * num_blocks + useless_variables
         super().__init__(dimension, noise_level=noise_level, noise_dissymmetry=noise_dissymmetry)
+        self._func = corefuncs.registry[name]
         self._aggregator = {"max": max, "mean": np.mean, "sum": sum}[aggregator]
         self._transforms: List[utils.Transform] = []
         # special case
@@ -130,10 +132,17 @@ class ArtificialFunction(ArtificiallyNoisyBaseFunction):
         """
         results = []
         for block in x:
-            results.append(corefuncs.registry[self._parameters["name"]](block))
+            results.append(self._func(block))
         return float(self._aggregator(results))
 
     def duplicate(self) -> "ArtificialFunction":
         """Create an equivalent instance, initialized with the same settings
         """
         return self.__class__(**self._parameters)  # type: ignore
+
+    def get_postponing_delay(self, arguments: Tuple[Tuple[Any, ...], Dict[str, Any]], value: float) -> float:
+        """Delay before returning results in steady state mode benchmarks (fake execution time)
+        """
+        if isinstance(self._func, PostponedObject):
+            return self._func.get_postponing_delay(arguments, value)
+        return 0
diff --git a/nevergrad/optimization/recaster.py b/nevergrad/optimization/recaster.py
index d648a446..6bb9c1cf 100644
--- a/nevergrad/optimization/recaster.py
+++ b/nevergrad/optimization/recaster.py
@@ -228,9 +228,7 @@ class RecastOptimizer(base.Optimizer):
         """Returns the underlying optimizer output if provided (ie if the optimizer did finish)
         else the best pessimistic point.
         """
-        self._check_error()
         assert self._messaging_thread is not None, 'Optimization was not even started'
-        self._messaging_thread.stop()
         if self._messaging_thread.output is not None:
             return self._messaging_thread.output
         return self.current_bests["pessimistic"].x
diff --git a/requirements.txt b/requirements.txt
index 5c928f68..859aab26 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 numpy>=1.15.0
 pandas>=0.23.4
 cma>=2.6.0
-bayesian-optimization>=0.6.0
+bayesian-optimization==0.6.0
 matplotlib>=2.2.3
 nose>=1.3.7
 genty>=1.3.2
diff --git a/setup.py b/setup.py
index 8557178f..55ad6449 100644
--- a/setup.py
+++ b/setup.py
@@ -16,7 +16,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name='nevergrad',
-    version='0.1.1',
+    version='0.1.2',
     license='MIT',
     description='A Python toolbox for performing gradient-free optimization',
     long_description=long_description,
@@ -27,6 +27,7 @@ setup(
                  'Intended Audience :: Science/Research',
                  'Topic :: Scientific/Engineering',
                  'Programming Language :: Python'],
-    data_files=[('', ['LICENSE', 'requirements.txt'])],
+    data_files=[('', ['LICENSE', 'requirements.txt']),
+                ('nevergrad', ["nevergrad/benchmark/additional/example.py"])],
     install_requires=requirements,
 )


Test patch:
diff --git a/nevergrad/benchmark/test_execution.py b/nevergrad/benchmark/test_execution.py
new file mode 100644
index 00000000..a7860189
--- /dev/null
+++ b/nevergrad/benchmark/test_execution.py
@@ -0,0 +1,66 @@
+from operator import add
+from unittest import TestCase
+from typing import List, Tuple, Dict, Any, Optional, Callable
+import genty
+import numpy as np
+from . import execution
+
+
+class Function(execution.PostponedObject):
+
+    def __call__(self, x: int, y: int) -> float:
+        return x + y
+
+    # pylint: disable=unused-argument
+    def get_postponing_delay(self, arguments: Tuple[Tuple[Any, ...], Dict[str, Any]], value: float) -> float:
+        print("waiting", 5 - value)
+        return 5 - value
+
+
+@genty.genty
+class ExecutorTest(TestCase):
+
+    @genty.genty_dataset(  # type: ignore
+        simple=(add, list(range(10))),
+        delayed=(Function(), [5, 6, 7, 8, 9, 4, 3, 2, 1, 0])
+    )
+    def test_mocked_steady_executor(self, func: Callable, expected: List[int]) -> None:
+        executor = execution.MockedSteadyExecutor()
+        jobs: List[execution.MockedSteadyJob] = []
+        for k in range(10):
+            jobs.append(executor.submit(func, k, 0))
+        results: List[float] = []
+        while jobs:
+            finished = [j for j in jobs if j.done()]
+            np.testing.assert_(len(finished) == 1)
+            results.append(finished[0].result())
+            jobs.remove(finished[0])
+        np.testing.assert_array_equal(results, expected)
+
+
+def test_mocked_steady_executor_time() -> None:
+    func = Function()
+    executor = execution.MockedSteadyExecutor()
+    jobs = [executor.submit(func, 0, 0)]
+    np.testing.assert_equal(jobs[0].done(), True)
+    jobs.append(executor.submit(func, 2, 0))
+    np.testing.assert_equal(jobs[0].done(), False)  # now there is one job before
+    np.testing.assert_equal(jobs[1].done(), True)
+    np.testing.assert_equal(executor._time, 0)
+    jobs[1].result()
+    np.testing.assert_equal(executor._time, 3)
+    # now making sure that jobs start from new time
+    jobs.append(executor.submit(func, 4, 0))
+    jobs.append(executor.submit(func, 3, 0))
+    np.testing.assert_array_equal([j.release_time for j in executor.priority_queue], [4, 5, 5])
+    new_finished: Optional[List[execution.MockedSteadyJob]] = None
+    order: List[int] = []
+    # pylint: disable=unsubscriptable-object
+    while new_finished is None or new_finished:
+        if new_finished is not None:
+            assert len(new_finished) == 1, f'Weird list: {new_finished}'
+            order.append(jobs.index(new_finished[0]))
+            new_finished[0].result()
+        print(jobs)
+        new_finished = [j for j in jobs if j.done() and not j._is_read]
+    np.testing.assert_array_equal(order, [2, 0, 3])
diff --git a/nevergrad/benchmark/test_experiments.py b/nevergrad/benchmark/test_experiments.py
index a61ae1c1..7841a9ad 100644
--- a/nevergrad/benchmark/test_experiments.py
+++ b/nevergrad/benchmark/test_experiments.py
@@ -54,7 +54,7 @@ def check_seedable(maker: Any) -> None:
     algo = "OnePlusOne"  # for simplifying the test
     for seed in [random_seed, random_seed, random_seed + 1]:
         xps = list(itertools.islice(maker(seed), 0, 8))
-        simplified = [Experiment(xp.function, algo, budget=2, num_workers=min(2, xp._optimizer_parameters["num_workers"]), seed=xp.seed)
+        simplified = [Experiment(xp.function, algo, budget=2, num_workers=min(2, xp.optimsettings.num_workers), seed=xp.seed)
                       for xp in xps]
         np.random.shuffle(simplified)  # compute in any order
         selector = Selector(data=[xp.run() for xp in simplified])
diff --git a/nevergrad/benchmark/test_xpbase.py b/nevergrad/benchmark/test_xpbase.py
index a7bb9d31..a69e1bce 100644
--- a/nevergrad/benchmark/test_xpbase.py
+++ b/nevergrad/benchmark/test_xpbase.py
@@ -7,22 +7,35 @@ import sys
 import contextlib
 from unittest import TestCase
 from unittest.mock import patch
-from typing import Optional, List
+from typing import Optional, List, Tuple, Any, Dict
 import genty
 import numpy as np
 from ..common import testing
+from ..optimization import test_base
+from ..functions.base import BaseFunction
 from ..functions import ArtificialFunction
 from ..functions.test_functionlib import DESCRIPTION_KEYS as ARTIFICIAL_KEYS
-from .xpbase import Experiment
-from .xpbase import create_seed_generator
+from . import execution
+from . import xpbase
 
 
-DESCRIPTION_KEYS = {"seed", "elapsed_time", "elapsed_budget", "loss", "optimizer_name", "num_workers", "budget", "error"} | ARTIFICIAL_KEYS
+DESCRIPTION_KEYS = {"seed", "elapsed_time", "elapsed_budget", "loss", "optimizer_name",
+                    "num_workers", "budget", "error", "batch_mode"} | ARTIFICIAL_KEYS
+
+
+class Function(BaseFunction, execution.PostponedObject):
+
+    def oracle_call(self, x) -> float:
+        return x[0]
+
+    # pylint: disable=unused-argument
+    def get_postponing_delay(self, arguments: Tuple[Tuple[Any, ...], Dict[str, Any]], value: float) -> float:
+        return 5 - value
 
 
 def test_run_artificial_function() -> None:
     func = ArtificialFunction(name="sphere", block_dimension=2)
-    xp = Experiment(func, optimizer_name="OnePlusOne", budget=300, num_workers=1)
+    xp = xpbase.Experiment(func, optimizer_name="OnePlusOne", budget=300, num_workers=1)
     summary = xp.run()
     assert summary["elapsed_time"] < .5  # should be much faster
     assert summary["loss"] < .001
@@ -32,7 +45,7 @@ def test_run_artificial_function() -> None:
 
 def test_run_with_error() -> None:
     func = ArtificialFunction(name="sphere", block_dimension=2)
-    xp = Experiment(func, optimizer_name="OnePlusOne", budget=300, num_workers=1)
+    xp = xpbase.Experiment(func, optimizer_name="OnePlusOne", budget=300, num_workers=1)
     with patch("nevergrad.optimization.optimizerlib.OnePlusOne.optimize") as run:
         run.side_effect = ValueError("test error string")
         with contextlib.redirect_stderr(sys.stdout):
@@ -50,7 +63,7 @@ class ExperimentsTests(TestCase):
     )
     def test_is_incoherent(self, optimizer: str, num_workers: int, expected: bool) -> None:
         func = ArtificialFunction(name="sphere", block_dimension=2)
-        xp = Experiment(func, optimizer_name=optimizer, budget=300, num_workers=num_workers)
+        xp = xpbase.Experiment(func, optimizer_name=optimizer, budget=300, num_workers=num_workers)
         np.testing.assert_equal(xp.is_incoherent, expected)
 
     @genty.genty_dataset(  # type: ignore
@@ -61,7 +74,7 @@ class ExperimentsTests(TestCase):
     )
     def test_seed_generator(self, seed: Optional[int], randsize: int, expected: List[Optional[int]]) -> None:
         output = []
-        generator = create_seed_generator(seed)
+        generator = xpbase.create_seed_generator(seed)
         for _ in range(4):
             if randsize:  # call the standard random generator
                 np.random.normal(0, 1, size=randsize)
@@ -69,9 +82,21 @@ class ExperimentsTests(TestCase):
             output.append(value if value is None else value % 1000)
         np.testing.assert_array_equal(output, expected)
 
+    @genty.genty_dataset(  # type: ignore
+        w3_batch=(True, ['s0', 's1', 's2', 'u0', 'u1', 'u2', 's3', 's4', 'u3', 'u4']),
+        w3_steady=(False, ['s0', 's1', 's2', 'u2', 's3', 'u1', 's4', 'u0', 'u3', 'u4']),  # u0 and u1 are delayed
+    )
+    def test_batch_mode_parameter(self, batch_mode, expected):
+        func = Function(dimension=1)
+        optim = test_base.LoggingOptimizer(3)
+        with patch.object(xpbase.OptimizerSettings, "instanciate", return_value=optim):
+            xp = xpbase.Experiment(func, optimizer_name="OnePlusOne", budget=10, num_workers=3, batch_mode=batch_mode)
+            xp._run_with_error()
+            testing.printed_assert_equal(optim.logs, expected)
+
 
 def test_equality() -> None:
     func = ArtificialFunction(name="sphere", block_dimension=2)
-    xp1 = Experiment(func, optimizer_name="OnePlusOne", budget=300, num_workers=2)
-    xp2 = Experiment(func, optimizer_name="RandomSearch", budget=300, num_workers=2)
+    xp1 = xpbase.Experiment(func, optimizer_name="OnePlusOne", budget=300, num_workers=2)
+    xp2 = xpbase.Experiment(func, optimizer_name="RandomSearch", budget=300, num_workers=2)
     assert xp1 != xp2
diff --git a/nevergrad/functions/test_functionlib.py b/nevergrad/functions/test_functionlib.py
index bd71364f..24fd2708 100644
--- a/nevergrad/functions/test_functionlib.py
+++ b/nevergrad/functions/test_functionlib.py
@@ -99,3 +99,10 @@ def test_artifificial_function_with_jump() -> None:
     func2 = functionlib.ArtificialFunction("jump5", 5)
     np.testing.assert_equal(func1._only_index_transform, False)
     np.testing.assert_equal(func2._only_index_transform, True)
+
+
+def test_get_posptoning_delay() -> None:
+    func = functionlib.ArtificialFunction("sphere", 2)
+    np.testing.assert_equal(func.get_postponing_delay((([2, 2],), {}), 3), 0)
+    func = functionlib.ArtificialFunction("DelayedSphere", 2)
+    np.testing.assert_equal(func.get_postponing_delay((([2, 2],), {}), 3), 0.0005)