Repository: boschresearch/blackboxopt (language: python)

Issue:
add discrete acquisition function optimization for discrete spaces with gpbo
When a search space is purely discrete, we randomly sample by default 5k points to evaluate / optimize the acquisition function at, to avoid issues that arise when the acquisition function gets stuck proposing points that can't be evaluated (could happen due to continuous representation via parameterspace).
For mixed spaces the state remains unchanged, so there we might still suffer from the mentioned issue.


Discussion / hints:
> Also, this PR fixes that the GP model was never updated in the BO's `report` method.

Turns out this is a little bit larger of a change than expected, so it's now a separate PR #81. I'll rebase this PR once the other one is merged. Feel free to already review this one as well, though, because the breaking builds are resolved as soon as #81 arrives.

Fix patch:
diff --git a/blackboxopt/__init__.py b/blackboxopt/__init__.py
index 4eaa01ff..16f413bd 100644
--- a/blackboxopt/__init__.py
+++ b/blackboxopt/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "4.9.2"
+__version__ = "4.10.0"
 
 from parameterspace import ParameterSpace
 
diff --git a/blackboxopt/optimizers/botorch_base.py b/blackboxopt/optimizers/botorch_base.py
index bbabfd5e..fc5ec902 100644
--- a/blackboxopt/optimizers/botorch_base.py
+++ b/blackboxopt/optimizers/botorch_base.py
@@ -6,23 +6,19 @@
 import functools
 import logging
 import warnings
-from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Callable, Dict, Iterable, Optional, Tuple, Union
 
 from gpytorch.models import ExactGP
 
-from blackboxopt import (
-    ConstraintsError,
-    Evaluation,
-    EvaluationSpecification,
+from blackboxopt.base import (
     Objective,
     OptimizerNotReady,
-    sort_evaluations,
-)
-from blackboxopt.base import (
     SingleObjectiveOptimizer,
     call_functions_with_evaluations_and_collect_errors,
     validate_objectives,
 )
+from blackboxopt.evaluation import Evaluation, EvaluationSpecification
+from blackboxopt.utils import sort_evaluations
 
 try:
     import numpy as np
@@ -34,7 +30,12 @@
     from botorch.models.model import Model
     from botorch.optim import optimize_acqf, optimize_acqf_discrete
     from botorch.sampling.samplers import IIDNormalSampler
-    from sklearn.impute import SimpleImputer
+
+    from blackboxopt.optimizers.botorch_utils import (
+        filter_y_nans,
+        impute_nans_with_constant,
+        to_numerical,
+    )
 
 except ImportError as e:
     raise ImportError(
@@ -43,129 +44,6 @@
     ) from e
 
 
-def impute_nans_with_constant(x: torch.Tensor, c: float = -1.0) -> torch.Tensor:
-    """Impute `NaN` values with given constant value.
-
-    Args:
-        x: Input tensor of shape `n x d` or `b x n x d`.
-        c: Constant used as fill value to replace `NaNs`.
-
-    Returns:
-        - x_i - `x` where all `NaN`s are replaced with given constant.
-    """
-    if x.numel() == 0:  # empty tensor, nothing to impute
-        return x
-    x_i = x.clone()
-
-    # cast n x d to 1 x n x d (cover non-batch case)
-    if len(x.shape) == 2:
-        x_i = x_i.reshape(torch.Size((1,)) + x_i.shape)
-
-    for b in range(x_i.shape[0]):
-        x_1 = x_i[b, :, :]
-        x_1 = torch.tensor(
-            SimpleImputer(
-                missing_values=np.nan, strategy="constant", fill_value=c
-            ).fit_transform(x_1),
-            dtype=x.dtype,
-        )
-        x_i[b, :, :] = x_1
-
-    # cast 1 x n x d back to n x d if originally non-batch
-    if len(x.shape) == 2:
-        x_i = x_i.reshape(x.shape)
-    return x_i
-
-
-def to_numerical(
-    evaluations: Iterable[Evaluation],
-    search_space: ps.ParameterSpace,
-    objective: Objective,
-    constraint_names: Optional[List[str]] = None,
-    batch_shape: torch.Size = torch.Size(),
-    torch_dtype: torch.dtype = torch.float32,
-) -> Tuple[torch.Tensor, torch.Tensor]:
-    """Convert evaluations to one `(#batch, #evaluations, #parameters)` tensor
-    containing the numerical representations of the configurations and
-    one `(#batch, #evaluations, 1)` tensor containing the loss representation of
-    the evaluations' objective value (flips the sign for objective value
-    if `objective.greater_is_better=True`) and optionally constraints value.
-
-    Args:
-        evaluations: List of evaluations that were collected during optimization.
-        search_space: Search space used during optimization.
-        objective: Objective that was used for optimization.
-        constraint_names: Name of constraints that are used for optimization.
-        batch_shape: Batch dimension(s) used for batched models.
-        torch_dtype: Type of returned tensors.
-
-    Returns:
-        - X: Numerical representation of the configurations
-        - Y: Numerical representation of the objective values and optionally constraints
-
-    Raises:
-        ValueError: If one of configurations is not valid w.r.t. search space.
-        ValueError: If one of configurations includes parameters that are not part of
-            the search space.
-        ConstraintError: If one of the constraint names is not defined in evaluations.
-    """
-    # validate configuration values and dimensions
-    parameter_names = search_space.get_parameter_names() + list(
-        search_space.get_constant_names()
-    )
-    for e in evaluations:
-        with warnings.catch_warnings():
-            # we already raise error if search space not valid, thus can ignore warnings
-            warnings.filterwarnings(
-                "ignore", category=RuntimeWarning, message="Parameter"
-            )
-            if not search_space.check_validity(e.configuration):
-                raise ValueError(
-                    f"The provided configuration {e.configuration} is not valid."
-                )
-        if not set(parameter_names) >= set(e.configuration.keys()):
-            raise ValueError(
-                f"Mismatch in parameter names from search space {parameter_names} and "
-                + f"configuration {e.configuration}"
-            )
-
-    X = torch.tensor(
-        np.array([search_space.to_numerical(e.configuration) for e in evaluations]),
-        dtype=torch_dtype,
-    )
-    X = X.reshape(*batch_shape + X.shape)
-    Y = torch.tensor(
-        np.array([[e.objectives[objective.name]] for e in evaluations], dtype=float),
-        dtype=torch_dtype,
-    )
-
-    if objective.greater_is_better:
-        Y *= -1
-
-    if constraint_names is not None:
-        try:
-            Y_constraints = torch.tensor(
-                np.array(
-                    [[e.constraints[c] for c in constraint_names] for e in evaluations],
-                    dtype=float,
-                ),
-                dtype=torch_dtype,
-            )
-            Y = torch.cat((Y, Y_constraints), dim=1)
-        except KeyError as e:
-            raise ConstraintsError(
-                f"Constraint name {e} is not defined in input evaluations."
-            )
-        except TypeError:
-            raise ConstraintsError(
-                f"Constraint name(s) {constraint_names} are not defined in input evaluations."
-            )
-
-    Y = Y.reshape(*batch_shape + Y.shape)
-
-    return X, Y
-
-
 def _acquisition_function_optimizer_factory(
     search_space: ps.ParameterSpace,
     af_opt_kwargs: Optional[dict],
@@ -173,13 +51,14 @@ def _acquisition_function_optimizer_factory(
 ) -> Callable[[AcquisitionFunction], Tuple[torch.Tensor, torch.Tensor]]:
     """Prepare either BoTorch's `optimize_acqf_discrete` or `optimize_acqf` depending
     on whether the search space is fully discrete or not and set required defaults if
-    not overridden by `af_opt_kwargs`.
+    not overridden by `af_opt_kwargs`. If any of the af optimizer specific required
+    kwargs are set, this overrides the automatic discrete space detection.
 
     Args:
         search_space: Search space used for optimization.
         af_opt_kwargs: Acquisition function optimizer configuration, e.g. containing
-            values for `n_samples` for discrete optimization, and `num_restarts`,
-            `raw_samples` for the continuous optimization case.
+            values for `num_random_choices` for discrete optimization, and
+            `num_restarts`, `raw_samples` for the continuous optimization case.
         torch_dtype: Torch tensor type.
 
     Returns:
@@ -188,64 +67,32 @@ def _acquisition_function_optimizer_factory(
     """
     kwargs = {} if af_opt_kwargs is None else af_opt_kwargs.copy()
 
-    is_fully_discrete_space = not any(
+    space_has_continuous_parameters = any(
         search_space[n]["parameter"].is_continuous
         for n in search_space.get_parameter_names()
     )
-    if is_fully_discrete_space:
-        choices = torch.Tensor(
-            [
-                search_space.to_numerical(search_space.sample())
-                for _ in range(kwargs.pop("n_samples", 5_000))
-            ]
-        ).to(dtype=torch_dtype)
-        return functools.partial(optimize_acqf_discrete, q=1, choices=choices, **kwargs)
-
-    return functools.partial(
-        optimize_acqf,
-        q=1,
-        # The numerical representation always lives on the unit hypercube
-        bounds=torch.tensor([[0, 1]] * len(search_space), dtype=torch_dtype).T,
-        num_restarts=kwargs.pop("num_restarts", 4),
-        raw_samples=kwargs.pop("raw_samples", 1024),
-        **kwargs,
-    )
-
-
-def filter_y_nans(
-    x: torch.Tensor, y: torch.Tensor
-) -> Tuple[torch.Tensor, torch.Tensor]:
-    """Filter rows jointly for `x` and `y`, where `y` is `NaN`.
-
-    Args:
-        x: Input tensor of shape `n x d` or `1 x n x d`.
-        y: Input tensor of shape `n x m` or `1 x n x m`.
-
-    Returns:
-        - x_f: Filtered `x`.
-        - y_f: Filtered `y`.
-
-    Raises:
-        ValueError: If input is 3D (batched representation) with first dimension not
-            `1` (multiple batches).
-    """
-    if (len(x.shape) == 3 and x.shape[0] > 1) or (len(y.shape) == 3 and y.shape[0] > 1):
-        raise ValueError("Multiple batches are not supported for now.")
-
-    x_f = x.clone()
-    y_f = y.clone()
-
-    # filter rows jointly where y is NaN
-    x_f = x_f[~torch.any(y_f.isnan(), dim=-1)]
-    y_f = y_f[~torch.any(y_f.isnan(), dim=-1)]
-
-    # cast n x d back to 1 x n x d if originally batch case
-    if len(x.shape) == 3:
-        x_f = x_f.reshape(torch.Size((1,)) + x_f.shape)
-    if len(y.shape) == 3:
-        y_f = y_f.reshape(torch.Size((1,)) + y_f.shape)
+    if "num_random_choices" not in kwargs and (
+        "num_restarts" in kwargs
+        or "raw_samples" in kwargs
+        or space_has_continuous_parameters
+    ):
+        return functools.partial(
+            optimize_acqf,
+            q=1,
+            # The numerical representation always lives on the unit hypercube
+            bounds=torch.tensor([[0, 1]] * len(search_space), dtype=torch_dtype).T,
+            num_restarts=kwargs.pop("num_restarts", 4),
+            raw_samples=kwargs.pop("raw_samples", 1024),
+            **kwargs,
+        )
 
-    return x_f, y_f
+    choices = torch.Tensor(
+        [
+            search_space.to_numerical(search_space.sample())
+            for _ in range(kwargs.pop("num_random_choices", 5_000))
+        ]
+    ).to(dtype=torch_dtype)
+    return functools.partial(optimize_acqf_discrete, q=1, choices=choices, **kwargs)
 
 
 class SingleObjectiveBOTorchOptimizer(SingleObjectiveOptimizer):
@@ -280,7 +127,9 @@ def __init__(
                 `functools.partial(UpperConfidenceBound, beta=6.0, maximize=False)`.
             af_optimizer_kwargs: Settings for acquisition function optimizer,
                 see `botorch.optim.optimize_acqf` and in case the whole search space
-                is discrete: `botorch.optim.optimize_acqf_discrete`.
+                is discrete: `botorch.optim.optimize_acqf_discrete`. The former can be
+                enforced by providing `raw_samples` or `num_restarts`, the latter by
+                providing `num_random_choices`.
             num_initial_random_samples: Size of the initial space-filling design that
                 is used before starting BO. The points are sampled randomly in the
                 search space. If no random sampling is required, set it to 0.
diff --git a/blackboxopt/optimizers/botorch_utils.py b/blackboxopt/optimizers/botorch_utils.py
new file mode 100644
index 00000000..6d61da28
--- /dev/null
+++ b/blackboxopt/optimizers/botorch_utils.py
@@ -0,0 +1,175 @@
+# Copyright (c) 2023 - for information on the respective copyright owner
+# see the NOTICE file and/or the repository https://github.com/boschresearch/blackboxopt
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import warnings
+from typing import Iterable, List, Optional, Tuple
+
+import numpy as np
+import parameterspace as ps
+import torch
+from sklearn.impute import SimpleImputer
+
+from blackboxopt.base import ConstraintsError, Objective
+from blackboxopt.evaluation import Evaluation
+
+
+def filter_y_nans(
+    x: torch.Tensor, y: torch.Tensor
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """Filter rows jointly for `x` and `y`, where `y` is `NaN`.
+
+    Args:
+        x: Input tensor of shape `n x d` or `1 x n x d`.
+        y: Input tensor of shape `n x m` or `1 x n x m`.
+
+    Returns:
+        - x_f: Filtered `x`.
+        - y_f: Filtered `y`.
+
+    Raises:
+        ValueError: If input is 3D (batched representation) with first dimension not
+            `1` (multiple batches).
+    """
+    if (len(x.shape) == 3 and x.shape[0] > 1) or (len(y.shape) == 3 and y.shape[0] > 1):
+        raise ValueError("Multiple batches are not supported for now.")
+
+    x_f = x.clone()
+    y_f = y.clone()
+
+    # filter rows jointly where y is NaN
+    x_f = x_f[~torch.any(y_f.isnan(), dim=-1)]
+    y_f = y_f[~torch.any(y_f.isnan(), dim=-1)]
+
+    # cast n x d back to 1 x n x d if originally batch case
+    if len(x.shape) == 3:
+        x_f = x_f.reshape(torch.Size((1,)) + x_f.shape)
+    if len(y.shape) == 3:
+        y_f = y_f.reshape(torch.Size((1,)) + y_f.shape)
+
+    return x_f, y_f
+
+
+def impute_nans_with_constant(x: torch.Tensor, c: float = -1.0) -> torch.Tensor:
+    """Impute `NaN` values with given constant value.
+
+    Args:
+        x: Input tensor of shape `n x d` or `b x n x d`.
+        c: Constant used as fill value to replace `NaNs`.
+
+    Returns:
+        - x_i - `x` where all `NaN`s are replaced with given constant.
+    """
+    if x.numel() == 0:  # empty tensor, nothing to impute
+        return x
+    x_i = x.clone()
+
+    # cast n x d to 1 x n x d (cover non-batch case)
+    if len(x.shape) == 2:
+        x_i = x_i.reshape(torch.Size((1,)) + x_i.shape)
+
+    for b in range(x_i.shape[0]):
+        x_1 = x_i[b, :, :]
+        x_1 = torch.tensor(
+            SimpleImputer(
+                missing_values=np.nan, strategy="constant", fill_value=c
+            ).fit_transform(x_1),
+            dtype=x.dtype,
+        )
+        x_i[b, :, :] = x_1
+
+    # cast 1 x n x d back to n x d if originally non-batch
+    if len(x.shape) == 2:
+        x_i = x_i.reshape(x.shape)
+    return x_i
+
+
+def to_numerical(
+    evaluations: Iterable[Evaluation],
+    search_space: ps.ParameterSpace,
+    objective: Objective,
+    constraint_names: Optional[List[str]] = None,
+    batch_shape: torch.Size = torch.Size(),
+    torch_dtype: torch.dtype = torch.float32,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """Convert evaluations to one `(#batch, #evaluations, #parameters)` tensor
+    containing the numerical representations of the configurations and
+    one `(#batch, #evaluations, 1)` tensor containing the loss representation of
+    the evaluations' objective value (flips the sign for objective value
+    if `objective.greater_is_better=True`) and optionally constraints value.
+
+    Args:
+        evaluations: List of evaluations that were collected during optimization.
+        search_space: Search space used during optimization.
+        objective: Objective that was used for optimization.
+        constraint_names: Name of constraints that are used for optimization.
+        batch_shape: Batch dimension(s) used for batched models.
+        torch_dtype: Type of returned tensors.
+
+    Returns:
+        - X: Numerical representation of the configurations
+        - Y: Numerical representation of the objective values and optionally constraints
+
+    Raises:
+        ValueError: If one of configurations is not valid w.r.t. search space.
+        ValueError: If one of configurations includes parameters that are not part of
+            the search space.
+        ConstraintError: If one of the constraint names is not defined in evaluations.
+    """
+    # validate configuration values and dimensions
+    parameter_names = search_space.get_parameter_names() + list(
+        search_space.get_constant_names()
+    )
+    for e in evaluations:
+        with warnings.catch_warnings():
+            # we already raise error if search space not valid, thus can ignore warnings
+            warnings.filterwarnings(
+                "ignore", category=RuntimeWarning, message="Parameter"
+            )
+            if not search_space.check_validity(e.configuration):
+                raise ValueError(
+                    f"The provided configuration {e.configuration} is not valid."
+                )
+        if not set(parameter_names) >= set(e.configuration.keys()):
+            raise ValueError(
+                f"Mismatch in parameter names from search space {parameter_names} and "
+                + f"configuration {e.configuration}"
+            )
+
+    X = torch.tensor(
+        np.array([search_space.to_numerical(e.configuration) for e in evaluations]),
+        dtype=torch_dtype,
+    )
+    X = X.reshape(*batch_shape + X.shape)
+    Y = torch.tensor(
+        np.array([[e.objectives[objective.name]] for e in evaluations], dtype=float),
+        dtype=torch_dtype,
+    )
+
+    if objective.greater_is_better:
+        Y *= -1
+
+    if constraint_names is not None:
+        try:
+            Y_constraints = torch.tensor(
+                np.array(
+                    [[e.constraints[c] for c in constraint_names] for e in evaluations],
+                    dtype=float,
+                ),
+                dtype=torch_dtype,
+            )
+            Y = torch.cat((Y, Y_constraints), dim=1)
+        except KeyError as e:
+            raise ConstraintsError(
+                f"Constraint name {e} is not defined in input evaluations."
+            )
+        except TypeError:
+            raise ConstraintsError(
+                f"Constraint name(s) {constraint_names} are not defined in input "
+                + "evaluations."
+            )
+
+    Y = Y.reshape(*batch_shape + Y.shape)
+
+    return X, Y
diff --git a/pyproject.toml b/pyproject.toml
index cb48c40c..48ca393c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "blackboxopt"
-version = "4.9.2"
+version = "4.10.0"
 description = "A common interface for blackbox optimization algorithms along with useful helpers like parallel optimization loops, analysis and visualization scripts."
 readme = "README.md"
 repository = "https://github.com/boschresearch/blackboxopt"


Test patch:
diff --git a/tests/optimizers/botorch_base_test.py b/tests/optimizers/botorch_base_test.py
index 86d6a874..d2736863 100644
--- a/tests/optimizers/botorch_base_test.py
+++ b/tests/optimizers/botorch_base_test.py
@@ -5,7 +5,6 @@
 
 from functools import partial
 
-import numpy as np
 import parameterspace as ps
 import pytest
 import torch
@@ -13,13 +12,10 @@
 from botorch.models import SingleTaskGP
 from botorch.optim import optimize_acqf, optimize_acqf_discrete
 
-from blackboxopt import ConstraintsError, Evaluation, Objective
+from blackboxopt.base import Objective
 from blackboxopt.optimizers.botorch_base import (
     SingleObjectiveBOTorchOptimizer,
     _acquisition_function_optimizer_factory,
-    filter_y_nans,
-    impute_nans_with_constant,
-    to_numerical,
 )
 from blackboxopt.optimizers.testing import (
     ALL_REFERENCE_TESTS,
@@ -28,8 +24,6 @@
     respects_fixed_parameter,
 )
 
-from .conftest import constraint_name_1, constraint_name_2, objective_name
-
 
 @pytest.mark.parametrize("reference_test", ALL_REFERENCE_TESTS)
 def test_all_reference_tests(reference_test, seed):
@@ -97,6 +91,42 @@ def test_acquisition_function_optimizer_factory_with_mixed_space():
     assert af_opt.func == optimize_acqf  # pylint: disable=no-member
 
 
+def test_acquisition_function_optimizer_factory_force_discrete():
+    continous_space = ps.ParameterSpace()
+    continous_space.add(ps.ContinuousParameter("conti", (0.0, 1.0)))
+
+    af_opt = _acquisition_function_optimizer_factory(
+        continous_space,
+        af_opt_kwargs={"num_random_choices": 1_000},
+        torch_dtype=torch.float64,
+    )
+
+    assert af_opt.func == optimize_acqf_discrete  # pylint: disable=no-member
+
+
+def test_acquisition_function_optimizer_factory_force_continuous():
+    discrete_space = ps.ParameterSpace()
+    discrete_space.add(ps.IntegerParameter("integ", (-5, 10)))
+    discrete_space.add(ps.OrdinalParameter("ordin", ("small", "medium", "large")))
+    discrete_space.add(ps.CategoricalParameter("categ", ("woof", "miaow", "moo")))
+
+    af_opt = _acquisition_function_optimizer_factory(
+        discrete_space,
+        af_opt_kwargs={"num_restarts": 10},
+        torch_dtype=torch.float64,
+    )
+
+    assert af_opt.func == optimize_acqf  # pylint: disable=no-member
+
+    af_opt = _acquisition_function_optimizer_factory(
+        discrete_space,
+        af_opt_kwargs={"raw_samples": 5_000},
+        torch_dtype=torch.float64,
+    )
+
+    assert af_opt.func == optimize_acqf  # pylint: disable=no-member
+
+
 def test_find_optimum_in_1d_discrete_space(seed):
     space = ps.ParameterSpace()
     space.add(ps.IntegerParameter("integ", (0, 2)))
@@ -125,217 +155,3 @@ def test_find_optimum_in_1d_discrete_space(seed):
     assert (
         sum(l == 0 for l in losses) > 5
     ), "After figuring out the best of the three points, it should only propose that."
-
-
-def test_impute_nans_with_constant():
-    x1_no_nans = torch.tensor([[0.1, 0.1], [0.7, 0.2], [1.0, 0.3]])
-    x1_some_nans = torch.tensor([[0.1, 0.1], [0.7, float("nan")], [1.0, 0.3]])
-    x1_all_nans = torch.tensor(
-        [[0.1, float("nan")], [0.7, float("nan")], [1.0, float("nan")]]
-    )
-
-    x1_no_nans_i = impute_nans_with_constant(x1_no_nans)
-    x1_some_nans_i = impute_nans_with_constant(x1_some_nans)
-    x1_all_nans_i = impute_nans_with_constant(x1_all_nans)
-
-    assert torch.equal(x1_no_nans_i, x1_no_nans)
-    assert torch.equal(
-        x1_some_nans_i, torch.tensor([[0.1, 0.1], [0.7, -1.0], [1.0, 0.3]])
-    )
-    assert torch.equal(
-        x1_all_nans_i, torch.tensor([[0.1, -1.0], [0.7, -1.0], [1.0, -1.0]])
-    )
-
-    # test batched representation
-    x1_some_nans_batched = x1_some_nans.reshape(torch.Size((1,)) + x1_some_nans.shape)
-    x1_some_nans_batched_i = impute_nans_with_constant(x1_some_nans_batched)
-    assert torch.equal(
-        x1_some_nans_batched_i,
-        torch.tensor([[0.1, 0.1], [0.7, -1.0], [1.0, 0.3]]).reshape(
-            x1_some_nans_batched.shape
-        ),
-    )
-
-
-@pytest.mark.parametrize("greater_is_better", [False, True])
-def test_to_numerical(evaluations, search_space, greater_is_better):
-    # Remove None from list it is up to user/specific optimizer what to do with None
-    del evaluations[3]
-
-    objective = Objective(objective_name, greater_is_better)
-
-    X, Y = to_numerical(evaluations, search_space, objective)
-
-    assert X.dtype == torch.float32
-    assert Y.dtype == torch.float32
-    assert X.size() == (len(evaluations), len(search_space))
-    assert Y.size() == (len(evaluations), 1)
-
-    # check that numerical representation on inputs is in hypercube
-    assert torch.all(X[X >= 0]) and torch.all(X[X <= 1])
-
-    # check sign of objective values
-    objectives_original = torch.Tensor(
-        [evaluations[i]["objectives"]["objective"] for i in range(len(evaluations))]
-    ).reshape(Y.shape)
-    assert (
-        torch.equal(Y, -1 * objectives_original)
-        if greater_is_better
-        else torch.equal(Y, objectives_original)
-    )
-
-
-def test_to_numerical_with_batch(evaluations, search_space):
-    # Remove None from list it is up to user/specific optimizer what to do with None
-    del evaluations[3]
-
-    objective = Objective(objective_name, False)
-
-    batch_shape = torch.Size((1,))
-    X, Y = to_numerical(evaluations, search_space, objective, batch_shape=batch_shape)
-
-    assert X.dtype == torch.float32
-    assert Y.dtype == torch.float32
-    assert X.size() == (batch_shape[0], len(evaluations), len(search_space))
-    assert Y.size() == (batch_shape[0], len(evaluations), 1)
-
-    # check that numerical representation on inputs is in hypercube
-    assert torch.all(X[X >= 0]) and torch.all(X[X <= 1])
-
-
-def test_to_numerical_raises_errors(search_space):
-    objective = Objective(objective_name, False)
-
-    # configuration has parameter not part of search space
-    eval_err = Evaluation(
-        configuration={"x0": 0.57, "x1": False, "x2": "small", "x3": 0.5, "fp": 0.5},
-        objectives={objective_name: 0.64},
-    )
-    with pytest.raises(ValueError, match="Mismatch"):
-        to_numerical([eval_err], search_space, objective)
-
-    # parameter not within defined bounds -> invalid configuration
-    eval_err = Evaluation(
-        configuration={"x0": 0.1, "x1": False, "x2": "small", "fp": 0.5},
-        objectives={objective_name: 0.64},
-    )
-    with pytest.raises(ValueError, match="not valid"):
-        to_numerical([eval_err], search_space, objective)
-
-    # conditional parameter should be active, but is inactive -> invalid configuration
-    eval_err = Evaluation(
-        configuration={"x0": 0.57, "x1": True, "x2": "small", "fp": 0.5},
-        objectives={objective_name: 0.64},
-    )
-    with pytest.raises(ValueError, match="not valid"):
-        to_numerical([eval_err], search_space, objective)
-
-    # conditional parameter should be inactive, but is active -> invalid configuration
-    eval_err = Evaluation(
-        configuration={"x0": 0.57, "x1": False, "x2": "small", "cp": 0.3, "fp": 0.5},
-        objectives={objective_name: 0.64},
-    )
-    with pytest.raises(ValueError, match="not valid"):
-        to_numerical([eval_err], search_space, objective)
-
-
-@pytest.mark.parametrize(
-    "constraints",
-    [
-        [constraint_name_1, constraint_name_2],
-        [constraint_name_2, constraint_name_1],
-        [constraint_name_1],
-    ],
-)
-def test_to_numerical_with_constraints(
-    evaluations_with_constraints, search_space, constraints
-):
-    objective = Objective(objective_name, greater_is_better=False)
-    num_eval = len(evaluations_with_constraints)
-    num_constrains = len(constraints)
-
-    _, Y = to_numerical(
-        evaluations_with_constraints,
-        search_space,
-        objective,
-        constraints,
-    )
-
-    assert Y.dtype == torch.float32
-    assert Y.size() == (num_eval, 1 + num_constrains)
-
-    # check order of values in the output tensor: the first is always an objective value,
-    # the order of constraints depends on order in the list of
-    for i in range(num_eval):
-        assert Y[i, 0] == evaluations_with_constraints[i].objectives[objective_name]
-        for c_i, c in enumerate(constraints):
-            assert Y[i, 1 + c_i] == evaluations_with_constraints[i].constraints[c]
-
-
-def test_to_numerical_raises_error_on_wrong_constraints(
-    search_space, evaluations_with_constraints
-):
-    # If wrong constraint name is requested raises an error.
-    objective = Objective(objective_name, False)
-
-    with pytest.raises(ConstraintsError, match="Constraint name"):
-        to_numerical(
-            evaluations_with_constraints,
-            search_space,
-            objective,
-            constraint_names=["WRONG_NAME"],
-        )
-
-    # If evaluation does not contain constraints at all
-    objective = Objective(objective_name, False)
-
-    evaluations = [
-        Evaluation(
-            configuration={"x0": 0.57, "x1": True, "x2": "small", "cp": 0.3, "fp": 0.5},
-            objectives={objective_name: 0.64},
-        )
-    ]
-
-    with pytest.raises(ConstraintsError, match="Constraint name"):
-        to_numerical(
-            evaluations,
-            search_space,
-            objective,
-            constraint_names=[constraint_name_1],
-        )
-
-
-def test_filter_y_nans():
-    x1 = torch.tensor([[0.1], [0.7], [1.0]])
-    y1 = torch.tensor([[0.8], [0.3], [0.5]])
-    x1_f, y1_f = filter_y_nans(x1, y1)
-    assert x1_f.size() == torch.Size([3, 1])
-    assert y1_f.size() == torch.Size([3, 1])
-    assert torch.equal(x1, x1_f)
-    assert torch.equal(y1, y1_f)
-
-    x2 = torch.tensor([[0.1], [0.7], [1.0]])
-    y2 = torch.tensor([[0.8], [np.nan], [0.5]])
-    x2_f, y2_f = filter_y_nans(x2, y2)
-    assert x2_f.size() == torch.Size([2, 1])
-    assert y2_f.size() == torch.Size([2, 1])
-
-    x3 = torch.tensor([[0.1], [0.7], [1.0]])
-    y3 = torch.tensor([[np.nan], [np.nan], [np.nan]])
-    x3_f, y3_f = filter_y_nans(x3, y3)
-    assert x3_f.size() == torch.Size([0, 1])
-    assert y3_f.size() == torch.Size([0, 1])
-
-    # test batched representation
-    x2_batched = x2.reshape(torch.Size((1,)) + x2.shape)
-    y2_batched = y2.reshape(torch.Size((1,)) + y2.shape)
-    x2_batched_f, y2_batched_f = filter_y_nans(x2_batched, y2_batched)
-    assert x2_batched_f.size() == torch.Size([1, 2, 1])
-    assert y2_batched_f.size() == torch.Size([1, 2, 1])
-    assert torch.equal(x2_batched_f, x2_f.reshape(torch.Size((1,)) + x2_f.shape))
-    assert torch.equal(y2_batched_f, y2_f.reshape(torch.Size((1,)) + y2_f.shape))
-
-    x_multi_batch = torch.tensor([[[0.1], [1.0]], [[0.2], [2.0]]])
-    y_multi_batch = torch.tensor([[[0.4], [0.4]], [[0.8], [np.nan]]])
-    with pytest.raises(ValueError, match="Multiple batches"):
-        filter_y_nans(x_multi_batch, y_multi_batch)
diff --git a/tests/optimizers/botorch_utils_test.py b/tests/optimizers/botorch_utils_test.py
new file mode 100644
index 00000000..1750f179
--- /dev/null
+++ b/tests/optimizers/botorch_utils_test.py
@@ -0,0 +1,231 @@
+# Copyright (c) 2023 - for information on the respective copyright owner
+# see the NOTICE file and/or the repository https://github.com/boschresearch/blackboxopt
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import numpy as np
+import pytest
+import torch
+
+from blackboxopt import ConstraintsError, Evaluation, Objective
+from blackboxopt.optimizers.botorch_base import (
+    filter_y_nans,
+    impute_nans_with_constant,
+    to_numerical,
+)
+
+from .conftest import constraint_name_1, constraint_name_2, objective_name
+
+
+def test_impute_nans_with_constant():
+    x1_no_nans = torch.tensor([[0.1, 0.1], [0.7, 0.2], [1.0, 0.3]])
+    x1_some_nans = torch.tensor([[0.1, 0.1], [0.7, float("nan")], [1.0, 0.3]])
+    x1_all_nans = torch.tensor(
+        [[0.1, float("nan")], [0.7, float("nan")], [1.0, float("nan")]]
+    )
+
+    x1_no_nans_i = impute_nans_with_constant(x1_no_nans)
+    x1_some_nans_i = impute_nans_with_constant(x1_some_nans)
+    x1_all_nans_i = impute_nans_with_constant(x1_all_nans)
+
+    assert torch.equal(x1_no_nans_i, x1_no_nans)
+    assert torch.equal(
+        x1_some_nans_i, torch.tensor([[0.1, 0.1], [0.7, -1.0], [1.0, 0.3]])
+    )
+    assert torch.equal(
+        x1_all_nans_i, torch.tensor([[0.1, -1.0], [0.7, -1.0], [1.0, -1.0]])
+    )
+
+    # test batched representation
+    x1_some_nans_batched = x1_some_nans.reshape(torch.Size((1,)) + x1_some_nans.shape)
+    x1_some_nans_batched_i = impute_nans_with_constant(x1_some_nans_batched)
+    assert torch.equal(
+        x1_some_nans_batched_i,
+        torch.tensor([[0.1, 0.1], [0.7, -1.0], [1.0, 0.3]]).reshape(
+            x1_some_nans_batched.shape
+        ),
+    )
+
+
+@pytest.mark.parametrize("greater_is_better", [False, True])
+def test_to_numerical(evaluations, search_space, greater_is_better):
+    # Remove None from list it is up to user/specific optimizer what to do with None
+    del evaluations[3]
+
+    objective = Objective(objective_name, greater_is_better)
+
+    X, Y = to_numerical(evaluations, search_space, objective)
+
+    assert X.dtype == torch.float32
+    assert Y.dtype == torch.float32
+    assert X.size() == (len(evaluations), len(search_space))
+    assert Y.size() == (len(evaluations), 1)
+
+    # check that numerical representation on inputs is in hypercube
+    assert torch.all(X[X >= 0]) and torch.all(X[X <= 1])
+
+    # check sign of objective values
+    objectives_original = torch.Tensor(
+        [evaluations[i]["objectives"]["objective"] for i in range(len(evaluations))]
+    ).reshape(Y.shape)
+    assert (
+        torch.equal(Y, -1 * objectives_original)
+        if greater_is_better
+        else torch.equal(Y, objectives_original)
+    )
+
+
+def test_to_numerical_with_batch(evaluations, search_space):
+    # Remove None from list it is up to user/specific optimizer what to do with None
+    del evaluations[3]
+
+    objective = Objective(objective_name, False)
+
+    batch_shape = torch.Size((1,))
+    X, Y = to_numerical(evaluations, search_space, objective, batch_shape=batch_shape)
+
+    assert X.dtype == torch.float32
+    assert Y.dtype == torch.float32
+    assert X.size() == (batch_shape[0], len(evaluations), len(search_space))
+    assert Y.size() == (batch_shape[0], len(evaluations), 1)
+
+    # check that numerical representation on inputs is in hypercube
+    assert torch.all(X[X >= 0]) and torch.all(X[X <= 1])
+
+
+def test_to_numerical_raises_errors(search_space):
+    objective = Objective(objective_name, False)
+
+    # configuration has parameter not part of search space
+    eval_err = Evaluation(
+        configuration={"x0": 0.57, "x1": False, "x2": "small", "x3": 0.5, "fp": 0.5},
+        objectives={objective_name: 0.64},
+    )
+    with pytest.raises(ValueError, match="Mismatch"):
+        to_numerical([eval_err], search_space, objective)
+
+    # parameter not within defined bounds -> invalid configuration
+    eval_err = Evaluation(
+        configuration={"x0": 0.1, "x1": False, "x2": "small", "fp": 0.5},
+        objectives={objective_name: 0.64},
+    )
+    with pytest.raises(ValueError, match="not valid"):
+        to_numerical([eval_err], search_space, objective)
+
+    # conditional parameter should be active, but is inactive -> invalid configuration
+    eval_err = Evaluation(
+        configuration={"x0": 0.57, "x1": True, "x2": "small", "fp": 0.5},
+        objectives={objective_name: 0.64},
+    )
+    with pytest.raises(ValueError, match="not valid"):
+        to_numerical([eval_err], search_space, objective)
+
+    # conditional parameter should be inactive, but is active -> invalid configuration
+    eval_err = Evaluation(
+        configuration={"x0": 0.57, "x1": False, "x2": "small", "cp": 0.3, "fp": 0.5},
+        objectives={objective_name: 0.64},
+    )
+    with pytest.raises(ValueError, match="not valid"):
+        to_numerical([eval_err], search_space, objective)
+
+
+@pytest.mark.parametrize(
+    "constraints",
+    [
+        [constraint_name_1, constraint_name_2],
+        [constraint_name_2, constraint_name_1],
+        [constraint_name_1],
+    ],
+)
+def test_to_numerical_with_constraints(
+    evaluations_with_constraints, search_space, constraints
+):
+    objective = Objective(objective_name, greater_is_better=False)
+    num_eval = len(evaluations_with_constraints)
+    num_constrains = len(constraints)
+
+    _, Y = to_numerical(
+        evaluations_with_constraints,
+        search_space,
+        objective,
+        constraints,
+    )
+
+    assert Y.dtype == torch.float32
+    assert Y.size() == (num_eval, 1 + num_constrains)
+
+    # check order of values in the output tensor: the first is always an objective value,
+    # the order of constraints depends on order in the list of
+    for i in range(num_eval):
+        assert Y[i, 0] == evaluations_with_constraints[i].objectives[objective_name]
+        for c_i, c in enumerate(constraints):
+            assert Y[i, 1 + c_i] == evaluations_with_constraints[i].constraints[c]
+
+
+def test_to_numerical_raises_error_on_wrong_constraints(
+    search_space, evaluations_with_constraints
+):
+    # If wrong constraint name is requested raises an error.
+    objective = Objective(objective_name, False)
+
+    with pytest.raises(ConstraintsError, match="Constraint name"):
+        to_numerical(
+            evaluations_with_constraints,
+            search_space,
+            objective,
+            constraint_names=["WRONG_NAME"],
+        )
+
+    # If evaluation does not contain constraints at all
+    objective = Objective(objective_name, False)
+
+    evaluations = [
+        Evaluation(
+            configuration={"x0": 0.57, "x1": True, "x2": "small", "cp": 0.3, "fp": 0.5},
+            objectives={objective_name: 0.64},
+        )
+    ]
+
+    with pytest.raises(ConstraintsError, match="Constraint name"):
+        to_numerical(
+            evaluations,
+            search_space,
+            objective,
+            constraint_names=[constraint_name_1],
+        )
+
+
+def test_filter_y_nans():
+    x1 = torch.tensor([[0.1], [0.7], [1.0]])
+    y1 = torch.tensor([[0.8], [0.3], [0.5]])
+    x1_f, y1_f = filter_y_nans(x1, y1)
+    assert x1_f.size() == torch.Size([3, 1])
+    assert y1_f.size() == torch.Size([3, 1])
+    assert torch.equal(x1, x1_f)
+    assert torch.equal(y1, y1_f)
+
+    x2 = torch.tensor([[0.1], [0.7], [1.0]])
+    y2 = torch.tensor([[0.8], [np.nan], [0.5]])
+    x2_f, y2_f = filter_y_nans(x2, y2)
+    assert x2_f.size() == torch.Size([2, 1])
+    assert y2_f.size() == torch.Size([2, 1])
+
+    x3 = torch.tensor([[0.1], [0.7], [1.0]])
+    y3 = torch.tensor([[np.nan], [np.nan], [np.nan]])
+    x3_f, y3_f = filter_y_nans(x3, y3)
+    assert x3_f.size() == torch.Size([0, 1])
+    assert y3_f.size() == torch.Size([0, 1])
+
+    # test batched representation
+    x2_batched = x2.reshape(torch.Size((1,)) + x2.shape)
+    y2_batched = y2.reshape(torch.Size((1,)) + y2.shape)
+    x2_batched_f, y2_batched_f = filter_y_nans(x2_batched, y2_batched)
+    assert x2_batched_f.size() == torch.Size([1, 2, 1])
+    assert y2_batched_f.size() == torch.Size([1, 2, 1])
+    assert torch.equal(x2_batched_f, x2_f.reshape(torch.Size((1,)) + x2_f.shape))
+    assert torch.equal(y2_batched_f, y2_f.reshape(torch.Size((1,)) + y2_f.shape))
+
+    x_multi_batch = torch.tensor([[[0.1], [1.0]], [[0.2], [2.0]]])
+    y_multi_batch = torch.tensor([[[0.4], [0.4]], [[0.8], [np.nan]]])
+    with pytest.raises(ValueError, match="Multiple batches"):
+        filter_y_nans(x_multi_batch, y_multi_batch)