Repository: microsoft/MLOS

Issue:
Add unit tests for `MlosCoreOptimizer._to_df()` functionality
In particular, test how it handles the input when some tunables have special values.


Discussion:

Resolving patch:
diff --git a/mlos_bench/mlos_bench/optimizers/convert_configspace.py b/mlos_bench/mlos_bench/optimizers/convert_configspace.py
--- a/mlos_bench/mlos_bench/optimizers/convert_configspace.py
+++ b/mlos_bench/mlos_bench/optimizers/convert_configspace.py
@@ -30,6 +30,7 @@ class TunableValueKind:
     It is not a true enum because ConfigSpace wants string values.
     """
 
+    # pylint: disable=too-few-public-methods
     SPECIAL = "special"
     RANGE = "range"
 
@@ -82,7 +83,7 @@ def _tunable_to_configspace(
     # Create three hyperparameters: one for regular values,
     # one for special values, and one to choose between the two.
     (special_name, type_name) = special_param_names(tunable.name)
-    cs = ConfigurationSpace({
+    conf_space = ConfigurationSpace({
         tunable.name: hp_type(
             name=tunable.name, lower=tunable.range[0], upper=tunable.range[1],
             default_value=tunable.default if tunable.in_range(tunable.default) else None,
@@ -97,10 +98,12 @@ def _tunable_to_configspace(
             default_value=TunableValueKind.SPECIAL,
             weights=[0.5, 0.5]),  # TODO: Make weights configurable; FLAML requires uniform weights.
     })
-    cs.add_condition(EqualsCondition(cs[special_name], cs[type_name], TunableValueKind.SPECIAL))
-    cs.add_condition(EqualsCondition(cs[tunable.name], cs[type_name], TunableValueKind.RANGE))
+    conf_space.add_condition(EqualsCondition(
+        conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL))
+    conf_space.add_condition(EqualsCondition(
+        conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE))
 
-    return cs
+    return conf_space
 
 
 def tunable_groups_to_configspace(tunables: TunableGroups, seed: Optional[int] = None) -> ConfigurationSpace:


Test patch:
diff --git a/mlos_bench/mlos_bench/tests/optimizers/conftest.py b/mlos_bench/mlos_bench/tests/optimizers/conftest.py
--- a/mlos_bench/mlos_bench/tests/optimizers/conftest.py
+++ b/mlos_bench/mlos_bench/tests/optimizers/conftest.py
@@ -6,6 +6,8 @@
 Test fixtures for mlos_bench optimizers.
 """
 
+from typing import List
+
 import pytest
 
 from mlos_bench.tunables.tunable_groups import TunableGroups
@@ -15,6 +17,39 @@
 from mlos_bench.tests import SEED
 
 
+@pytest.fixture
+def mock_configs() -> List[dict]:
+    """
+    Mock configurations of earlier experiments.
+    """
+    return [
+        {
+            'vmSize': 'Standard_B4ms',
+            'idle': 'halt',
+            'kernel_sched_migration_cost_ns': 50000,
+            'kernel_sched_latency_ns': 1000000,
+        },
+        {
+            'vmSize': 'Standard_B4ms',
+            'idle': 'halt',
+            'kernel_sched_migration_cost_ns': 40000,
+            'kernel_sched_latency_ns': 2000000,
+        },
+        {
+            'vmSize': 'Standard_B4ms',
+            'idle': 'mwait',
+            'kernel_sched_migration_cost_ns': -1,  # Special value
+            'kernel_sched_latency_ns': 3000000,
+        },
+        {
+            'vmSize': 'Standard_B2s',
+            'idle': 'mwait',
+            'kernel_sched_migration_cost_ns': 200000,
+            'kernel_sched_latency_ns': 4000000,
+        }
+    ]
+
+
 @pytest.fixture
 def mock_opt_no_defaults(tunable_groups: TunableGroups) -> MockOptimizer:
     """
@@ -102,12 +137,12 @@ def flaml_opt_max(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
         },
     )
 
+
 # FIXME: SMAC's RF model can be non-deterministic at low iterations, which are
 # normally calculated as a percentage of the max_iterations and number of
 # tunable dimensions, so for now we set the initial random samples equal to the
 # number of iterations and control them with a seed.
 
-
 SMAC_ITERATIONS = 10
 
 
diff --git a/mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_df_test.py b/mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_df_test.py
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_df_test.py
@@ -0,0 +1,83 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+"""
+Unit tests for internal methods of the `MlosCoreOptimizer`.
+"""
+
+from typing import List
+
+import pandas
+import pytest
+
+from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer
+from mlos_bench.tunables.tunable_groups import TunableGroups
+
+from mlos_bench.tests import SEED
+
+# pylint: disable=redefined-outer-name, protected-access
+
+
+@pytest.fixture
+def mlos_core_optimizer(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
+    """
+    An instance of a mlos_core optimizer (FLAML-based).
+    """
+    test_opt_config = {
+        'optimizer_type': 'FLAML',
+        'max_iterations': 10,
+        'seed': SEED,
+    }
+    return MlosCoreOptimizer(tunable_groups, test_opt_config)
+
+
+def test_df(mlos_core_optimizer: MlosCoreOptimizer, mock_configs: List[dict]) -> None:
+    """
+    Test `MlosCoreOptimizer._to_df()` method on tunables that have special values.
+    """
+    df_config = mlos_core_optimizer._to_df(mock_configs)
+    assert isinstance(df_config, pandas.DataFrame)
+    assert df_config.shape == (4, 6)
+    assert set(df_config.columns) == {
+        'kernel_sched_latency_ns',
+        'kernel_sched_migration_cost_ns',
+        'kernel_sched_migration_cost_ns!type',
+        'kernel_sched_migration_cost_ns!special',
+        'idle',
+        'vmSize',
+    }
+    assert df_config.to_dict(orient='records') == [
+        {
+            'idle': 'halt',
+            'kernel_sched_latency_ns': 1000000,
+            'kernel_sched_migration_cost_ns': 50000,
+            'kernel_sched_migration_cost_ns!special': None,
+            'kernel_sched_migration_cost_ns!type': 'range',
+            'vmSize': 'Standard_B4ms',
+        },
+        {
+            'idle': 'halt',
+            'kernel_sched_latency_ns': 2000000,
+            'kernel_sched_migration_cost_ns': 40000,
+            'kernel_sched_migration_cost_ns!special': None,
+            'kernel_sched_migration_cost_ns!type': 'range',
+            'vmSize': 'Standard_B4ms',
+        },
+        {
+            'idle': 'mwait',
+            'kernel_sched_latency_ns': 3000000,
+            'kernel_sched_migration_cost_ns': None,  # The value is special!
+            'kernel_sched_migration_cost_ns!special': -1,
+            'kernel_sched_migration_cost_ns!type': 'special',
+            'vmSize': 'Standard_B4ms',
+        },
+        {
+            'idle': 'mwait',
+            'kernel_sched_latency_ns': 4000000,
+            'kernel_sched_migration_cost_ns': 200000,
+            'kernel_sched_migration_cost_ns!special': None,
+            'kernel_sched_migration_cost_ns!type': 'range',
+            'vmSize': 'Standard_B2s',
+        },
+    ]
diff --git a/mlos_bench/mlos_bench/tests/optimizers/opt_bulk_register_test.py b/mlos_bench/mlos_bench/tests/optimizers/opt_bulk_register_test.py
--- a/mlos_bench/mlos_bench/tests/optimizers/opt_bulk_register_test.py
+++ b/mlos_bench/mlos_bench/tests/optimizers/opt_bulk_register_test.py
@@ -18,39 +18,6 @@
 # pylint: disable=redefined-outer-name
 
 
-@pytest.fixture
-def mock_configs() -> List[dict]:
-    """
-    Mock configurations of earlier experiments.
-    """
-    return [
-        {
-            'vmSize': 'Standard_B4ms',
-            'idle': 'halt',
-            'kernel_sched_migration_cost_ns': 50000,
-            'kernel_sched_latency_ns': 1000000,
-        },
-        {
-            'vmSize': 'Standard_B4ms',
-            'idle': 'halt',
-            'kernel_sched_migration_cost_ns': 40000,
-            'kernel_sched_latency_ns': 2000000,
-        },
-        {
-            'vmSize': 'Standard_B4ms',
-            'idle': 'mwait',
-            'kernel_sched_migration_cost_ns': 100000,
-            'kernel_sched_latency_ns': 3000000,
-        },
-        {
-            'vmSize': 'Standard_B2s',
-            'idle': 'mwait',
-            'kernel_sched_migration_cost_ns': 200000,
-            'kernel_sched_latency_ns': 4000000,
-        }
-    ]
-
-
 @pytest.fixture
 def mock_configs_str(mock_configs: List[dict]) -> List[dict]:
     """
@@ -91,7 +58,7 @@ def _test_opt_update_min(opt: Optimizer, configs: List[dict],
     assert tunables.get_param_values() == {
         "vmSize": "Standard_B4ms",
         "idle": "mwait",
-        "kernel_sched_migration_cost_ns": 100000,
+        "kernel_sched_migration_cost_ns": -1,
         'kernel_sched_latency_ns': 3000000,
     }