Repository: microsoft/MLOS

Issue:
Storage schema doesn't allow an empty config
In some cases its useful to test deployment and other scripts without a config (i.e., vnet deployment with no tunable params).

Currently some combination of the storage schema and the way we serialize the config representation to hash it to create a config id prevents us from doing this.

It would be nice if we handled that as an "empty" config somehow as well.


Discussion:
@motus, could you please take a look at this?
commit 428cd2280d4e67fa52878a2364a85d9481ef9fd6 from #606 adds a small workaround for this for now

Resolving patch:
diff --git a/mlos_bench/mlos_bench/storage/sql/experiment.py b/mlos_bench/mlos_bench/storage/sql/experiment.py
--- a/mlos_bench/mlos_bench/storage/sql/experiment.py
+++ b/mlos_bench/mlos_bench/storage/sql/experiment.py
@@ -168,6 +168,8 @@ def _get_params(conn: Connection, table: Table, **kwargs: Any) -> Dict[str, Any]
     @staticmethod
     def _save_params(conn: Connection, table: Table,
                      params: Dict[str, Any], **kwargs: Any) -> None:
+        if not params:
+            return
         conn.execute(table.insert(), [
             {
                 **kwargs,
diff --git a/mlos_bench/mlos_bench/tunables/tunable_groups.py b/mlos_bench/mlos_bench/tunables/tunable_groups.py
--- a/mlos_bench/mlos_bench/tunables/tunable_groups.py
+++ b/mlos_bench/mlos_bench/tunables/tunable_groups.py
@@ -36,6 +36,9 @@ def __init__(self, config: Optional[dict] = None):
         for (name, group_config) in config.items():
             self._add_group(CovariantTunableGroup(name, group_config))
 
+    def __bool__(self) -> bool:
+        return bool(self._index)
+
     def __eq__(self, other: object) -> bool:
         """
         Check if two TunableGroups are equal.


Test patch:
diff --git a/mlos_bench/mlos_bench/tests/storage/conftest.py b/mlos_bench/mlos_bench/tests/storage/conftest.py
--- a/mlos_bench/mlos_bench/tests/storage/conftest.py
+++ b/mlos_bench/mlos_bench/tests/storage/conftest.py
@@ -15,6 +15,7 @@
 # Expose some of those as local names so they can be picked up as fixtures by pytest.
 storage = sql_storage_fixtures.storage
 exp_storage = sql_storage_fixtures.exp_storage
+exp_storage_no_tunables = sql_storage_fixtures.exp_storage_no_tunables
 mixed_numerics_exp_storage = sql_storage_fixtures.mixed_numerics_exp_storage
 exp_storage_with_trials = sql_storage_fixtures.exp_storage_with_trials
 mixed_numerics_exp_storage_with_trials = sql_storage_fixtures.mixed_numerics_exp_storage_with_trials
diff --git a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
--- a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
+++ b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
@@ -63,6 +63,31 @@ def exp_storage(
     assert not exp._in_context
 
 
+@pytest.fixture
+def exp_storage_no_tunables(
+    storage: SqlStorage,
+) -> Generator[SqlStorage.Experiment, None, None]:
+    """
+    Test fixture for Experiment using in-memory SQLite3 storage.
+    Note: It has already entered the context upon return.
+    """
+    opt_target = "score"
+    opt_direction = "min"
+    empty_config: dict = {}
+    with storage.experiment(
+        experiment_id="Test-001",
+        trial_id=1,
+        root_env_config="environment.jsonc",
+        description="pytest experiment",
+        tunables=TunableGroups(empty_config),
+        opt_target=opt_target,
+        opt_direction=opt_direction,
+    ) as exp:
+        yield exp
+    # pylint: disable=protected-access
+    assert not exp._in_context
+
+
 @pytest.fixture
 def mixed_numerics_exp_storage(
     storage: SqlStorage,
diff --git a/mlos_bench/mlos_bench/tests/storage/trial_config_test.py b/mlos_bench/mlos_bench/tests/storage/trial_config_test.py
--- a/mlos_bench/mlos_bench/tests/storage/trial_config_test.py
+++ b/mlos_bench/mlos_bench/tests/storage/trial_config_test.py
@@ -62,3 +62,19 @@ def test_exp_trial_configs(exp_storage: Storage.Experiment,
     assert len(pending_ids) == 6
     assert len(set(pending_ids)) == 2
     assert set(pending_ids) == {trials1[0].tunable_config_id, trials2[0].tunable_config_id}
+
+
+def test_exp_trial_no_config(exp_storage_no_tunables: Storage.Experiment) -> None:
+    """
+    Schedule a trial that has an empty tunable groups config.
+    """
+    empty_config: dict = {}
+    tunable_groups = TunableGroups(config=empty_config)
+    trial = exp_storage_no_tunables.new_trial(tunable_groups, config=empty_config)
+    (pending,) = exp_storage_no_tunables.pending_trials(datetime.utcnow(), running=True)
+    assert pending.trial_id == trial.trial_id
+    assert pending.tunables == tunable_groups
+    assert pending.config() == {
+        "experiment_id": "Test-001",
+        "trial_id": trial.trial_id,
+    }
diff --git a/mlos_bench/mlos_bench/tests/tunables/test_empty_tunable_group.py b/mlos_bench/mlos_bench/tests/tunables/test_empty_tunable_group.py
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/tunables/test_empty_tunable_group.py
@@ -0,0 +1,24 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+"""
+Unit tests for empty tunable groups.
+"""
+
+from mlos_bench.tunables.tunable_groups import TunableGroups
+
+
+def test_empty_tunable_group() -> None:
+    """
+    Test __nonzero__ property of tunable groups.
+    """
+    tunable_groups = TunableGroups(config={})
+    assert not tunable_groups
+
+
+def test_non_empty_tunable_group(tunable_groups: TunableGroups) -> None:
+    """
+    Test __nonzero__ property of tunable groups.
+    """
+    assert tunable_groups