Repository: microsoft/MLOS

Issue:
Add tests for non-numeric "metrics"
              Looks ok for now.  I'll make an issue to add tests for this later.

_Originally posted by @bpkroth in https://github.com/microsoft/MLOS/pull/538#discussion_r1355298477_
            


Discussion:

Resolving patch:
diff --git a/mlos_bench/mlos_bench/storage/base_storage.py b/mlos_bench/mlos_bench/storage/base_storage.py
--- a/mlos_bench/mlos_bench/storage/base_storage.py
+++ b/mlos_bench/mlos_bench/storage/base_storage.py
@@ -294,8 +294,8 @@ def config(self, global_config: Optional[Dict[str, Any]] = None) -> Dict[str, An
 
         @abstractmethod
         def update(self, status: Status, timestamp: datetime,
-                   metrics: Optional[Union[Dict[str, float], float]] = None
-                   ) -> Optional[Dict[str, float]]:
+                   metrics: Optional[Union[Dict[str, Any], float]] = None
+                   ) -> Optional[Dict[str, Any]]:
             """
             Update the storage with the results of the experiment.
 
@@ -305,13 +305,13 @@ def update(self, status: Status, timestamp: datetime,
                 Status of the experiment run.
             timestamp: datetime
                 Timestamp of the status and metrics.
-            metrics : Optional[Union[Dict[str, float], float]]
+            metrics : Optional[Union[Dict[str, Any], float]]
                 One or several metrics of the experiment run.
-                Must contain the optimization target if the status is SUCCEEDED.
+                Must contain the (float) optimization target if the status is SUCCEEDED.
 
             Returns
             -------
-            metrics : Optional[Dict[str, float]]
+            metrics : Optional[Dict[str, Any]]
                 Same as `metrics`, but always in the dict format.
             """
             _LOG.info("Store trial: %s :: %s %s", self, status, metrics)
diff --git a/mlos_bench/mlos_bench/storage/sql/trial.py b/mlos_bench/mlos_bench/storage/sql/trial.py
--- a/mlos_bench/mlos_bench/storage/sql/trial.py
+++ b/mlos_bench/mlos_bench/storage/sql/trial.py
@@ -42,8 +42,8 @@ def __init__(self, *,
         self._schema = schema
 
     def update(self, status: Status, timestamp: datetime,
-               metrics: Optional[Union[Dict[str, float], float]] = None
-               ) -> Optional[Dict[str, float]]:
+               metrics: Optional[Union[Dict[str, Any], float]] = None
+               ) -> Optional[Dict[str, Any]]:
         metrics = super().update(status, timestamp, metrics)
         with self._engine.begin() as conn:
             try:


Test patch:
diff --git a/mlos_bench/mlos_bench/tests/storage/exp_load_test.py b/mlos_bench/mlos_bench/tests/storage/exp_load_test.py
--- a/mlos_bench/mlos_bench/tests/storage/exp_load_test.py
+++ b/mlos_bench/mlos_bench/tests/storage/exp_load_test.py
@@ -82,6 +82,25 @@ def test_exp_trial_success(exp_storage_memory_sql: Storage.Experiment,
     assert not trials
 
 
+def test_exp_trial_update_categ(exp_storage_memory_sql: Storage.Experiment,
+                                tunable_groups: TunableGroups) -> None:
+    """
+    Update the trial with multiple metrics, some of which are categorical.
+    """
+    trial = exp_storage_memory_sql.new_trial(tunable_groups)
+    trial.update(Status.SUCCEEDED, datetime.utcnow(), {"score": 99.9, "benchmark": "test"})
+    assert exp_storage_memory_sql.load() == (
+        [{
+            'idle': 'halt',
+            'kernel_sched_latency_ns': '2000000',
+            'kernel_sched_migration_cost_ns': '-1',
+            'vmSize': 'Standard_B4ms'
+        }],
+        [99.9],
+        [Status.SUCCEEDED]
+    )
+
+
 def test_exp_trial_update_twice(exp_storage_memory_sql: Storage.Experiment,
                                 tunable_groups: TunableGroups) -> None:
     """