Repository: microsoft/MLOS

Issue:
SMAC optimizer messes up mlos_bench logging
SMAC optimizer completely overrides our logging setup and installs its own formatter, output handler, and so on. As a result, as soon as SMAC optimizer is initialized, mlos_bench stops writing to its log file, and all logging goes to stdout, in different format, and at different log level (always INFO). We need to find a way to make SMAC use our logger instead of setting up its own from scratch


Discussion:
here's an example of my stdout:
```
2023-08-03 15:17:19,076 util.py:197 instantiate_from_config INFO Instantiating: mlos_bench.optimizers.MlosCoreOptimizer :: <class 'mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer'>
. . . . . . . . .
2023-08-03 15:17:20,494 abstract_initial_design.py:82 __init__ INFO Using `n_configs` and ignoring `n_configs_per_hyperparameter`.
[INFO][abstract_initial_design.py:147] Using 10 initial design configurations and 0 additional configurations.
[INFO][config_persistence.py:241] Created: Optimizer MlosCoreOptimizer:min(score)
[INFO][launcher.py:108] Init optimizer: MlosCoreOptimizer:min(score)
```
Note the change in the output format and switch to INFO level after `2023-08-03 15:17:20,494`. The latter is also the last line I see in our log file. ☹️ 
Can you see about filing this upstream too?
I'll also try and poke around and see if it's something we can control.

Resolving patch:
diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py
--- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py
+++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py
@@ -124,6 +124,7 @@ def __init__(self, *,  # pylint: disable=too-many-locals
             random_design=random_design,
             config_selector=config_selector,
             overwrite=True,
+            logging_level=False,  # Use the existing logger
         )
 
     def __del__(self) -> None:


Test patch:
diff --git a/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc b/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
--- a/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
@@ -13,7 +13,6 @@
     ],
 
     "environment": "environments/mock/mock_env.jsonc",
-    "storage": "storage/in-memory.jsonc",
 
     "tunable_values": [
         "tunable-values/tunable-values-example.jsonc"
diff --git a/mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc b/mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc
@@ -0,0 +1,28 @@
+//
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+//
+
+// *** DO *NOT* CHANGE! This config is used for tests! ***
+{
+    "$schema": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/cli/cli-schema.json",
+
+    "config_path": [
+        // relative to the root of the repo (for now), where this is expected to be executed from
+        "mlos_bench/mlos_bench/tests/config",
+        "mlos_bench/mlos_bench/config"
+    ],
+
+    "environment": "environments/mock/mock_env.jsonc",
+    "optimizer": "optimizers/mlos_core_default_opt.jsonc",
+
+    // "globals": ["global_config.json"],
+
+    "experiment_id": "MockExperiment",
+    "trial_id": 1,
+
+    "teardown": true,
+
+    // "log_file": "mock-opt.log",
+    "log_level": "DEBUG"
+}
diff --git a/mlos_bench/mlos_bench/tests/launcher_test.py b/mlos_bench/mlos_bench/tests/launcher_test.py
--- a/mlos_bench/mlos_bench/tests/launcher_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_test.py
@@ -6,6 +6,8 @@
 Unit tests to check the main CLI launcher.
 """
 import os
+import re
+from typing import List
 
 import pytest
 
@@ -37,27 +39,73 @@ def local_exec_service() -> LocalExecService:
     }))
 
 
-def test_launch_main_app(root_path: str,
-                         local_exec_service: LocalExecService) -> None:
+def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
+                     cli_config: str, re_expected: List[str]) -> None:
     """
-    Run mlos_bench command-line application with mock config and check the results in the log.
+    Run mlos_bench command-line application with given config
+    and check the results in the log.
     """
     with local_exec_service.temp_dir_context() as temp_dir:
 
-        log_path = path_join(temp_dir, "mock-bench.log")
-        cmd = "./mlos_bench/mlos_bench/run.py" + \
-              " --config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc" + \
-              f" --log_file '{log_path}'"
-        (return_code, _stdout, _stderr) = local_exec_service.local_exec([cmd], cwd=root_path)
-
+        # Test developers note: for local debugging,
+        # uncomment the following line to use a known file path that can be examined:
+        # temp_dir = '/tmp'
+        log_path = path_join(temp_dir, "mock-test.log")
+        (return_code, _stdout, _stderr) = local_exec_service.local_exec(
+            [f"./mlos_bench/mlos_bench/run.py {cli_config} --log_file '{log_path}'"],
+            cwd=root_path)
         assert return_code == 0
 
-        with open(log_path, "rt", encoding="utf-8") as fh_out:
-            best_score_lines = [
-                ln.strip() for ln in fh_out.readlines()
-                if " INFO Env: Mock environment best score: " in ln
-            ]
-            assert len([
-                ln for ln in best_score_lines
-                if " best score: 65.67" in ln
-            ]) == 1
+        try:
+            iter_expected = iter(re_expected)
+            re_log = re.compile(next(iter_expected))
+            with open(log_path, "rt", encoding="utf-8") as fh_out:
+                for ln in fh_out:
+                    if re_log.match(ln):
+                        re_log = re.compile(next(iter_expected))
+            assert False, f"Pattern not found: '{re_log.pattern}'"
+        except StopIteration:
+            pass  # Success: all patterns found
+
+
+_RE_DATE = r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}"
+
+
+def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecService) -> None:
+    """
+    Run mlos_bench command-line application with mock benchmark config
+    and check the results in the log.
+    """
+    _launch_main_app(
+        root_path, local_exec_service,
+        "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
+        [
+            f"^{_RE_DATE} run\\.py:\\d+ " +
+            r"_optimize INFO Env: Mock environment best score: 65\.67\d+\s*$",
+        ]
+    )
+
+
+def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecService) -> None:
+    """
+    Run mlos_bench command-line application with mock optimization config
+    and check the results in the log.
+    """
+    _launch_main_app(
+        root_path, local_exec_service,
+        "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc --max_iterations 3",
+        [
+            # Iteration 1: Expect first value to be the baseline
+            f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +
+            r"register DEBUG Score: 65\.67\d+ Dataframe:\s*$",
+            # Iteration 2: The result may not always be deterministic
+            f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +
+            r"register DEBUG Score: \d+\.\d+ Dataframe:\s*$",
+            # Iteration 3: non-deterministic (depends on the optimizer)
+            f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +
+            r"register DEBUG Score: \d+\.\d+ Dataframe:\s*$",
+            # Final result: baseline is the optimum for the mock environment
+            f"^{_RE_DATE} run\\.py:\\d+ " +
+            r"_optimize INFO Env: Mock environment best score: 65\.67\d+\s*$",
+        ]
+    )