Repository: microsoft/MLOS

Issue:
Provide a way to setup top-level services in configs for easier reuse
Right now environments specify which services they need.

That means that if you want to change how an environment is created (e.g., on AWS (if that existed) vs. Azure), you'd need to edit the environment config to link to the new service provider.

That limits reuse of environment configs.

Alternatively, we could specify the services in the cli using something like `--service some/service-list.mlos.jsonc`, and include that in the `--config cli/config.mlos.jsonc`.

Note: this method still has limitations, especially if you're mixing and matching remote_exec methods (e.g., Azure VM remote_exec for one VM and SSH for another), but I think that use case is probably less common and can handle it in the current way - copying/editing the environment configs.


Discussion:

Resolving patch:
diff --git a/mlos_bench/mlos_bench/launcher.py b/mlos_bench/mlos_bench/launcher.py
--- a/mlos_bench/mlos_bench/launcher.py
+++ b/mlos_bench/mlos_bench/launcher.py
@@ -31,9 +31,12 @@
 
 from mlos_bench.storage.base_storage import Storage
 
+from mlos_bench.services.base_service import Service
 from mlos_bench.services.local.local_exec import LocalExecService
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 
+from mlos_bench.services.types.config_loader_type import SupportsConfigLoading
+
 
 _LOG_LEVEL = logging.INFO
 _LOG_FORMAT = '%(asctime)s %(filename)s:%(lineno)d %(funcName)s %(levelname)s %(message)s'
@@ -88,7 +91,7 @@ def __init__(self, description: str, long_text: str = "", argv: Optional[List[st
             log_handler.setFormatter(logging.Formatter(_LOG_FORMAT))
             logging.root.addHandler(log_handler)
 
-        self._parent_service = LocalExecService(parent=self._config_loader)
+        self._parent_service: Service = LocalExecService(parent=self._config_loader)
 
         self.global_config = self._load_config(
             config.get("globals", []) + (args.globals or []),
@@ -106,6 +109,11 @@ def __init__(self, description: str, long_text: str = "", argv: Optional[List[st
         # configs but is typically controlled by the run optimize loop.
         self.global_config.setdefault('trial_id', 1)
 
+        # --service cli args should override the config file values.
+        service_files: List[str] = config.get("services", []) + (args.service or [])
+        assert isinstance(self._parent_service, SupportsConfigLoading)
+        self._parent_service = self._parent_service.load_services(service_files, self.global_config, self._parent_service)
+
         env_path = args.environment or config.get("environment")
         if not env_path:
             _LOG.error("No environment config specified.")
@@ -140,6 +148,13 @@ def config_loader(self) -> ConfigPersistenceService:
         """
         return self._config_loader
 
+    @property
+    def service(self) -> Service:
+        """
+        Get the parent service.
+        """
+        return self._parent_service
+
     @staticmethod
     def _parse_args(parser: argparse.ArgumentParser, argv: Optional[List[str]]) -> Tuple[argparse.Namespace, List[str]]:
         """
@@ -167,9 +182,14 @@ def _parse_args(parser: argparse.ArgumentParser, argv: Optional[List[str]]) -> T
             nargs="+", action='extend', required=False,
             help='One or more locations of JSON config files.')
 
+        parser.add_argument(
+            '--service', '--services',
+            nargs='+', action='extend', required=False,
+            help='Path to JSON file with the configuration of the service(s) for environment(s) to use.')
+
         parser.add_argument(
             '--environment', required=False,
-            help='Path to JSON file with the configuration of the benchmarking environment.')
+            help='Path to JSON file with the configuration of the benchmarking environment(s).')
 
         parser.add_argument(
             '--optimizer', required=False,
diff --git a/mlos_bench/mlos_bench/services/types/__init__.py b/mlos_bench/mlos_bench/services/types/__init__.py
--- a/mlos_bench/mlos_bench/services/types/__init__.py
+++ b/mlos_bench/mlos_bench/services/types/__init__.py
@@ -6,6 +6,7 @@
 Service types for implementing declaring Service behavior for Environments to use in mlos_bench.
 """
 
+from mlos_bench.services.types.authenticator_type import SupportsAuth
 from mlos_bench.services.types.config_loader_type import SupportsConfigLoading
 from mlos_bench.services.types.fileshare_type import SupportsFileShareOps
 from mlos_bench.services.types.host_provisioner_type import SupportsHostProvisioning
@@ -15,6 +16,7 @@
 
 
 __all__ = [
+    'SupportsAuth',
     'SupportsConfigLoading',
     'SupportsFileShareOps',
     'SupportsHostProvisioning',
diff --git a/mlos_bench/mlos_bench/services/types/config_loader_type.py b/mlos_bench/mlos_bench/services/types/config_loader_type.py
--- a/mlos_bench/mlos_bench/services/types/config_loader_type.py
+++ b/mlos_bench/mlos_bench/services/types/config_loader_type.py
@@ -6,7 +6,7 @@
 Protocol interface for helper functions to lookup and load configs.
 """
 
-from typing import Dict, List, Iterable, Optional, Union, Protocol, runtime_checkable, TYPE_CHECKING
+from typing import Any, Dict, List, Iterable, Optional, Union, Protocol, runtime_checkable, TYPE_CHECKING
 
 from mlos_bench.config.schemas import ConfigSchema
 from mlos_bench.tunables.tunable import TunableValue
@@ -127,3 +127,25 @@ def load_environment_list(  # pylint: disable=too-many-arguments
         env : List[Environment]
             A list of new benchmarking environments.
         """
+
+    def load_services(self, json_file_names: Iterable[str],
+                      global_config: Optional[Dict[str, Any]] = None,
+                      parent: Optional["Service"] = None) -> "Service":
+        """
+        Read the configuration files and bundle all service methods
+        from those configs into a single Service object.
+
+        Parameters
+        ----------
+        json_file_names : list of str
+            A list of service JSON configuration files.
+        global_config : dict
+            Global parameters to add to the service config.
+        parent : Service
+            An optional reference of the parent service to mix in.
+
+        Returns
+        -------
+        service : Service
+            A collection of service methods.
+        """


Test patch:
diff --git a/mlos_bench/mlos_bench/tests/config/cli/test-cli-config.jsonc b/mlos_bench/mlos_bench/tests/config/cli/test-cli-config.jsonc
--- a/mlos_bench/mlos_bench/tests/config/cli/test-cli-config.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/cli/test-cli-config.jsonc
@@ -9,6 +9,10 @@
         "bar"
     ],
 
+    "services": [
+        "services/remote/mock/mock_fileshare_service.jsonc"
+    ],
+
     "random_seed": 42,
     "random_init": true
 }
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/bad/invalid/min-services-items.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/bad/invalid/min-services-items.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/bad/invalid/min-services-items.jsonc
@@ -0,0 +1,5 @@
+{
+    "services": [
+        // must have at least one item
+    ]
+}
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/bad/invalid/wrong-services-type.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/bad/invalid/wrong-services-type.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/bad/invalid/wrong-services-type.jsonc
@@ -0,0 +1,5 @@
+{
+    "services": [
+        "conf/path/wrong.ext"
+    ]
+}
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/good/full/full-cli.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/good/full/full-cli.jsonc
--- a/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/good/full/full-cli.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/cli/test-cases/good/full/full-cli.jsonc
@@ -4,6 +4,13 @@
         "mlos_bench/mlos_bench/tests/config"
     ],
 
+    "services": [
+        // start by default with ssh based exec and file services
+        // can be overridden at the environment level as well
+        "services/remote/ssh/ssh_host_service.jsonc",
+        "services/remote/ssh/ssh_fileshare_service.jsonc"
+    ],
+
     "environment": "environments/root/root-azure-redis.jsonc",
     "optimizer": "optimizers/one_shot_opt.jsonc",
     "storage": "storage/sqlite.jsonc",
diff --git a/mlos_bench/mlos_bench/tests/launcher_parse_args_test.py b/mlos_bench/mlos_bench/tests/launcher_parse_args_test.py
--- a/mlos_bench/mlos_bench/tests/launcher_parse_args_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_parse_args_test.py
@@ -18,6 +18,13 @@
 from mlos_bench.optimizers import MockOptimizer
 from mlos_bench.config.schemas import ConfigSchema
 from mlos_bench.util import path_join
+from mlos_bench.services.types import (
+    SupportsAuth,
+    SupportsConfigLoading,
+    SupportsFileShareOps,
+    SupportsLocalExec,
+    SupportsRemoteExec,
+)
 from mlos_bench.tests import check_class_name
 
 if sys.version_info < (3, 10):
@@ -62,11 +69,18 @@ def test_launcher_args_parse_1(config_paths: List[str]) -> None:
     # This is part of the minimal required args by the Launcher.
     env_conf_path = 'environments/mock/mock_env.jsonc'
     cli_args = '--config-paths ' + ' '.join(config_paths) + \
+        ' --service services/remote/mock/mock_auth_service.jsonc' + \
+        ' --service services/remote/mock/mock_remote_exec_service.jsonc' + \
         f' --environment {env_conf_path}' + \
         ' --globals globals/global_test_config.jsonc' + \
         ' --globals globals/global_test_extra_config.jsonc' \
         ' --test_global_value_2 from-args'
     launcher = Launcher(description=__name__, argv=cli_args.split())
+    # Check that the parent service
+    assert isinstance(launcher.service, SupportsAuth)
+    assert isinstance(launcher.service, SupportsConfigLoading)
+    assert isinstance(launcher.service, SupportsLocalExec)
+    assert isinstance(launcher.service, SupportsRemoteExec)
     # Check that the first --globals file is loaded and $var expansion is handled.
     assert launcher.global_config['experiment_id'] == 'MockExperiment'
     assert launcher.global_config['testVmName'] == 'MockExperiment-vm'
@@ -107,12 +121,20 @@ def test_launcher_args_parse_2(config_paths: List[str]) -> None:
     config_file = 'cli/test-cli-config.jsonc'
     cli_args = ' '.join([f"--config-path {config_path}" for config_path in config_paths]) + \
         f' --config {config_file}' + \
+        ' --service services/remote/mock/mock_auth_service.jsonc' + \
+        ' --service services/remote/mock/mock_remote_exec_service.jsonc' + \
         ' --globals globals/global_test_config.jsonc' + \
         ' --experiment_id MockeryExperiment' + \
         ' --no-teardown' + \
         ' --random-init' + \
         ' --random-seed 1234'
     launcher = Launcher(description=__name__, argv=cli_args.split())
+    # Check that the parent service
+    assert isinstance(launcher.service, SupportsAuth)
+    assert isinstance(launcher.service, SupportsConfigLoading)
+    assert isinstance(launcher.service, SupportsFileShareOps)
+    assert isinstance(launcher.service, SupportsLocalExec)
+    assert isinstance(launcher.service, SupportsRemoteExec)
     # Check that the --globals file is loaded and $var expansion is handled
     # using the value provided on the CLI.
     assert launcher.global_config['experiment_id'] == 'MockeryExperiment'