Repository: microsoft/MLOS

Issue:
Document the importance of `experiment_id` parameter.
We need to add docs on why this parameter and what it controls (e.g., reloading trial data, etc.) and how changes to scripts, tunable params, configs, etc. can affect reusability of trial data in ways that we can't always detect (e.g., code, version, etc. lineage).


Discussion:

Resolving patch:
diff --git a/mlos_bench/mlos_bench/launcher.py b/mlos_bench/mlos_bench/launcher.py
--- a/mlos_bench/mlos_bench/launcher.py
+++ b/mlos_bench/mlos_bench/launcher.py
@@ -14,6 +14,7 @@
 import logging
 import os
 import sys
+import textwrap
 
 from string import Template
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
@@ -50,7 +51,15 @@ class Launcher:
 
     def __init__(self, description: str, long_text: str = "", argv: Optional[List[str]] = None):
         _LOG.info("Launch: %s", description)
-        parser = argparse.ArgumentParser(description=f"{description} : {long_text}")
+        epilog = textwrap.dedent("""
+            Additional --key=value pairs can be specified to augment or override values listed in --globals.
+            Other required_args values can also be pulled from shell environment variables.
+
+            For additional details, please see the website or the README.md files in the source tree:
+            <https://github.com/microsoft/MLOS/tree/main/mlos_bench/>
+            """)
+        parser = argparse.ArgumentParser(description=f"{description} : {long_text}",
+                                         epilog=epilog)
         (args, args_rest) = self._parse_args(parser, argv)
 
         # Bootstrap config loader: command line takes priority.
@@ -88,6 +97,10 @@ def __init__(self, description: str, long_text: str = "", argv: Optional[List[st
             args_rest,
             {key: val for (key, val) in config.items() if key not in vars(args)},
         )
+        # experiment_id is generally taken from --globals files, but we also allow overriding it on the CLI.
+        # It's useful to keep it there explicitly mostly for the --help output.
+        if args.experiment_id:
+            self.global_config['experiment_id'] = args.experiment_id
         self.global_config = self._expand_vars(self.global_config)
         assert isinstance(self.global_config, dict)
         # Ensure that the trial_id is present since it gets used by some other
@@ -194,6 +207,20 @@ def _parse_args(parser: argparse.ArgumentParser, argv: Optional[List[str]]) -> T
             dest='teardown', action='store_false',
             help='Disable teardown of the environment after the benchmark.')
 
+        parser.add_argument(
+            '--experiment_id', '--experiment-id', required=False, default=None,
+            help=textwrap.dedent("""
+                Experiment ID to use for the benchmark.
+                If omitted, the value from the --cli config or --globals is used.
+
+                This is used to store and reload trial results from the storage.
+                NOTE: It is **important** to change this value when incompatible
+                changes are made to config files, scripts, versions, etc.
+                This is left as a manual operation as detection of what is
+                "incompatible" is not easily automatable across systems.
+                """)
+        )
+
         # By default we use the command line arguments, but allow the caller to
         # provide some explicitly for testing purposes.
         if argv is None:


Test patch: