Repository: facebookresearch/hydra

Issue:
[Feature Request] Define a sweep in config.yaml file rather than command line
# 🚀 Feature Request

Currently a sweep can only be defined on the command line. Could it be supported through the config files as well?

## Motivation / pitch
Situations where this particularly makes sense: `python train.py seed="range(0,10)" xval_fold="range(0,5)" actual_hyperparam_I_want_to_change=1,2,3`
-- if for any new hyperparameter combination we always want to run over those several seeds and cross-validation folds.

For those cases I'd like to be able to have those sweeps in `config.yaml`, to declutter the command line.

Couldn't find prior discussions about this, but somewhat related is #394 

cc @liujas000 


Discussion:
Defining a sweep in config file is supported in all Hydra sweepers([example](https://hydra.cc/docs/next/plugins/ax_sweeper)) except for the BasicSweeper (the one you are using)

@omry does it make sense for us to add this support to the BasicSweeper? 
I think this can be added as a part of the change to standardize the configuration of the search space of HPO sweepers.

We have some work in that direction, but it's stalled. (cc @shagunsodhani).
I can't find a corresponding issue for this feature request.

To set expectations:
None of this is likely to happen in time for Hydra 1.1.

Resolving patch:
diff --git a/examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/my_app.py b/examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/my_app.py
new file mode 100644
--- /dev/null
+++ b/examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/my_app.py
@@ -0,0 +1,13 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from omegaconf import DictConfig
+
+import hydra
+
+
+@hydra.main(config_path="conf", config_name="config")
+def my_app(cfg: DictConfig) -> None:
+    print(f"driver={cfg.db.driver}, timeout={cfg.db.timeout}")
+
+
+if __name__ == "__main__":
+    my_app()
diff --git a/hydra/_internal/core_plugins/basic_sweeper.py b/hydra/_internal/core_plugins/basic_sweeper.py
--- a/hydra/_internal/core_plugins/basic_sweeper.py
+++ b/hydra/_internal/core_plugins/basic_sweeper.py
@@ -19,9 +19,10 @@
 import itertools
 import logging
 import time
+from collections import OrderedDict
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Iterable, List, Optional, Sequence
+from typing import Any, Dict, Iterable, List, Optional, Sequence
 
 from omegaconf import DictConfig, OmegaConf
 
@@ -39,6 +40,7 @@
 class BasicSweeperConf:
     _target_: str = "hydra._internal.core_plugins.basic_sweeper.BasicSweeper"
     max_batch_size: Optional[int] = None
+    params: Optional[Dict[str, str]] = None
 
 
 ConfigStore.instance().store(
@@ -54,14 +56,20 @@ class BasicSweeper(Sweeper):
     Basic sweeper
     """
 
-    def __init__(self, max_batch_size: Optional[int]) -> None:
+    def __init__(
+        self, max_batch_size: Optional[int], params: Optional[Dict[str, str]] = None
+    ) -> None:
         """
         Instantiates
         """
         super(BasicSweeper, self).__init__()
+
+        if params is None:
+            params = {}
         self.overrides: Optional[Sequence[Sequence[Sequence[str]]]] = None
         self.batch_index = 0
         self.max_batch_size = max_batch_size
+        self.params = params
 
         self.hydra_context: Optional[HydraContext] = None
         self.config: Optional[DictConfig] = None
@@ -101,12 +109,13 @@ def split_arguments(
     ) -> List[List[List[str]]]:
 
         lists = []
+        final_overrides = OrderedDict()
         for override in overrides:
             if override.is_sweep_override():
                 if override.is_discrete_sweep():
                     key = override.get_key_element()
                     sweep = [f"{key}={val}" for val in override.sweep_string_iterator()]
-                    lists.append(sweep)
+                    final_overrides[key] = sweep
                 else:
                     assert override.value_type is not None
                     raise HydraException(
@@ -115,7 +124,10 @@ def split_arguments(
             else:
                 key = override.get_key_element()
                 value = override.get_value_element_as_str()
-                lists.append([f"{key}={value}"])
+                final_overrides[key] = [f"{key}={value}"]
+
+        for _, v in final_overrides.items():
+            lists.append(v)
 
         all_batches = [list(x) for x in itertools.product(*lists)]
         assert max_batch_size is None or max_batch_size > 0
@@ -127,13 +139,22 @@ def split_arguments(
             )
             return [x for x in chunks_iter]
 
+    def _parse_config(self) -> List[str]:
+        params_conf = []
+        for k, v in self.params.items():
+            params_conf.append(f"{k}={v}")
+        return params_conf
+
     def sweep(self, arguments: List[str]) -> Any:
         assert self.config is not None
         assert self.launcher is not None
         assert self.hydra_context is not None
 
+        params_conf = self._parse_config()
+        params_conf.extend(arguments)
+
         parser = OverridesParser.create(config_loader=self.hydra_context.config_loader)
-        overrides = parser.parse_overrides(arguments)
+        overrides = parser.parse_overrides(params_conf)
 
         self.overrides = self.split_arguments(overrides, self.max_batch_size)
         returns: List[Sequence[JobReturn]] = []


Test patch:
diff --git a/tests/test_examples/test_basic_sweep.py b/tests/test_examples/test_basic_sweep.py
new file mode 100644
--- /dev/null
+++ b/tests/test_examples/test_basic_sweep.py
@@ -0,0 +1,85 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from pathlib import Path
+from textwrap import dedent
+from typing import List
+
+from pytest import mark
+
+from hydra.test_utils.test_utils import (
+    assert_regex_match,
+    chdir_hydra_root,
+    run_python_script,
+)
+
+chdir_hydra_root()
+
+
+@mark.parametrize(
+    "args,expected",
+    [
+        (
+            [],
+            dedent(
+                """\
+                [HYDRA] Launching 4 jobs locally
+                [HYDRA] \t#0 : db=mysql db.timeout=5
+                driver=mysql, timeout=5
+                [HYDRA] \t#1 : db=mysql db.timeout=10
+                driver=mysql, timeout=10
+                [HYDRA] \t#2 : db=postgresql db.timeout=5
+                driver=postgresql, timeout=5
+                [HYDRA] \t#3 : db=postgresql db.timeout=10
+                driver=postgresql, timeout=10"""
+            ),
+        ),
+        (
+            ["db=glob([m*],exclude=postgresql)"],
+            dedent(
+                """\
+                [HYDRA] Launching 2 jobs locally
+                [HYDRA] \t#0 : db=mysql db.timeout=5
+                driver=mysql, timeout=5
+                [HYDRA] \t#1 : db=mysql db.timeout=10
+                driver=mysql, timeout=10"""
+            ),
+        ),
+        (
+            ["db=mysql", "db.user=choice(one,two)"],
+            dedent(
+                """\
+                [HYDRA] Launching 4 jobs locally
+                [HYDRA] \t#0 : db=mysql db.timeout=5 db.user=one
+                driver=mysql, timeout=5
+                [HYDRA] \t#1 : db=mysql db.timeout=5 db.user=two
+                driver=mysql, timeout=5
+                [HYDRA] \t#2 : db=mysql db.timeout=10 db.user=one
+                driver=mysql, timeout=10
+                [HYDRA] \t#3 : db=mysql db.timeout=10 db.user=two
+                driver=mysql, timeout=10"""
+            ),
+        ),
+    ],
+)
+def test_basic_sweep_example(
+    tmpdir: Path,
+    args: List[str],
+    expected: str,
+) -> None:
+    app_path = "examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/my_app.py"
+
+    cmd = [
+        app_path,
+        "--multirun",
+        "hydra.run.dir=" + str(tmpdir),
+        "hydra.hydra_logging.formatters.simple.format='[HYDRA] %(message)s'",
+        "hydra.job_logging.formatters.simple.format='[JOB] %(message)s'",
+    ]
+    cmd.extend(args)
+    result, _err = run_python_script(cmd)
+
+    assert_regex_match(
+        from_line=expected,
+        to_line=result,
+        from_name="Expected output",
+        to_name="Actual output",
+    )