Repository: facebookresearch/hydra

Issue:
Command line interpolations are not being evaluated lazily, leading to changed multirun behaviour
In v0.11, when I wanted to do multirun with certain parameters dependent on each other, I would do something like
```
./train.py param1=1,2,3 param2=\${param1} -m
```
and this would launch 3 jobs, with same values for param1 and param2.

With v1.0, this seems to be launching 3*3 = 9 jobs, with all possible combinations of param1 and param2. Wondering if that's expected behavior? My understanding was that the interpolations are evaluated lazily, so it should have the same behavior as in v0.11.

## System information
- **Hydra Version** :  1.0rc1
- **Python version** : python 3.7.7
- **Virtual environment type and version** : conda 4.7.10
- **Operating system** : Linux



Discussion:
Here is my actual usecase: basically I have been using this functionality to couple variables, for eg, if I am running a model on 5 folds, and I want to load a pre-trained model corresponding to the specific fold, I would do something like this:
```
./train.py fold_id=0,1,2,3,4 pretrained=/checkpoint/\${fold_id}
```
will take a look, for now you can use this as a possible workaround:

Use a config group with this:

**things/model_per_fold.yaml**
```yaml
fold_id: ???
pretrained: /checkpoint/${fold_id}
```
and then from the command line do:
```
$ python foo.py +things=model_per_fold fold_id=0,1,2,3,4
```

Patch:
diff --git a/hydra/_internal/config_loader_impl.py b/hydra/_internal/config_loader_impl.py
--- a/hydra/_internal/config_loader_impl.py
+++ b/hydra/_internal/config_loader_impl.py
@@ -82,7 +82,7 @@ def __init__(
             config_search_path=config_search_path
         )
 
-    def split_overrides(
+    def split_by_override_type(
         self, overrides: List[Override],
     ) -> Tuple[List[Override], List[Override]]:
         config_group_overrides = []
@@ -141,41 +141,51 @@ def load_configuration(
         strict: Optional[bool] = None,
         from_shell: bool = True,
     ) -> DictConfig:
+        if config_name is not None and not self.repository.config_exists(config_name):
+            self.missing_config_error(
+                config_name=config_name,
+                msg=f"Cannot find primary config : {config_name}, check that it's in your config search path",
+                with_search_path=True,
+            )
+
         if strict is None:
             strict = self.default_strict
 
         parser = OverridesParser()
         parsed_overrides = parser.parse_overrides(overrides=overrides)
-        filtered_parsed_overrides = []
+        config_overrides = []
+        sweep_overrides = []
         for x in parsed_overrides:
             if x.is_sweep_override():
-                if from_shell:
-                    quoted = f"{x.get_key_element()}=\\'{x.get_value_string()}\\'"
-                else:
-                    quoted = f"{x.get_key_element()}='{x.get_value_string()}'"
-                if run_mode == RunMode.RUN:
-                    msg = f"""Ambiguous value for argument '{x.input_line}'
-1. To use it as a list, use {x.get_key_element()}=[{x.get_value_string()}]
-2. To use it as string use {quoted}
-3. To sweep over it, add --multirun to your command line"""
-                    raise HydraException(msg)
-                elif run_mode == RunMode.MULTIRUN:
+                if run_mode == RunMode.MULTIRUN:
                     if x.is_hydra_override():
                         raise HydraException(
                             f"Sweeping over Hydra's configuration is not supported : '{x.input_line}'"
                         )
+                    sweep_overrides.append(x)
+                elif run_mode == RunMode.RUN:
+                    if from_shell:
+                        example_override = (
+                            f"{x.get_key_element()}=\\'{x.get_value_string()}\\'"
+                        )
                     else:
-                        # filter, the sweeper will pass pass concrete overrides to individual jobs
-                        pass
+                        example_override = (
+                            f"{x.get_key_element()}='{x.get_value_string()}'"
+                        )
+
+                    msg = f"""Ambiguous value for argument '{x.input_line}'
+1. To use it as a list, use {x.get_key_element()}=[{x.get_value_string()}]
+2. To use it as string use {example_override}
+3. To sweep over it, add --multirun to your command line"""
+                    raise HydraException(msg)
+                else:
+                    assert False
             else:
-                filtered_parsed_overrides.append(x)
+                config_overrides.append(x)
 
-        if config_name is not None and not self.repository.config_exists(config_name):
-            self.missing_config_error(
-                config_name=config_name,
-                msg=f"Cannot find primary config : {config_name}, check that it's in your config search path",
-                with_search_path=True,
-            )
+        config_group_overrides, config_overrides = self.split_by_override_type(
+            config_overrides
+        )
 
         # Load hydra config
         hydra_cfg, _load_trace = self._load_primary_config(cfg_filename="hydra_config")
@@ -203,10 +213,6 @@ def load_configuration(
             defaults.append(DefaultElement(config_group=None, config_name="__SELF__"))
         split_at = len(defaults)
 
-        config_group_overrides, config_overrides = self.split_overrides(
-            filtered_parsed_overrides
-        )
-
         self._combine_default_lists(defaults, job_defaults)
         ConfigLoaderImpl._apply_overrides_to_defaults(config_group_overrides, defaults)
 
@@ -345,7 +351,6 @@ def _apply_overrides_to_defaults(
                         f"\nUse ~{override.key_or_group}"
                     )
                     warnings.warn(category=UserWarning, message=msg)
-
             if (
                 not (override.is_delete() or override.is_package_rename())
                 and value is None
diff --git a/hydra/_internal/core_plugins/basic_sweeper.py b/hydra/_internal/core_plugins/basic_sweeper.py
--- a/hydra/_internal/core_plugins/basic_sweeper.py
+++ b/hydra/_internal/core_plugins/basic_sweeper.py
@@ -19,6 +19,7 @@
 from typing import Any, Iterable, List, Optional, Sequence
 
 from omegaconf import MISSING, DictConfig, OmegaConf
+from omegaconf._utils import get_yaml_loader
 
 from hydra.core.config_loader import ConfigLoader
 from hydra.core.config_store import ConfigStore
@@ -72,17 +73,22 @@ def setup(
             config=config, config_loader=config_loader, task_function=task_function
         )
 
+    loader = get_yaml_loader()
+
     @staticmethod
     def split_overrides_to_chunks(
-        lst: Sequence[Sequence[str]], n: Optional[int]
-    ) -> Iterable[Sequence[Sequence[str]]]:
+        lst: List[List[str]], n: Optional[int]
+    ) -> Iterable[List[List[str]]]:
         if n is None or n == -1:
             n = len(lst)
         assert n > 0
         for i in range(0, len(lst), n):
             yield lst[i : i + n]
 
-    def initialize_arguments(self, arguments: List[str]) -> None:
+    @staticmethod
+    def split_arguments(
+        arguments: List[str], max_batch_size: Optional[int]
+    ) -> List[List[List[str]]]:
         parser = OverridesParser()
         parsed = parser.parse_overrides(arguments)
 
@@ -98,19 +104,20 @@ def initialize_arguments(self, arguments: List[str]) -> None:
                 key = override.get_key_element()
                 value = override.get_value_element()
                 lists.append([f"{key}={value}"])
-        all_batches = list(itertools.product(*lists))
-        assert self.max_batch_size is None or self.max_batch_size > 0
-        if self.max_batch_size is None:
-            self.overrides = [all_batches]
+        all_batches = [list(x) for x in itertools.product(*lists)]
+        assert max_batch_size is None or max_batch_size > 0
+        if max_batch_size is None:
+            return [all_batches]
         else:
-            self.overrides = list(
-                self.split_overrides_to_chunks(all_batches, self.max_batch_size)
+            chunks_iter = BasicSweeper.split_overrides_to_chunks(
+                all_batches, max_batch_size
             )
+            return [x for x in chunks_iter]
 
     def sweep(self, arguments: List[str]) -> Any:
         assert self.config is not None
         assert self.launcher is not None
-        self.initialize_arguments(arguments)
+        self.overrides = self.split_arguments(arguments, self.max_batch_size)
         returns: List[Sequence[JobReturn]] = []
 
         # Save sweep run config in top level sweep working directory
diff --git a/hydra/_internal/hydra.py b/hydra/_internal/hydra.py
--- a/hydra/_internal/hydra.py
+++ b/hydra/_internal/hydra.py
@@ -127,7 +127,8 @@ def multirun(
         sweeper = Plugins.instance().instantiate_sweeper(
             config=cfg, config_loader=self.config_loader, task_function=task_function
         )
-        task_overrides = cfg.hydra.overrides.task
+        task_overrides = OmegaConf.to_container(cfg.hydra.overrides.task, resolve=False)
+        assert isinstance(task_overrides, list)
         return sweeper.sweep(arguments=task_overrides)
 
     @staticmethod
diff --git a/hydra/core/utils.py b/hydra/core/utils.py
--- a/hydra/core/utils.py
+++ b/hydra/core/utils.py
@@ -189,7 +189,7 @@ def split_config_path(
         config_dir: Optional[str] = dirname(config_path)
         msg = (
             "\nUsing config_path to specify the config name is deprecated, specify the config name via config_name"
-            "\nSee https://hydra.cc/next/upgrades/0.11_to_1.0/config_path_changes"
+            "\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/config_path_changes"
         )
         warnings.warn(
             category=UserWarning, message=msg,


Test patch:
diff --git a/tests/test_apps/simple_app/my_app.py b/tests/test_apps/simple_app/my_app.py
--- a/tests/test_apps/simple_app/my_app.py
+++ b/tests/test_apps/simple_app/my_app.py
@@ -6,7 +6,7 @@
 
 @hydra.main()
 def my_app(cfg: DictConfig) -> None:
-    print(cfg.pretty())
+    print(cfg.pretty(resolve=True))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_basic_sweeper.py b/tests/test_basic_sweeper.py
new file mode 100644
--- /dev/null
+++ b/tests/test_basic_sweeper.py
@@ -0,0 +1,51 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from typing import List, Optional
+
+import pytest
+
+from hydra._internal.core_plugins.basic_sweeper import BasicSweeper
+
+
+@pytest.mark.parametrize(  # type:ignore
+    "args,max_batch_size,expected",
+    [
+        pytest.param(["x=10"], None, [[["x=10"]]], id="simple"),
+        pytest.param(["x=10,20"], None, [[["x=10"], ["x=20"]]], id="split_1d"),
+        pytest.param(["x=[10,20]"], None, [[["x=[10,20]"]]], id="not_split_yaml_list"),
+        pytest.param(
+            ["x=[a,b,c],[d,e,f]"],
+            None,
+            [[["x=[a,b,c]"], ["x=[d,e,f]"]]],
+            id="list_of_lists",
+        ),
+        pytest.param(
+            ["a=1,2", "b=10,11"],
+            None,
+            [[["a=1", "b=10"], ["a=1", "b=11"], ["a=2", "b=10"], ["a=2", "b=11"]]],
+            id="no_batching",
+        ),
+        pytest.param(
+            ["a=1,2", "b=10,11"],
+            1,
+            [
+                [["a=1", "b=10"]],
+                [["a=1", "b=11"]],
+                [["a=2", "b=10"]],
+                [["a=2", "b=11"]],
+            ],
+            id="batches_of_1",
+        ),
+        pytest.param(
+            ["a=1,2", "b=10,11"],
+            2,
+            [[["a=1", "b=10"], ["a=1", "b=11"]], [["a=2", "b=10"], ["a=2", "b=11"]]],
+            id="batches_of_2",
+        ),
+    ],
+)
+def test_split(
+    args: List[str], max_batch_size: Optional[int], expected: List[List[List[str]]]
+) -> None:
+    ret = BasicSweeper.split_arguments(args, max_batch_size=max_batch_size)
+    lret = [list(x) for x in ret]
+    assert lret == expected
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -213,7 +213,7 @@ def test_app_with_config_path_backward_compatibility(
 ) -> None:
     msg = (
         "\nUsing config_path to specify the config name is deprecated, specify the config name via config_name"
-        "\nSee https://hydra.cc/next/upgrades/0.11_to_1.0/config_path_changes"
+        "\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/config_path_changes"
     )
 
     with pytest.warns(expected_warning=UserWarning, match=re.escape(msg)):
@@ -888,24 +888,103 @@ def test_multirun_structured_conflict(
         assert ret == expected
 
 
-@pytest.mark.parametrize(  # type: ignore
-    "sweep", [pytest.param(False, id="run"), pytest.param(True, id="sweep")]
+@pytest.mark.parametrize(
+    "cmd_base",
+    [
+        (
+            [
+                sys.executable,
+                "tests/test_apps/simple_app/my_app.py",
+                "hydra/hydra_logging=disabled",
+            ]
+        )
+    ],
 )
-def test_run_with_missing_default(tmpdir: Any, sweep: bool) -> None:
-    cmd = [
-        sys.executable,
-        "tests/test_apps/simple_app/my_app.py",
-        "--config-name=unspecified_mandatory_default",
-        "--config-path=../../../hydra/test_utils/configs",
-        "hydra/hydra_logging=disabled",
-        "hydra.sweep.dir=" + str(tmpdir),
-    ]
-    if sweep:
-        cmd.append("-m")
-    expected = """You must specify 'group1', e.g, group1=<OPTION>
+class TestVariousRuns:
+    @pytest.mark.parametrize(  # type: ignore
+        "sweep", [pytest.param(False, id="run"), pytest.param(True, id="sweep")]
+    )
+    def test_run_with_missing_default(
+        self, cmd_base: List[str], tmpdir: Any, sweep: bool
+    ) -> None:
+        cmd = cmd_base + [
+            "hydra.sweep.dir=" + str(tmpdir),
+            "--config-name=unspecified_mandatory_default",
+            "--config-path=../../../hydra/test_utils/configs",
+        ]
+        if sweep:
+            cmd.append("-m")
+        expected = """You must specify 'group1', e.g, group1=<OPTION>
 Available options:
 \tabc.cde
 \tfile1
 \tfile2"""
-    ret = run_with_error(cmd)
-    assert re.search(re.escape(expected), ret) is not None
+        ret = run_with_error(cmd)
+        assert re.search(re.escape(expected), ret) is not None
+
+    def test_command_line_interpolations_evaluated_lazily(
+        self, cmd_base: List[str], tmpdir: Any
+    ) -> None:
+
+        cmd = cmd_base + [
+            "hydra.sweep.dir=" + str(tmpdir),
+            "+foo=10,20",
+            "+bar=${foo}",
+            "--multirun",
+        ]
+        expected = """foo: 10
+bar: 10
+
+foo: 20
+bar: 20
+
+"""
+        ret = str(subprocess.check_output(cmd).decode("utf-8"))
+        assert normalize_newlines(ret) == normalize_newlines(expected)
+
+    def test_multirun_config_overrides_evaluated_lazily(
+        self, cmd_base: List[str], tmpdir: Any
+    ) -> None:
+
+        cmd = cmd_base + [
+            "hydra.sweep.dir=" + str(tmpdir),
+            "+foo=10,20",
+            "+bar=${foo}",
+            "--multirun",
+        ]
+        expected = """foo: 10
+bar: 10
+
+foo: 20
+bar: 20
+
+"""
+        ret = str(subprocess.check_output(cmd).decode("utf-8"))
+        assert normalize_newlines(ret) == normalize_newlines(expected)
+
+    def test_multirun_defaults_override(self, cmd_base: List[str], tmpdir: Any) -> None:
+        cmd = cmd_base + [
+            "hydra.sweep.dir=" + str(tmpdir),
+            "group1=file1,file2",
+            "--multirun",
+            "--config-path=../../../hydra/test_utils/configs",
+            "--config-name=compose",
+        ]
+        expected = """foo: 10
+bar: 100
+
+foo: 20
+bar: 100
+
+"""
+        ret = str(subprocess.check_output(cmd).decode("utf-8"))
+        assert normalize_newlines(ret) == normalize_newlines(expected)
+
+    def test_run_pass_list(self, cmd_base: List[str], tmpdir: Any) -> None:
+        cmd = cmd_base + [
+            "hydra.sweep.dir=" + str(tmpdir),
+            "+foo=[1,2,3]",
+        ]
+        expected = {"foo": [1, 2, 3]}
+        ret = str(subprocess.check_output(cmd).decode("utf-8"))
+        assert OmegaConf.create(ret) == OmegaConf.create(expected)