Repository: facebookresearch/hydra

Issue:
[Bug] Sweeping when some defaults refer others through interpolation fails
# 🐛 Bug

When sweeping over configs, and using those config values to load different config files based on the value of the swept parameters, the defaults are not loaded correctly. 

## To reproduce
To mirror my codebase, I made this structure:
```
.
├── a.py
├── conf
│   ├── config.yaml
│   ├── dynamics_model
│   │   ├── de.yaml
│   │   ├── d.yaml
│   │   ├── env.yaml
│   │   ├── pe.yaml
│   │   └── p.yaml
│   ├── env
│   │   ├── cartpole.yaml
│   │   └── halfcheetah.yaml
│   ├── env_details
│   │   ├── cartpole
│   │   │   ├── dynamics_model
│   │   │   │   ├── de.yaml
│   │   │   │   ├── d.yaml
│   │   │   │   ├── pe.yaml
│   │   │   │   └── p.yaml
│   │   │   └── optimizer
│   │   │       ├── cem.yaml
│   │   │       └── random.yaml
│   │   └── halfcheetah
│   │       ├── dynamics_model
│   │       │   ├── de.yaml
│   │       │   ├── d.yaml
│   │       │   ├── pe.yaml
│   │       │   └── p.yaml
│   │       └── optimizer
│   │           ├── cem.yaml
│   │           └── random.yaml
│   └── optimizer
│       ├── cem.yaml
│       └── random.yaml

```
Files found here: https://drive.google.com/file/d/1Nl2umsvEwOKjkVlYe1QBapWKpZMMl9Ly/view?usp=sharing

And base config:
```
defaults:
  - env: ???
  - dynamics_model: pe
  - env_details: ${defaults.0.env}/dynamics_model/${defaults.1.dynamics_model}
  - optimizer: cem
  - env_details: ${defaults.0.env}/optimizer/${defaults.3.optimizer}
```

Running the code (on the newest version) produces an error:
```
python a.py -m env=cartpole optimizer=random,cem
```
And this code runs:
```
python a.py -m env=cartpole optimizer=random
```

** Stack trace/error message **
```
  File "a.py", line 7, in <module>
    fo()
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/main.py", line 25, in decorated_main
    strict,
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/utils.py", line 50, in run_hydra
    hydra.multirun(overrides=args.overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/hydra.py", line 115, in multirun
    cfg = self._load_config(overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/hydra.py", line 271, in _load_config
    cfg = self.config_loader.load_configuration(overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 56, in load_configuration
    cfg = self._merge_defaults(hydra_cfg, defaults, split_at)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 394, in _merge_defaults
    cfg = merge_defaults(cfg, user_list)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 367, in merge_defaults
    name = default1[family]
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 75, in __getitem__
    return self.__getattr__(key)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 67, in __getattr__
    return self.get(key=key, default_value=None)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 78, in get
    return self._resolve_with_default(key=key, value=self.get_node(key, default_value), default_value=default_value)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 132, in _resolve_with_default
    return self._resolve_single(value) if isinstance(value, str) else value
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 433, in _resolve_single
    new_val = Config._resolve_value(root, match.group(1), match.group(2))
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 408, in _resolve_value
    raise KeyError("{} interpolation key '{}' not found".format(inter_type, inter_key))
KeyError: "str interpolation key 'defaults.3.optimizer' not found"

```
## Expected Behavior
Starting multiple jobs swept over the parameters, with new configs loaded from the directories

## System information
- <0.9.0>
- <macOS 10.14.6, linux as well>

[Bug] Sweeping when some defaults refer others through interpolation fails
# 🐛 Bug

When sweeping over configs, and using those config values to load different config files based on the value of the swept parameters, the defaults are not loaded correctly. 

## To reproduce
To mirror my codebase, I made this structure:
```
.
├── a.py
├── conf
│   ├── config.yaml
│   ├── dynamics_model
│   │   ├── de.yaml
│   │   ├── d.yaml
│   │   ├── env.yaml
│   │   ├── pe.yaml
│   │   └── p.yaml
│   ├── env
│   │   ├── cartpole.yaml
│   │   └── halfcheetah.yaml
│   ├── env_details
│   │   ├── cartpole
│   │   │   ├── dynamics_model
│   │   │   │   ├── de.yaml
│   │   │   │   ├── d.yaml
│   │   │   │   ├── pe.yaml
│   │   │   │   └── p.yaml
│   │   │   └── optimizer
│   │   │       ├── cem.yaml
│   │   │       └── random.yaml
│   │   └── halfcheetah
│   │       ├── dynamics_model
│   │       │   ├── de.yaml
│   │       │   ├── d.yaml
│   │       │   ├── pe.yaml
│   │       │   └── p.yaml
│   │       └── optimizer
│   │           ├── cem.yaml
│   │           └── random.yaml
│   └── optimizer
│       ├── cem.yaml
│       └── random.yaml

```
Files found here: https://drive.google.com/file/d/1Nl2umsvEwOKjkVlYe1QBapWKpZMMl9Ly/view?usp=sharing

And base config:
```
defaults:
  - env: ???
  - dynamics_model: pe
  - env_details: ${defaults.0.env}/dynamics_model/${defaults.1.dynamics_model}
  - optimizer: cem
  - env_details: ${defaults.0.env}/optimizer/${defaults.3.optimizer}
```

Running the code (on the newest version) produces an error:
```
python a.py -m env=cartpole optimizer=random,cem
```
And this code runs:
```
python a.py -m env=cartpole optimizer=random
```

** Stack trace/error message **
```
  File "a.py", line 7, in <module>
    fo()
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/main.py", line 25, in decorated_main
    strict,
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/utils.py", line 50, in run_hydra
    hydra.multirun(overrides=args.overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/hydra.py", line 115, in multirun
    cfg = self._load_config(overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/hydra.py", line 271, in _load_config
    cfg = self.config_loader.load_configuration(overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 56, in load_configuration
    cfg = self._merge_defaults(hydra_cfg, defaults, split_at)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 394, in _merge_defaults
    cfg = merge_defaults(cfg, user_list)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 367, in merge_defaults
    name = default1[family]
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 75, in __getitem__
    return self.__getattr__(key)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 67, in __getattr__
    return self.get(key=key, default_value=None)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 78, in get
    return self._resolve_with_default(key=key, value=self.get_node(key, default_value), default_value=default_value)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 132, in _resolve_with_default
    return self._resolve_single(value) if isinstance(value, str) else value
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 433, in _resolve_single
    new_val = Config._resolve_value(root, match.group(1), match.group(2))
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 408, in _resolve_value
    raise KeyError("{} interpolation key '{}' not found".format(inter_type, inter_key))
KeyError: "str interpolation key 'defaults.3.optimizer' not found"

```
## Expected Behavior
Starting multiple jobs swept over the parameters, with new configs loaded from the directories

## System information
- <0.9.0>
- <macOS 10.14.6, linux as well>

[Bug] Sweeping when some defaults refer others through interpolation fails
# 🐛 Bug

When sweeping over configs, and using those config values to load different config files based on the value of the swept parameters, the defaults are not loaded correctly. 

## To reproduce
To mirror my codebase, I made this structure:
```
.
├── a.py
├── conf
│   ├── config.yaml
│   ├── dynamics_model
│   │   ├── de.yaml
│   │   ├── d.yaml
│   │   ├── env.yaml
│   │   ├── pe.yaml
│   │   └── p.yaml
│   ├── env
│   │   ├── cartpole.yaml
│   │   └── halfcheetah.yaml
│   ├── env_details
│   │   ├── cartpole
│   │   │   ├── dynamics_model
│   │   │   │   ├── de.yaml
│   │   │   │   ├── d.yaml
│   │   │   │   ├── pe.yaml
│   │   │   │   └── p.yaml
│   │   │   └── optimizer
│   │   │       ├── cem.yaml
│   │   │       └── random.yaml
│   │   └── halfcheetah
│   │       ├── dynamics_model
│   │       │   ├── de.yaml
│   │       │   ├── d.yaml
│   │       │   ├── pe.yaml
│   │       │   └── p.yaml
│   │       └── optimizer
│   │           ├── cem.yaml
│   │           └── random.yaml
│   └── optimizer
│       ├── cem.yaml
│       └── random.yaml

```
Files found here: https://drive.google.com/file/d/1Nl2umsvEwOKjkVlYe1QBapWKpZMMl9Ly/view?usp=sharing

And base config:
```
defaults:
  - env: ???
  - dynamics_model: pe
  - env_details: ${defaults.0.env}/dynamics_model/${defaults.1.dynamics_model}
  - optimizer: cem
  - env_details: ${defaults.0.env}/optimizer/${defaults.3.optimizer}
```

Running the code (on the newest version) produces an error:
```
python a.py -m env=cartpole optimizer=random,cem
```
And this code runs:
```
python a.py -m env=cartpole optimizer=random
```

** Stack trace/error message **
```
  File "a.py", line 7, in <module>
    fo()
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/main.py", line 25, in decorated_main
    strict,
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/utils.py", line 50, in run_hydra
    hydra.multirun(overrides=args.overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/hydra.py", line 115, in multirun
    cfg = self._load_config(overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/hydra.py", line 271, in _load_config
    cfg = self.config_loader.load_configuration(overrides)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 56, in load_configuration
    cfg = self._merge_defaults(hydra_cfg, defaults, split_at)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 394, in _merge_defaults
    cfg = merge_defaults(cfg, user_list)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/hydra/_internal/config_loader.py", line 367, in merge_defaults
    name = default1[family]
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 75, in __getitem__
    return self.__getattr__(key)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 67, in __getattr__
    return self.get(key=key, default_value=None)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/dictconfig.py", line 78, in get
    return self._resolve_with_default(key=key, value=self.get_node(key, default_value), default_value=default_value)
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 132, in _resolve_with_default
    return self._resolve_single(value) if isinstance(value, str) else value
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 433, in _resolve_single
    new_val = Config._resolve_value(root, match.group(1), match.group(2))
  File "/private/home/nol/.conda/envs/pets/lib/python3.6/site-packages/omegaconf/config.py", line 408, in _resolve_value
    raise KeyError("{} interpolation key '{}' not found".format(inter_type, inter_key))
KeyError: "str interpolation key 'defaults.3.optimizer' not found"

```
## Expected Behavior
Starting multiple jobs swept over the parameters, with new configs loaded from the directories

## System information
- <0.9.0>
- <macOS 10.14.6, linux as well>



Discussion:

Resolving patch:
diff --git a/hydra/_internal/config_loader.py b/hydra/_internal/config_loader.py
--- a/hydra/_internal/config_loader.py
+++ b/hydra/_internal/config_loader.py
@@ -136,9 +136,9 @@ def _apply_defaults_overrides(overrides, defaults):
             key, value = override.split("=")
             if key in key_to_idx:
                 if "," in value:
-                    # If this is a multirun config (comma separated list), unset the default for the off chance
-                    # It's a ???. it will get added again when we construct the config for the job.
-                    value = "null"
+                    # If this is a multirun config (comma separated list), flag the default to prevent it from being
+                    # loaded until we are constructing the config for individual jobs.
+                    value = "_SKIP_"
 
                 if value == "null":
                     del defaults[key_to_idx[key]]
@@ -366,7 +366,7 @@ def merge_defaults(merged_cfg, def_list):
                     family = next(iter(default1.keys()))
                     name = default1[family]
                     # Name is none if default value is removed
-                    if name is not None:
+                    if name is not None and "_SKIP_" not in name:
                         merged_cfg = self._merge_config(
                             cfg=merged_cfg,
                             family=family,
@@ -375,12 +375,13 @@ def merge_defaults(merged_cfg, def_list):
                         )
                 else:
                     assert isinstance(default1, str)
-                    merged_cfg = self._merge_config(
-                        cfg=merged_cfg,
-                        family="",
-                        name=get_filename(default1),
-                        required=True,
-                    )
+                    if "_SKIP_" not in default1:
+                        merged_cfg = self._merge_config(
+                            cfg=merged_cfg,
+                            family="",
+                            name=get_filename(default1),
+                            required=True,
+                        )
             return merged_cfg
 
         system_list = []


Test patch:
diff --git a/tests/test_app/sweep_cfg/config.yaml b/tests/test_app/sweep_cfg/config.yaml
deleted file mode 100644
--- a/tests/test_app/sweep_cfg/config.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-defaults:
-  - hydra/launcher: fairtask
\ No newline at end of file
diff --git a/tests/test_app/__init__.py b/tests/test_apps/__init__.py
similarity index 100%
rename from tests/test_app/__init__.py
rename to tests/test_apps/__init__.py
diff --git a/tests/test_app/cfg_groups/__init__.py b/tests/test_apps/app_with_cfg/__init__.py
similarity index 100%
rename from tests/test_app/cfg_groups/__init__.py
rename to tests/test_apps/app_with_cfg/__init__.py
diff --git a/tests/test_app/config.yaml b/tests/test_apps/app_with_cfg/config.yaml
similarity index 100%
rename from tests/test_app/config.yaml
rename to tests/test_apps/app_with_cfg/config.yaml
diff --git a/tests/test_app/app_with_cfg.py b/tests/test_apps/app_with_cfg/my_app.py
similarity index 81%
rename from tests/test_app/app_with_cfg.py
rename to tests/test_apps/app_with_cfg/my_app.py
--- a/tests/test_app/app_with_cfg.py
+++ b/tests/test_apps/app_with_cfg/my_app.py
@@ -3,9 +3,9 @@
 
 
 @hydra.main(config_path="config.yaml")
-def experiment(_):
+def my_app(_):
     pass
 
 
 if __name__ == "__main__":
-    experiment()
+    my_app()
diff --git a/tests/test_app/cfg_groups/optimizer/__init__.py b/tests/test_apps/app_with_cfg_groups/__init__.py
similarity index 100%
rename from tests/test_app/cfg_groups/optimizer/__init__.py
rename to tests/test_apps/app_with_cfg_groups/__init__.py
diff --git a/tests/test_app/split_cfg/__init__.py b/tests/test_apps/app_with_cfg_groups/conf/__init__.py
similarity index 100%
rename from tests/test_app/split_cfg/__init__.py
rename to tests/test_apps/app_with_cfg_groups/conf/__init__.py
diff --git a/tests/test_app/cfg_groups/config.yaml b/tests/test_apps/app_with_cfg_groups/conf/config.yaml
similarity index 100%
rename from tests/test_app/cfg_groups/config.yaml
rename to tests/test_apps/app_with_cfg_groups/conf/config.yaml
diff --git a/tests/test_app/split_cfg/imagenet.yaml b/tests/test_apps/app_with_cfg_groups/conf/dataset/imagenet.yaml
similarity index 100%
rename from tests/test_app/split_cfg/imagenet.yaml
rename to tests/test_apps/app_with_cfg_groups/conf/dataset/imagenet.yaml
diff --git a/tests/test_app/cfg_groups/optimizer/nesterov.yaml b/tests/test_apps/app_with_cfg_groups/conf/dataset/nesterov.yaml
similarity index 100%
rename from tests/test_app/cfg_groups/optimizer/nesterov.yaml
rename to tests/test_apps/app_with_cfg_groups/conf/dataset/nesterov.yaml
diff --git a/tests/test_app/cfg_groups/optimizer/adam.yaml b/tests/test_apps/app_with_cfg_groups/conf/optimizer/adam.yaml
similarity index 100%
rename from tests/test_app/cfg_groups/optimizer/adam.yaml
rename to tests/test_apps/app_with_cfg_groups/conf/optimizer/adam.yaml
diff --git a/tests/test_app/split_cfg/nesterov.yaml b/tests/test_apps/app_with_cfg_groups/conf/optimizer/nesterov.yaml
similarity index 100%
rename from tests/test_app/split_cfg/nesterov.yaml
rename to tests/test_apps/app_with_cfg_groups/conf/optimizer/nesterov.yaml
diff --git a/tests/test_apps/app_with_cfg_groups/my_app.py b/tests/test_apps/app_with_cfg_groups/my_app.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_cfg_groups/my_app.py
@@ -0,0 +1,11 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import hydra
+
+
+@hydra.main(config_path="conf/")
+def my_app(_):
+    pass
+
+
+if __name__ == "__main__":
+    my_app()
diff --git a/tests/test_apps/app_with_config_with_free_group/__init__.py b/tests/test_apps/app_with_config_with_free_group/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_config_with_free_group/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
diff --git a/tests/test_app/config_with_free_group/config.yaml b/tests/test_apps/app_with_config_with_free_group/conf/config.yaml
similarity index 100%
rename from tests/test_app/config_with_free_group/config.yaml
rename to tests/test_apps/app_with_config_with_free_group/conf/config.yaml
diff --git a/tests/test_app/config_with_free_group/free_group/opt1.yaml b/tests/test_apps/app_with_config_with_free_group/conf/free_group/opt1.yaml
similarity index 100%
rename from tests/test_app/config_with_free_group/free_group/opt1.yaml
rename to tests/test_apps/app_with_config_with_free_group/conf/free_group/opt1.yaml
diff --git a/tests/test_app/config_with_free_group/free_group/opt2.yaml b/tests/test_apps/app_with_config_with_free_group/conf/free_group/opt2.yaml
similarity index 100%
rename from tests/test_app/config_with_free_group/free_group/opt2.yaml
rename to tests/test_apps/app_with_config_with_free_group/conf/free_group/opt2.yaml
diff --git a/tests/test_app/config_with_free_group/group/opt1.yaml b/tests/test_apps/app_with_config_with_free_group/conf/group/opt1.yaml
similarity index 100%
rename from tests/test_app/config_with_free_group/group/opt1.yaml
rename to tests/test_apps/app_with_config_with_free_group/conf/group/opt1.yaml
diff --git a/tests/test_app/config_with_free_group/group/opt2.yaml b/tests/test_apps/app_with_config_with_free_group/conf/group/opt2.yaml
similarity index 100%
rename from tests/test_app/config_with_free_group/group/opt2.yaml
rename to tests/test_apps/app_with_config_with_free_group/conf/group/opt2.yaml
diff --git a/tests/test_apps/app_with_config_with_free_group/my_app.py b/tests/test_apps/app_with_config_with_free_group/my_app.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_config_with_free_group/my_app.py
@@ -0,0 +1,11 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import hydra
+
+
+@hydra.main(config_path="conf/config.yaml")
+def my_app(_):
+    pass
+
+
+if __name__ == "__main__":
+    my_app()
diff --git a/tests/test_apps/app_with_custom_launcher/__init__.py b/tests/test_apps/app_with_custom_launcher/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_custom_launcher/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
diff --git a/tests/test_apps/app_with_custom_launcher/config.yaml b/tests/test_apps/app_with_custom_launcher/config.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_custom_launcher/config.yaml
@@ -0,0 +1,2 @@
+defaults:
+  - hydra/launcher: custom_launcher
\ No newline at end of file
diff --git a/tests/test_apps/app_with_custom_launcher/my_app.py b/tests/test_apps/app_with_custom_launcher/my_app.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_custom_launcher/my_app.py
@@ -0,0 +1,11 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import hydra
+
+
+@hydra.main(config_path="config.yaml")
+def my_app(_):
+    pass
+
+
+if __name__ == "__main__":
+    my_app()
diff --git a/tests/test_apps/app_with_split_cfg/__init__.py b/tests/test_apps/app_with_split_cfg/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_split_cfg/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
diff --git a/tests/test_app/split_cfg/config.yaml b/tests/test_apps/app_with_split_cfg/config.yaml
similarity index 100%
rename from tests/test_app/split_cfg/config.yaml
rename to tests/test_apps/app_with_split_cfg/config.yaml
diff --git a/tests/test_apps/app_with_split_cfg/imagenet.yaml b/tests/test_apps/app_with_split_cfg/imagenet.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_split_cfg/imagenet.yaml
@@ -0,0 +1,3 @@
+dataset:
+  name: imagenet
+  path: /datasets/imagenet
\ No newline at end of file
diff --git a/tests/test_apps/app_with_split_cfg/my_app.py b/tests/test_apps/app_with_split_cfg/my_app.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_split_cfg/my_app.py
@@ -0,0 +1,11 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import hydra
+
+
+@hydra.main(config_path="config.yaml")
+def my_app(_):
+    pass
+
+
+if __name__ == "__main__":
+    my_app()
diff --git a/tests/test_apps/app_with_split_cfg/nesterov.yaml b/tests/test_apps/app_with_split_cfg/nesterov.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_split_cfg/nesterov.yaml
@@ -0,0 +1,3 @@
+optimizer:
+  type: nesterov
+  lr: 0.001
\ No newline at end of file
diff --git a/tests/test_apps/app_without_config/__init__.py b/tests/test_apps/app_without_config/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_without_config/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
diff --git a/tests/test_app/app.py b/tests/test_apps/app_without_config/my_app.py
similarity index 79%
rename from tests/test_app/app.py
rename to tests/test_apps/app_without_config/my_app.py
--- a/tests/test_app/app.py
+++ b/tests/test_apps/app_without_config/my_app.py
@@ -3,9 +3,9 @@
 
 
 @hydra.main()
-def experiment(_):
+def my_app(_):
     pass
 
 
 if __name__ == "__main__":
-    experiment()
+    my_app()
diff --git a/tests/test_apps/sweep_complex_defaults/__init__.py b/tests/test_apps/sweep_complex_defaults/__init__.py
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/sweep_complex_defaults/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
diff --git a/tests/test_apps/sweep_complex_defaults/conf/config.yaml b/tests/test_apps/sweep_complex_defaults/conf/config.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/sweep_complex_defaults/conf/config.yaml
@@ -0,0 +1,5 @@
+defaults:
+  - optimizer: adam
+  - dataset: imagenet
+  - optimizer_dataset: ${defaults.0.optimizer}_${defaults.1.dataset}
+    optional: true
\ No newline at end of file
diff --git a/tests/test_apps/sweep_complex_defaults/conf/dataset/imagenet.yaml b/tests/test_apps/sweep_complex_defaults/conf/dataset/imagenet.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/sweep_complex_defaults/conf/dataset/imagenet.yaml
@@ -0,0 +1,3 @@
+dataset:
+  name: imagenet
+  path: /datasets/imagenet
\ No newline at end of file
diff --git a/tests/test_apps/sweep_complex_defaults/conf/dataset/nesterov.yaml b/tests/test_apps/sweep_complex_defaults/conf/dataset/nesterov.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/sweep_complex_defaults/conf/dataset/nesterov.yaml
@@ -0,0 +1,3 @@
+optimizer:
+  type: nesterov
+  lr: 0.001
\ No newline at end of file
diff --git a/tests/test_apps/sweep_complex_defaults/conf/optimizer/adam.yaml b/tests/test_apps/sweep_complex_defaults/conf/optimizer/adam.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/sweep_complex_defaults/conf/optimizer/adam.yaml
@@ -0,0 +1,4 @@
+optimizer:
+  type: adam
+  lr: 0.1
+  beta: 0.01
\ No newline at end of file
diff --git a/tests/test_apps/sweep_complex_defaults/conf/optimizer/nesterov.yaml b/tests/test_apps/sweep_complex_defaults/conf/optimizer/nesterov.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/sweep_complex_defaults/conf/optimizer/nesterov.yaml
@@ -0,0 +1,3 @@
+optimizer:
+  type: nesterov
+  lr: 0.001
\ No newline at end of file
diff --git a/tests/test_app/app_with_free_group.py b/tests/test_apps/sweep_complex_defaults/my_app.py
similarity index 73%
rename from tests/test_app/app_with_free_group.py
rename to tests/test_apps/sweep_complex_defaults/my_app.py
--- a/tests/test_app/app_with_free_group.py
+++ b/tests/test_apps/sweep_complex_defaults/my_app.py
@@ -2,7 +2,7 @@
 import hydra
 
 
-@hydra.main(config_path="config_with_free_group/config.yaml")
+@hydra.main(config_path="conf/config.yaml")
 def my_app(_cfg):
     print(_cfg.pretty())
 
diff --git a/tests/test_basic_launcher.py b/tests/test_basic_launcher.py
--- a/tests/test_basic_launcher.py
+++ b/tests/test_basic_launcher.py
@@ -2,9 +2,6 @@
 import pytest
 
 from hydra.test_utils.launcher_common_tests import LauncherTestSuite
-from hydra.test_utils.test_utils import chdir_hydra_root
-
-chdir_hydra_root()
 
 # noinspection PyUnresolvedReferences
 from hydra.test_utils.test_utils import sweep_runner  # noqa: F401
diff --git a/tests/test_hydra.py b/tests/test_hydra.py
--- a/tests/test_hydra.py
+++ b/tests/test_hydra.py
@@ -27,7 +27,8 @@ def test_missing_conf_dir(task_runner, calling_file, calling_module):  # noqa: F
 
 
 @pytest.mark.parametrize(
-    "calling_file, calling_module", [("hydra/app.py", None), (None, "hydra.app")]
+    "calling_file, calling_module",
+    [("invalid_dir/app.py", None), (None, "invalid_module.app")],
 )
 def test_missing_conf_file(task_runner, calling_file, calling_module):  # noqa: F811
     with pytest.raises(MissingConfigException):
@@ -41,7 +42,10 @@ def test_missing_conf_file(task_runner, calling_file, calling_module):  # noqa:
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app.py", None), (None, "tests.test_app.app")],
+    [
+        ("tests/test_apps/app_without_config/my_app.py", None),
+        (None, "tests.test_apps.app_without_config.my_app"),
+    ],
 )
 def test_app_without_config___no_overrides(
     task_runner, calling_file, calling_module  # noqa: F811
@@ -54,7 +58,10 @@ def test_app_without_config___no_overrides(
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app.py", None), (None, "tests.test_app.app")],
+    [
+        ("tests/test_apps/app_without_config/my_app.py", None),
+        (None, "tests.test_apps.app_without_config.my_app"),
+    ],
 )
 def test_app_without_config__with_overrides(
     task_runner, calling_file, calling_module  # noqa: F811
@@ -71,7 +78,10 @@ def test_app_without_config__with_overrides(
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app_with_cfg.py", None), (None, "tests.test_app.app_with_cfg")],
+    [
+        ("tests/test_apps/app_with_cfg/my_app.py", None),
+        (None, "tests.test_apps.app_with_cfg.my_app"),
+    ],
 )
 def test_app_with_config_file__no_overrides(
     task_runner, calling_file, calling_module  # noqa: F811
@@ -90,7 +100,10 @@ def test_app_with_config_file__no_overrides(
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app_with_cfg.py", None), (None, "tests.test_app.app_with_cfg")],
+    [
+        ("tests/test_apps/app_with_cfg/my_app.py", None),
+        (None, "tests.test_apps.app_with_cfg.my_app"),
+    ],
 )
 def test_app_with_config_file__with_overide(
     task_runner, calling_file, calling_module  # noqa: F811
@@ -109,13 +122,16 @@ def test_app_with_config_file__with_overide(
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app_with_cfg.py", None), (None, "tests.test_app.app_with_cfg")],
+    [
+        ("tests/test_apps/app_with_split_cfg/my_app.py", None),
+        (None, "tests.test_apps.app_with_split_cfg.my_app"),
+    ],
 )
 def test_app_with_split_config(task_runner, calling_file, calling_module):  # noqa: F811
     with task_runner(
         calling_file=calling_file,
         calling_module=calling_module,
-        config_path="split_cfg/config.yaml",
+        config_path="config.yaml",
     ) as task:
         assert task.job_ret.cfg == dict(
             dataset=dict(name="imagenet", path="/datasets/imagenet"),
@@ -126,7 +142,10 @@ def test_app_with_split_config(task_runner, calling_file, calling_module):  # no
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app_with_cfg.py", None), (None, "tests.test_app.app_with_cfg")],
+    [
+        ("tests/test_apps/app_with_cfg_groups/my_app.py", None),
+        (None, "tests.test_apps.app_with_cfg_groups.my_app"),
+    ],
 )
 def test_app_with_config_groups__override_dataset__wrong(
     task_runner, calling_file, calling_module  # noqa: F811
@@ -135,7 +154,7 @@ def test_app_with_config_groups__override_dataset__wrong(
         with task_runner(
             calling_file=calling_file,
             calling_module=calling_module,
-            config_path="cfg_groups",
+            config_path="conf",
             overrides=["optimizer=wrong_name"],
         ):
             pass
@@ -144,7 +163,10 @@ def test_app_with_config_groups__override_dataset__wrong(
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app_with_cfg.py", None), (None, "tests.test_app.app_with_cfg")],
+    [
+        ("tests/test_apps/app_with_cfg_groups/my_app.py", None),
+        (None, "tests.test_apps.app_with_cfg_groups.my_app"),
+    ],
 )
 def test_app_with_config_groups__override_all_configs(
     task_runner, calling_file, calling_module  # noqa: F811
@@ -152,7 +174,7 @@ def test_app_with_config_groups__override_all_configs(
     with task_runner(
         calling_file=calling_file,
         calling_module=calling_module,
-        config_path="cfg_groups",
+        config_path="conf",
         overrides=["optimizer=adam", "optimizer.lr=10"],
     ) as task:
         assert task.job_ret.cfg == dict(optimizer=dict(type="adam", lr=10, beta=0.01))
@@ -161,7 +183,10 @@ def test_app_with_config_groups__override_all_configs(
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app_with_cfg.py", None), (None, "tests.test_app.app_with_cfg")],
+    [
+        ("tests/test_apps/app_with_custom_launcher/my_app.py", None),
+        (None, "tests.test_apps.app_with_custom_launcher.my_app"),
+    ],
 )
 def test_app_with_sweep_cfg__override_to_basic_launcher(
     task_runner, calling_file, calling_module  # noqa: F811
@@ -169,7 +194,7 @@ def test_app_with_sweep_cfg__override_to_basic_launcher(
     with task_runner(
         calling_file=calling_file,
         calling_module=calling_module,
-        config_path="sweep_cfg/config.yaml",
+        config_path="config.yaml",
         overrides=["hydra/launcher=basic"],
     ) as task:
         hydra_cfg = task.job_ret.hydra_cfg
@@ -196,7 +221,10 @@ def test_short_module_name(tmpdir):
 
 @pytest.mark.parametrize(
     "calling_file, calling_module",
-    [("tests/test_app/app_with_cfg.py", None), (None, "tests.test_app.app_with_cfg")],
+    [
+        ("tests/test_apps/app_with_config_with_free_group/my_app.py", None),
+        (None, "tests.test_apps.app_with_config_with_free_group.my_app"),
+    ],
 )
 @pytest.mark.parametrize("overrides", [["free_group=opt1,opt2"]])
 def test_multirun_with_free_override(
@@ -205,7 +233,7 @@ def test_multirun_with_free_override(
     sweep = sweep_runner(
         calling_file=calling_file,
         calling_module=calling_module,
-        config_path="config_with_free_group/config.yaml",
+        config_path="conf/config.yaml",
         overrides=overrides,
         strict=True,
     )
@@ -215,3 +243,24 @@ def test_multirun_with_free_override(
         assert sweep.returns[0][0].cfg == {"group_opt1": True, "free_group_opt1": True}
         assert sweep.returns[0][1].overrides == ["free_group=opt2"]
         assert sweep.returns[0][1].cfg == {"group_opt1": True, "free_group_opt2": True}
+
+
+@pytest.mark.parametrize(
+    "calling_file, calling_module",
+    [
+        ("tests/test_apps/sweep_complex_defaults/my_app.py", None),
+        (None, "tests.test_apps.sweep_complex_defaults.my_app"),
+    ],
+)
+def test_sweep_complex_defaults(
+    sweep_runner, calling_file, calling_module  # noqa: F811
+):
+    with sweep_runner(
+        calling_file=calling_file,
+        calling_module=calling_module,
+        config_path="conf/config.yaml",
+        overrides=["optimizer=adam,nesterov"],
+    ) as sweep:
+        assert len(sweep.returns[0]) == 2
+        assert sweep.returns[0][0].overrides == ["optimizer=adam"]
+        assert sweep.returns[0][1].overrides == ["optimizer=nesterov"]