Repository: facebookresearch/hydra

Issue:
[Bug] example of override fail in multirun
This fails

`python examples/tutorial/5_composition/my_app.py -m db=mysql,postgresql db.user=omry`


Discussion:
While constructing the main config, Hydra is skipping the db config group overrides as they are a part of a sweep and will be loaded later.
when merging in db.user, there is no such node and this causes an error in strict mode.
This is expected behavior, solving it automatically will require validating that we can apply db.user to each of the sweep configs.
This can be very expensive for large sweeps so it's not a viable solution.
Instead, the solution is for the user to create scaffolding for db.user in the main config, for example:
```
db:
  user: ???
```

This makes sense because overriding db.user in a sweep essentially calls out db.user as existing in all sweep variants.
I want to keep thinking about this one.

Resolving patch:
diff --git a/hydra/_internal/config_loader.py b/hydra/_internal/config_loader.py
--- a/hydra/_internal/config_loader.py
+++ b/hydra/_internal/config_loader.py
@@ -31,7 +31,9 @@ def __init__(self, config_file, strict_cfg, config_search_path):
         self.all_config_checked = []
         self.strict_cfg = strict_cfg
 
-    def load_configuration(self, overrides=[]):
+    def load_configuration(self, overrides=[], strict=None):
+        if strict is None:
+            strict = self.strict_cfg
         assert overrides is None or isinstance(overrides, list)
         overrides = overrides or []
 
@@ -55,7 +57,8 @@ def load_configuration(self, overrides=[]):
         # Load and defaults and merge them into cfg
         cfg = self._merge_defaults(hydra_cfg, defaults, split_at)
 
-        OmegaConf.set_struct(cfg, self.strict_cfg)
+        OmegaConf.set_struct(cfg.hydra, True)
+        OmegaConf.set_struct(cfg, strict)
 
         # Merge all command line overrides after enabling strict flag
         all_consumed = consumed + consumed_free_job_defaults
@@ -86,10 +89,10 @@ def load_sweep_config(self, master_config, sweep_overrides):
         # Recreate the config for this sweep instance with the appropriate overrides
         hydra_overrides = OmegaConf.to_container(master_config.hydra.overrides.hydra)
         overrides = hydra_overrides + sweep_overrides
-        sweep_config = self.load_configuration(overrides)
+        sweep_config = self.load_configuration(overrides, self.strict_cfg)
 
         with open_dict(sweep_config):
-            sweep_config.hydra.runtime = master_config.hydra.runtime
+            sweep_config.hydra.runtime.merge_with(master_config.hydra.runtime)
 
         # Copy old config cache to ensure we get the same resolved values (for things like timestamps etc)
         OmegaConf.copy_cache(from_config=master_config, to_config=sweep_config)
diff --git a/hydra/_internal/hydra.py b/hydra/_internal/hydra.py
--- a/hydra/_internal/hydra.py
+++ b/hydra/_internal/hydra.py
@@ -99,7 +99,7 @@ def __init__(
             )
 
     def run(self, overrides):
-        cfg = self._load_config(overrides)
+        cfg = self._load_config(overrides, strict=None)
         HydraConfig().set_config(cfg)
         return run_job(
             config=cfg,
@@ -109,7 +109,7 @@ def run(self, overrides):
         )
 
     def multirun(self, overrides):
-        cfg = self._load_config(overrides)
+        cfg = self._load_config(overrides, strict=False)
         HydraConfig().set_config(cfg)
         sweeper = Plugins.instantiate_sweeper(
             config=cfg,
@@ -335,8 +335,14 @@ def _print_debug_info(self):
         self._print_search_path()
         self._print_composition_trace()
 
-    def _load_config(self, overrides):
-        cfg = self.config_loader.load_configuration(overrides)
+    def _load_config(self, overrides, strict=None):
+        """
+        :param overrides:
+        :param strict: None for default behavior (default to true for config file, false if no config file).
+                       otherwise forces specific behavior.
+        :return:
+        """
+        cfg = self.config_loader.load_configuration(overrides, strict=strict)
         with open_dict(cfg):
             from .. import __version__
 


Test patch:
diff --git a/hydra/test_utils/configs/db_conf.yaml b/hydra/test_utils/configs/db_conf.yaml
--- a/hydra/test_utils/configs/db_conf.yaml
+++ b/hydra/test_utils/configs/db_conf.yaml
@@ -1,5 +1,2 @@
 defaults:
   - db: mysql
-
-db:
-  user: ???
\ No newline at end of file
diff --git a/hydra/test_utils/launcher_common_tests.py b/hydra/test_utils/launcher_common_tests.py
--- a/hydra/test_utils/launcher_common_tests.py
+++ b/hydra/test_utils/launcher_common_tests.py
@@ -78,20 +78,6 @@ def test_sweep_over_unspecified_mandatory_default(
     def test_sweep_and_override(
         self, sweep_runner, launcher_name, overrides
     ):  # noqa: F811
-        """
-        This test validates the behavior that happens in issue #283.
-        While constructing the main config, Hydra is skipping the db config group overrides as they are a part of a
-        sweep and will be loaded later.
-        when merging in db.user, there is no such node and this causes an error in strict mode.
-        This is expected behavior, solving it automatically will require validating that we can apply db.user to
-        each of the sweep configs. This can be very expensive for large sweeps so it's not a viable solution.
-        Instead, the solution is for the user to create scaffolding for db.user in the main config, for example:
-        db:
-          user: ???
-
-        This makes sense because overriding db.user in a sweep essentially calls out db.user as
-        existing in all sweep variants.
-        """
         base_overrides = [
             "hydra/launcher=" + launcher_name,
             "db=mysql,postgresql",
@@ -183,7 +169,7 @@ def not_sweeping_hydra_overrides(sweep_runner, overrides):
     """
     Runs a sweep with two jobs
     """
-    overrides.extend(["a=0,1", "hydra.foo=1,2,3"])
+    overrides.extend(["a=0,1", "hydra.verbose=true,false"])
     sweep = sweep_runner(
         calling_file=None,
         calling_module="hydra.test_utils.a_module",