Repository: facebookresearch/hydra

Issue:
Crash after slurm training. 
```
(pytorch-c) yuandong@devfair0306:~/dl_theory2$ python two_layer_new2.py -s
[2019-08-12 11:28:50,983][hydra_plugins.fairtask.fairtask_launcher][INFO] - Sweep output dir : /checkpoint/yuandong/outputs/2019-08-12_11-28-50
[2019-08-12 11:28:51,039][hydra_plugins.fairtask.fairtask_launcher][INFO] - Launching 1 jobs to slurm queue
[2019-08-12 11:28:51,039][hydra_plugins.fairtask.fairtask_launcher][INFO] -     #0 :
/private/home/yuandong/.local/lib/python3.7/site-packages/distributed/deploy/local.py:112: UserWarning: The ip keyword has been moved to host
  warnings.warn("The ip keyword has been moved to host")
Dask dashboard for "slurm" at http://localhost:8006.
Traceback (most recent call last):
  File "two_layer_new2.py", line 222, in <module>
    main()
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/hydra/hydra.py", line 165, in decorated_main
    run_hydra(task_function, config_path, strict)
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/hydra/hydra.py", line 145, in run_hydra
    hydra.sweep(overrides=args.overrides)
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/hydra/hydra.py", line 88, in sweep
    return sweeper.sweep(arguments=cfg.hydra.overrides.task)
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/hydra/sweeper/step_sweeper.py", line 61, in sweep
    results = self.launcher.launch(batch)
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/hydra_plugins/fairtask/fairtask_launcher.py", line 96, in launch
    return loop.run_until_complete(self.run_sweep(queue, job_overrides))
  File "/private/home/yuandong/miniconda3/envs/pytorch-c/lib/python3.7/asyncio/base_events.py", line 584, in run_until_complete
    return future.result()
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/hydra_plugins/fairtask/fairtask_launcher.py", line 78, in run_sweep
    return await gatherl(runs)
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/fairtask/util.py", line 26, in gatherl
    return await multi(iters)
  File "/private/home/yuandong/miniconda3/envs/pytorch-c/lib/python3.7/site-packages/tornado/gen.py", line 501, in callback
    result_list.append(f.result())
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/fairtask/queue.py", line 418, in __call__
    return await self.submit_fn(*args, **kwargs)
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/fairtask/queue.py", line 248, in submit
    pure=pure,
  File "/private/home/yuandong/.local/lib/python3.7/site-packages/distributed/client.py", line 1412, in submit
    key = funcname(func) + "-" + tokenize(func, kwargs, *args)
  File "/private/home/yuandong/miniconda3/envs/pytorch-c/lib/python3.7/site-packages/dask/base.py", line 661, in tokenize
    return md5(str(tuple(map(normalize_token, args))).encode()).hexdigest()
  File "/private/home/yuandong/miniconda3/envs/pytorch-c/lib/python3.7/site-packages/dask/utils.py", line 486, in __call__
    return meth(arg, *args, **kwargs)
  File "/private/home/yuandong/miniconda3/envs/pytorch-c/lib/python3.7/site-packages/dask/base.py", line 705, in normalize_object
    method = getattr(o, "__dask_tokenize__", None)
  File "/private/home/yuandong/miniconda3/envs/pytorch-c/lib/python3.7/site-packages/omegaconf/dictconfig.py", line 72, in __getattr__
    return self.get(key=key, default_value=None)
  File "/private/home/yuandong/miniconda3/envs/pytorch-c/lib/python3.7/site-packages/omegaconf/dictconfig.py", line 83, in get
    return self._resolve_with_default(key=key, value=self.get_node(key), default_value=default_value)
  File "/private/home/yuandong/miniconda3/envs/pytorch-c/lib/python3.7/site-packages/omegaconf/dictconfig.py", line 88, in get_node
    raise KeyError("Accessing unknown key in a struct : {}".format(self.get_full_key(key)))
KeyError: 'Accessing unknown key in a struct : __dask_tokenize__'
```


Discussion:

Resolving patch:
diff --git a/hydra/config_loader.py b/hydra/config_loader.py
--- a/hydra/config_loader.py
+++ b/hydra/config_loader.py
@@ -31,7 +31,46 @@ def get_load_history(self):
         """
         return copy.deepcopy(self.all_config_checked)
 
-    def load_hydra_cfg(self, overrides):
+    def load_configuration(self, overrides=[]):
+        assert overrides is None or isinstance(overrides, list)
+
+        overrides = overrides or []
+        hydra_cfg = self._load_hydra_cfg(overrides)
+
+        job_overrides = [x for x in overrides if not x.startswith("hydra.")]
+        hydra_overrides = [x for x in overrides if x.startswith("hydra.")]
+        task_cfg, consumed_defaults = self._create_cfg(
+            cfg_dir=self.cfg_dir,
+            cfg_filename=self.conf_filename,
+            cli_overrides=job_overrides,
+            strict=self.strict_task_cfg,
+        )
+        cfg = OmegaConf.merge(hydra_cfg, task_cfg)
+
+        for item in job_overrides:
+            cfg.hydra.overrides.task.append(item)
+        for item in hydra_overrides:
+            cfg.hydra.overrides.hydra.append(item)
+
+        job = OmegaConf.create(dict(name=JobRuntime().get("name")))
+        cfg.hydra.job = OmegaConf.merge(job, cfg.hydra.job)
+        OmegaConf.set_struct(cfg, self.strict_task_cfg)
+        # Merge hydra overrides after combining both configs and applying the strict flag.
+        cfg.merge_with_dotlist(hydra_overrides)
+        return cfg
+
+    def load_sweep_config(self, master_config, sweep_overrides):
+        # Recreate the config for this sweep instance with the appropriate overrides
+        sweep_config = self.load_configuration(
+            master_config.hydra.overrides.hydra.to_container() + sweep_overrides
+        )
+
+        # Copy old config cache to ensure we get the same resolved values (for things like timestamps etc)
+        OmegaConf.copy_cache(from_config=master_config, to_config=sweep_config)
+
+        return sweep_config
+
+    def _load_hydra_cfg(self, overrides):
         """
         Loads Hydra configuration
         :param overrides: overrides from command line.
@@ -52,6 +91,7 @@ def load_hydra_cfg(self, overrides):
                 cfg_filename="hydra.yaml",
                 strict=False,
                 cli_overrides=overrides,
+                defaults_only=True,
             )
         else:
             hydra_cfg = OmegaConf.from_dotlist(overrides)
@@ -67,40 +107,6 @@ def load_hydra_cfg(self, overrides):
 
         return hydra_cfg
 
-    def load_configuration(self, overrides=[]):
-        assert overrides is None or isinstance(overrides, list)
-
-        overrides = overrides or []
-        hydra_cfg = self.load_hydra_cfg(overrides)
-
-        task_cfg, consumed_defaults = self._create_cfg(
-            cfg_dir=self.cfg_dir,
-            cfg_filename=self.conf_filename,
-            cli_overrides=overrides or [],
-            strict=self.strict_task_cfg,
-        )
-        cfg = OmegaConf.merge(hydra_cfg, task_cfg)
-        for item in [x for x in overrides if not x.startswith("hydra.")]:
-            cfg.hydra.overrides.task.append(item)
-        for item in [x for x in overrides if x.startswith("hydra.")]:
-            cfg.hydra.overrides.hydra.append(item)
-
-        job = OmegaConf.create(dict(name=JobRuntime().get("name")))
-        cfg.hydra.job = OmegaConf.merge(job, cfg.hydra.job)
-        OmegaConf.set_struct(cfg, self.strict_task_cfg)
-        return cfg
-
-    def load_sweep_config(self, master_config, sweep_overrides):
-        # Recreate the config for this sweep instance with the appropriate overrides
-        sweep_config = self.load_configuration(
-            master_config.hydra.overrides.hydra.to_container() + sweep_overrides
-        )
-
-        # Copy old config cache to ensure we get the same resolved values (for things like timestamps etc)
-        OmegaConf.copy_cache(from_config=master_config, to_config=sweep_config)
-
-        return sweep_config
-
     def _load_config_impl(self, is_pkg, filename):
         loaded_cfg = None
         if is_pkg:
@@ -154,7 +160,9 @@ def _exists(is_pkg, filename):
         else:
             return os.path.exists(filename)
 
-    def _create_cfg(self, cfg_dir, cfg_filename, strict, cli_overrides=[]):
+    def _create_cfg(
+        self, cfg_dir, cfg_filename, strict, cli_overrides=[], defaults_only=False
+    ):
         is_pkg = cfg_dir.startswith("pkg://")
         if is_pkg:
             cfg_dir = cfg_dir[len("pkg://") :]
@@ -228,7 +236,8 @@ def _create_cfg(self, cfg_dir, cfg_filename, strict, cli_overrides=[]):
         if strict:
             OmegaConf.set_struct(cfg, True)
         # merge in remaining overrides
-        cfg.merge_with_dotlist(overrides)
+        if not defaults_only:
+            cfg.merge_with_dotlist(overrides)
         # remove config block from resulting cfg.
         del cfg["defaults"]
         return cfg, consumed_defaults
diff --git a/plugins/fairtask/hydra_plugins/fairtask/fairtask_launcher.py b/plugins/fairtask/hydra_plugins/fairtask/fairtask_launcher.py
--- a/plugins/fairtask/hydra_plugins/fairtask/fairtask_launcher.py
+++ b/plugins/fairtask/hydra_plugins/fairtask/fairtask_launcher.py
@@ -28,7 +28,7 @@ def setup(self, config, config_loader, task_function, verbose):
         self.task_function = task_function
         self.verbose = verbose
 
-    def launch_job(self, config, sweep_overrides, job_dir_key, job_num):
+    def launch_job(self, sweep_overrides, job_dir_key, job_num):
         # stdout logging until we get the file logging going, logs will be in slurm job log files
         utils.configure_log(None, self.verbose)
         utils.setup_globals()
@@ -70,9 +70,7 @@ async def run_sweep(self, queue, job_overrides):
                 )
             )
             runs.append(
-                queue(self.launch_job)(
-                    self.config, sweep_override, "hydra.sweep.dir", job_num
-                )
+                queue(self.launch_job)(sweep_override, "hydra.sweep.dir", job_num)
             )
 
         return await gatherl(runs)


Test patch:
diff --git a/hydra/test_utils/launcher_common_tests.py b/hydra/test_utils/launcher_common_tests.py
--- a/hydra/test_utils/launcher_common_tests.py
+++ b/hydra/test_utils/launcher_common_tests.py
@@ -29,12 +29,15 @@ def demo_6_sweep_test_impl(tmpdir, overrides):
     subprocess.check_call(cmd)
 
 
-def demos_sweep_1_job_test_impl(sweep_runner, overrides):
+def demos_sweep_1_job_test_impl(sweep_runner, overrides, strict=False):
     """
     Runs a sweep with one job
     """
     sweep = sweep_runner(
-        conf_dir="demos/6_sweep/conf/", conf_filename="config.yaml", overrides=overrides
+        conf_dir="demos/6_sweep/conf/",
+        conf_filename="config.yaml",
+        overrides=overrides,
+        strict=strict,
     )
     with sweep:
         job_ret = sweep.returns[0]
diff --git a/plugins/fairtask/tests/test_fairtask_launcher.py b/plugins/fairtask/tests/test_fairtask_launcher.py
--- a/plugins/fairtask/tests/test_fairtask_launcher.py
+++ b/plugins/fairtask/tests/test_fairtask_launcher.py
@@ -1,4 +1,5 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import pytest
 
 from hydra.test_utils.launcher_common_tests import (
     demos_sweep_1_job_test_impl,
@@ -56,3 +57,29 @@ def test_not_sweeping_hydra_overrides(sweep_runner):  # noqa: F811
             "hydra.launcher.params.no_workers=true",
         ],
     )
+
+
+def test_fairtask_sweep_1_job_strict(sweep_runner):  # noqa: F811
+    demos_sweep_1_job_test_impl(
+        sweep_runner,
+        strict=True,
+        overrides=[
+            "launcher=fairtask",
+            "hydra.launcher.params.queue=local",
+            "hydra.launcher.params.no_workers=true",
+        ],
+    )
+
+
+def test_fairtask_sweep_1_job_strict_and_bad_key(sweep_runner):  # noqa: F811
+    with pytest.raises(KeyError):
+        demos_sweep_1_job_test_impl(
+            sweep_runner,
+            strict=True,
+            overrides=[
+                "launcher=fairtask",
+                "hydra.launcher.params.queue=local",
+                "hydra.launcher.params.no_workers=true",
+                "hydra.foo=bar",
+            ],
+        )
diff --git a/tests/test_conf_loader.py b/tests/test_conf_loader.py
--- a/tests/test_conf_loader.py
+++ b/tests/test_conf_loader.py
@@ -2,34 +2,34 @@
 import tempfile
 
 import pytest
-from omegaconf import OmegaConf
 
 from hydra import ConfigLoader
 from hydra.errors import MissingConfigException
 from hydra.test_utils.test_utils import chdir_hydra_root
+from omegaconf import OmegaConf
 
 chdir_hydra_root()
 
 
 def test_override_run_dir_without_hydra_cfg():
     config_loader = ConfigLoader(conf_dir=".", conf_filename=None)
-    hydra_cfg = config_loader.load_hydra_cfg(overrides=["hydra.run.dir=abc"])
-    assert hydra_cfg.hydra.run.dir == "abc"
+    cfg = config_loader.load_configuration(overrides=["hydra.run.dir=abc"])
+    assert cfg.hydra.run.dir == "abc"
 
 
 def test_override_run_dir_with_hydra_cfg():
     config_loader = ConfigLoader(
         conf_dir="demos/99_hydra_configuration/workdir/", conf_filename=None
     )
-    hydra_cfg = config_loader.load_hydra_cfg(overrides=["hydra.run.dir=abc"])
-    assert hydra_cfg.hydra.run.dir == "abc"
+    cfg = config_loader.load_configuration(overrides=["hydra.run.dir=abc"])
+    assert cfg.hydra.run.dir == "abc"
 
 
 def test_conf_dir_not_directory():
     with tempfile.NamedTemporaryFile() as f:
         config_loader = ConfigLoader(conf_dir=f.name, conf_filename=None)
         with pytest.raises(IOError):
-            config_loader.load_hydra_cfg(overrides=[])
+            config_loader.load_configuration(overrides=[])
 
 
 def test_load_configuration():