# Buggy code
def set_state(instances: Dict[type, "Singleton"]) -> None:
    Singleton._instances = instances


# Traceback (submitit.core.utils.FailedJobError)
$ python foo.py -m hydra/launcher=submitit_local
[2020-08-10 19:56:14,485][HYDRA] Submitit 'local' sweep output dir : multirun/2020-08-10/19-56-14
[2020-08-10 19:56:14,487][HYDRA] 	#0 :
Traceback (most recent call last):
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/hydra/_internal/utils.py", line 198, in run_and_report
return func()
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/hydra/_internal/utils.py", line 344, in <lambda>
overrides=args.overrides,
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/hydra/_internal/hydra.py", line 132, in multirun
return sweeper.sweep(arguments=task_overrides)
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/hydra/_internal/core_plugins/basic_sweeper.py", line 135, in sweep
results = self.launcher.launch(batch, initial_job_idx=initial_job_idx)
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py", line 149, in launch
return [j.results()[0] for j in jobs]
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py", line 149, in <listcomp>
return [j.results()[0] for j in jobs]
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/submitit/core/core.py", line 291, in results
raise job_exception  # pylint: disable=raising-bad-type
submitit.core.utils.FailedJobError: Job (task=0) failed during processing with trace:
----------------------
Traceback (most recent call last):
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/submitit/core/submission.py", line 47, in process_job
result = delayed.result()
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/submitit/core/utils.py", line 123, in result
self._result = self.function(*self.args, **self.kwargs)
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py", line 80, in __call__
job_subdir_key="hydra.sweep.subdir",
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/hydra/core/utils.py", line 123, in run_job
ret.return_value = task_function(task_cfg)
File "foo.py", line 14, in main
assert cfg.x == "foo"
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/omegaconf/dictconfig.py", line 315, in __getattr__
self._format_and_raise(key=key, value=None, cause=e)
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/omegaconf/base.py", line 101, in _format_and_raise
type_override=type_override,
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/omegaconf/_utils.py", line 669, in format_and_raise
_raise(ex, cause)
File "/private/home/calebh/miniconda3/envs/bench-detectron2/lib/python3.7/site-packages/omegaconf/_utils.py", line 583, in _raise
raise ex  # set end OC_CAUSE=1 for full backtrace
omegaconf.errors.UnsupportedInterpolationType: Unsupported interpolation type my_custom_resolver
full_key: x
reference_type=Optional[Dict[Any, Any]]
object_type=dict

----------------------
You can check full logs with 'job.stderr(0)' and 'job.stdout(0)'or at paths:
- /private/home/calebh/scratch/hydra-bug/multirun/2020-08-10/19-56-14/.submitit/34113/34113_0_log.err
- /private/home/calebh/scratch/hydra-bug/multirun/2020-08-10/19-56-14/.submitit/34113/34113_0_log.out

# Fixed code
def set_state(state: Any) -> None:
    Singleton._instances = state["instances"]
    BaseContainer._resolvers = deepcopy(state["omegaconf_resolvers"])