Repository: facebookresearch/hydra

Issue:
Task name lost with submitit and fairtask



Discussion:
I'd like to kill off some of my experiments on the cluster by name but can't because all of the names on the cluster are showing up as `submitit` -- that's what this issue is referring to, right? Is there a hack I can add in to make the task name `hydra.name` or something?
Current solution is to go into the `.slurm` directory and kill all the jobs with:

```bash
ls * | awk -F'_' '{print $1}' | uniq | xargs -I{} -n1 scancel {}
```
Actually not. this looks like an issue with submitit or the submitit plugin.
can you open a new issue?

in the mean time, this seems to work correctly with fairtask.

This issue about the job name getting lost on the job side, which results in the log name being UNKNOWN_JOB.log.


Resolving patch:
diff --git a/hydra/plugins/common/utils.py b/hydra/plugins/common/utils.py
--- a/hydra/plugins/common/utils.py
+++ b/hydra/plugins/common/utils.py
@@ -129,6 +129,14 @@ def __call__(cls, *args, **kwargs):
             cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
         return cls._instances[cls]
 
+    @staticmethod
+    def get_state():
+        return Singleton._instances
+
+    @staticmethod
+    def set_state(instances):
+        Singleton._instances = instances
+
 
 @six.add_metaclass(Singleton)
 class JobRuntime:
diff --git a/plugins/fairtask/hydra_plugins/fairtask/fairtask_launcher.py b/plugins/fairtask/hydra_plugins/fairtask/fairtask_launcher.py
--- a/plugins/fairtask/hydra_plugins/fairtask/fairtask_launcher.py
+++ b/plugins/fairtask/hydra_plugins/fairtask/fairtask_launcher.py
@@ -4,11 +4,10 @@
 import logging
 import os
 
-from omegaconf import open_dict
-
 import hydra.plugins.common.utils
 from hydra import utils
 from hydra.plugins import Launcher
+from omegaconf import open_dict
 
 # TODO: initialize logger before importing fairtask until comments at
 # https://github.com/fairinternal/fairtask/pull/23 are addressed
@@ -32,7 +31,9 @@ def setup(self, config, config_loader, task_function, verbose):
         self.task_function = task_function
         self.verbose = verbose
 
-    def launch_job(self, sweep_overrides, job_dir_key, job_num):
+    def launch_job(self, sweep_overrides, job_dir_key, job_num, singleton_state):
+        hydra.plugins.common.utils.Singleton.set_state(singleton_state)
+
         # stdout logging until we get the file logging going, logs will be in slurm job log files
         hydra.plugins.common.utils.configure_log(None, self.verbose)
         hydra.plugins.common.utils.setup_globals()
@@ -81,7 +82,12 @@ async def run_sweep(self, queue, job_overrides):
                 )
             )
             runs.append(
-                queue(self.launch_job)(sweep_override, "hydra.sweep.dir", job_num)
+                queue(self.launch_job)(
+                    sweep_override,
+                    "hydra.sweep.dir",
+                    job_num,
+                    hydra.plugins.common.utils.Singleton.get_state(),
+                )
             )
         return await gatherl(runs)
 
diff --git a/plugins/submitit/hydra_plugins/submitit/submitit_launcher.py b/plugins/submitit/hydra_plugins/submitit/submitit_launcher.py
--- a/plugins/submitit/hydra_plugins/submitit/submitit_launcher.py
+++ b/plugins/submitit/hydra_plugins/submitit/submitit_launcher.py
@@ -30,7 +30,8 @@ def setup(self, config, config_loader, task_function, verbose):
         self.task_function = task_function
         self.verbose = verbose
 
-    def launch_job(self, sweep_overrides, job_dir_key, job_num):
+    def launch_job(self, sweep_overrides, job_dir_key, job_num, singleton_state):
+        hydra.plugins.common.utils.Singleton.set_state(singleton_state)
         hydra.plugins.common.utils.configure_log(None, self.verbose)
         hydra.plugins.common.utils.setup_globals()
         sweep_config = self.config_loader.load_sweep_config(
@@ -103,7 +104,11 @@ def launch(self, job_overrides):
                 )
             )
             job = executor.submit(
-                self.launch_job, sweep_override, "hydra.sweep.dir", job_num
+                self.launch_job,
+                sweep_override,
+                "hydra.sweep.dir",
+                job_num,
+                hydra.plugins.common.utils.Singleton.get_state(),
             )
             jobs.append(job)
 


Test patch:
diff --git a/hydra/test_utils/launcher_common_tests.py b/hydra/test_utils/launcher_common_tests.py
--- a/hydra/test_utils/launcher_common_tests.py
+++ b/hydra/test_utils/launcher_common_tests.py
@@ -44,6 +44,7 @@ def demos_sweep_1_job_test_impl(sweep_runner, overrides, strict=False):
         assert len(job_ret) == 1
         assert job_ret[0].overrides == []
         assert job_ret[0].cfg == dict(optimizer=dict(type="nesterov", lr=0.001))
+        assert job_ret[0].hydra_cfg.hydra.job.name == "experiment"
         verify_dir_outputs(sweep.returns[0][0])
 
 
@@ -69,6 +70,7 @@ def demos_sweep_2_jobs_test_impl(sweep_runner, overrides):
             )
             assert job_ret.overrides == ["a={}".format(i)]
             assert job_ret.cfg == expected_conf
+            assert job_ret.hydra_cfg.hydra.job.name == "experiment"
             verify_dir_outputs(job_ret, job_ret.overrides)