Repository: facebookincubator/submitit

Issue:
Cannot override parameters with their generic/auto name
`local_cpus_per_task` does not work while `slurm_cpus_per_task` does.
@gwenzek any reason for this? Can we make it work too?


Discussion:

Patch:
diff --git a/submitit/auto/auto.py b/submitit/auto/auto.py
index ad502d5..e86daf7 100644
--- a/submitit/auto/auto.py
+++ b/submitit/auto/auto.py
@@ -163,7 +163,7 @@ class AutoExecutor(Executor):
                 continue
 
             valid = executors[ex]._valid_parameters()
-            if arg not in valid:
+            if arg not in valid | generics:
                 invalid.append(
                     f"Unknown argument '{arg}' for executor '{ex}' in parameter '{ex}_{arg}'."
                     + " Valid arguments: "
@@ -174,12 +174,20 @@ class AutoExecutor(Executor):
             invalid.append(f"Known executors: {', '.join(executors.keys())}")
             raise NameError("\n".join(invalid))
 
+        # add cluster specific generic overrides
+        kwargs.update(
+            **{
+                arg: kwargs.pop(f"{ex}_{arg}")
+                for ex, arg in specific
+                if ex == self.cluster and arg in generics
+            }
+        )
         parameters = self._executor._convert_parameters({k: kwargs[k] for k in kwargs if k in generics})
         # update parameters in the core executor
         for (ex, arg) in specific:
-            if ex != self.cluster:
-                continue
-            parameters[arg] = kwargs[f"{ex}_{arg}"]
+            # update cluster specific non-generic arguments
+            if arg not in generics and ex == self.cluster:
+                parameters[arg] = kwargs[f"{ex}_{arg}"]
 
         self._executor._internal_update_parameters(**parameters)
 


Test patch:
diff --git a/submitit/auto/test_auto.py b/submitit/auto/test_auto.py
index 79733da..defb232 100644
--- a/submitit/auto/test_auto.py
+++ b/submitit/auto/test_auto.py
@@ -40,6 +40,7 @@ def test_local_executor() -> None:
     with test_slurm.mocked_slurm():
         executor = auto.AutoExecutor(folder=".", cluster="local")
     assert executor.cluster == "local"
+    executor.update_parameters(local_cpus_per_task=2)
 
 
 def test_executor_argument() -> None:
@@ -79,10 +80,12 @@ def test_overriden_arguments() -> None:
     with test_slurm.mocked_slurm():
         slurm_ex = auto.AutoExecutor(folder=".", cluster="slurm")
 
-    slurm_ex.update_parameters(timeout_min=60, slurm_time=120)
+    slurm_ex.update_parameters(
+        timeout_min=60, slurm_timeout_min=120, tasks_per_node=2, slurm_ntasks_per_node=3
+    )
     slurm_params = slurm_ex._executor.parameters
     # slurm use time
-    assert slurm_params == {"time": 120}
+    assert slurm_params == {"time": 120, "ntasks_per_node": 3}
 
     # others use timeout_min
     local_ex = auto.AutoExecutor(folder=".", cluster="local")