Repository: facebookresearch/hydra

Issue:
[callbacks] call on_*_end events in reverse order



Discussion:

Patch:
diff --git a/hydra/core/callbacks.py b/hydra/core/callbacks.py
--- a/hydra/core/callbacks.py
+++ b/hydra/core/callbacks.py
@@ -14,8 +14,9 @@ def __init__(self, config: DictConfig) -> None:
         for params in config.hydra.callbacks.values():
             self.callbacks.append(instantiate(params))
 
-    def _notify(self, function_name: str, **kwargs: Any) -> None:
-        for c in self.callbacks:
+    def _notify(self, function_name: str, reverse: bool = False, **kwargs: Any) -> None:
+        callbacks = reversed(self.callbacks) if reverse else self.callbacks
+        for c in callbacks:
             try:
                 getattr(c, function_name)(**kwargs)
             except Exception as e:
@@ -27,13 +28,15 @@ def on_run_start(self, config: DictConfig, **kwargs: Any) -> None:
         self._notify(function_name="on_run_start", config=config, **kwargs)
 
     def on_run_end(self, config: DictConfig, **kwargs: Any) -> None:
-        self._notify(function_name="on_run_end", config=config, **kwargs)
+        self._notify(function_name="on_run_end", config=config, reverse=True, **kwargs)
 
     def on_multirun_start(self, config: DictConfig, **kwargs: Any) -> None:
         self._notify(function_name="on_multirun_start", config=config, **kwargs)
 
     def on_multirun_end(self, config: DictConfig, **kwargs: Any) -> None:
-        self._notify(function_name="on_multirun_end", config=config, **kwargs)
+        self._notify(
+            function_name="on_multirun_end", reverse=True, config=config, **kwargs
+        )
 
     def on_job_start(self, config: DictConfig, **kwargs: Any) -> None:
         self._notify(function_name="on_job_start", config=config, **kwargs)
@@ -42,5 +45,9 @@ def on_job_end(
         self, config: DictConfig, job_return: JobReturn, **kwargs: Any
     ) -> None:
         self._notify(
-            function_name="on_job_end", config=config, job_return=job_return, **kwargs
+            function_name="on_job_end",
+            config=config,
+            job_return=job_return,
+            reverse=True,
+            **kwargs,
         )


Test patch:
diff --git a/tests/test_apps/app_with_callbacks/custom_callback/config.yaml b/tests/test_apps/app_with_callbacks/custom_callback/config.yaml
--- a/tests/test_apps/app_with_callbacks/custom_callback/config.yaml
+++ b/tests/test_apps/app_with_callbacks/custom_callback/config.yaml
@@ -4,5 +4,4 @@ hydra:
   callbacks:
     custom_callback:
       _target_: my_app.CustomCallback
-      param1: value1
-      param2: value2
+      callback_name: custom_callback
diff --git a/tests/test_apps/app_with_callbacks/custom_callback/config_with_two_callbacks.yaml b/tests/test_apps/app_with_callbacks/custom_callback/config_with_two_callbacks.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test_apps/app_with_callbacks/custom_callback/config_with_two_callbacks.yaml
@@ -0,0 +1,8 @@
+hydra:
+  callbacks:
+    callback_1:
+      _target_: my_app.CustomCallback
+      callback_name: callback_1
+    callback_2:
+      _target_: my_app.CustomCallback
+      callback_name: callback_2
diff --git a/tests/test_apps/app_with_callbacks/custom_callback/my_app.py b/tests/test_apps/app_with_callbacks/custom_callback/my_app.py
--- a/tests/test_apps/app_with_callbacks/custom_callback/my_app.py
+++ b/tests/test_apps/app_with_callbacks/custom_callback/my_app.py
@@ -12,13 +12,9 @@
 
 
 class CustomCallback(Callback):
-    def __init__(self, param1, param2):
-        self.param1 = param1
-        self.param2 = param2
-        self.name = self.__class__.__name__
-        log.info(
-            f"Init {self.name} with self.param1={self.param1}, self.param2={self.param2}"
-        )
+    def __init__(self, callback_name):
+        self.name = callback_name
+        log.info(f"Init {self.name}")
 
     def on_job_start(self, config: DictConfig, **kwargs) -> None:
         log.info(f"{self.name} on_job_start")
@@ -41,7 +37,7 @@ def on_multirun_end(self, config: DictConfig, **kwargs) -> None:
 
 @hydra.main(config_path=".", config_name="config")
 def my_app(cfg: DictConfig) -> None:
-    print(OmegaConf.to_yaml(cfg))
+    log.info(OmegaConf.to_yaml(cfg))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py
--- a/tests/test_callbacks.py
+++ b/tests/test_callbacks.py
@@ -15,51 +15,72 @@
 
 
 @mark.parametrize(
-    "app_path,args,expected",
+    "args,expected",
     [
         (
-            "tests/test_apps/app_with_callbacks/custom_callback/my_app.py",
             [],
             dedent(
                 """\
-                [HYDRA] Init CustomCallback with self.param1=value1, self.param2=value2
-                [HYDRA] CustomCallback on_run_start
-                [HYDRA] Init CustomCallback with self.param1=value1, self.param2=value2
-                [JOB] CustomCallback on_job_start
-                foo: bar
+                [HYDRA] Init custom_callback
+                [HYDRA] custom_callback on_run_start
+                [HYDRA] Init custom_callback
+                [JOB] custom_callback on_job_start
+                [JOB] foo: bar
 
-                [JOB] CustomCallback on_job_end
-                [JOB] CustomCallback on_run_end"""
+                [JOB] custom_callback on_job_end
+                [JOB] custom_callback on_run_end"""
             ),
         ),
         (
-            "tests/test_apps/app_with_callbacks/custom_callback/my_app.py",
             [
                 "foo=bar",
                 "-m",
             ],
             dedent(
                 """\
-                [HYDRA] Init CustomCallback with self.param1=value1, self.param2=value2
-                [HYDRA] CustomCallback on_multirun_start
+                [HYDRA] Init custom_callback
+                [HYDRA] custom_callback on_multirun_start
                 [HYDRA] Launching 1 jobs locally
                 [HYDRA] \t#0 : foo=bar
-                [HYDRA] Init CustomCallback with self.param1=value1, self.param2=value2
-                [JOB] CustomCallback on_job_start
-                foo: bar
+                [HYDRA] Init custom_callback
+                [JOB] custom_callback on_job_start
+                [JOB] foo: bar
 
-                [JOB] CustomCallback on_job_end
-                [HYDRA] CustomCallback on_multirun_end"""
+                [JOB] custom_callback on_job_end
+                [HYDRA] custom_callback on_multirun_end"""
+            ),
+        ),
+        (
+            [
+                "--config-name",
+                "config_with_two_callbacks",
+            ],
+            dedent(
+                """\
+                [HYDRA] Init callback_1
+                [HYDRA] Init callback_2
+                [HYDRA] callback_1 on_run_start
+                [HYDRA] callback_2 on_run_start
+                [HYDRA] Init callback_1
+                [HYDRA] Init callback_2
+                [JOB] callback_1 on_job_start
+                [JOB] callback_2 on_job_start
+                [JOB] {}
+
+                [JOB] callback_2 on_job_end
+                [JOB] callback_1 on_job_end
+                [JOB] callback_2 on_run_end
+                [JOB] callback_1 on_run_end"""
             ),
         ),
     ],
 )
 def test_app_with_callbacks(
     tmpdir: Path,
-    app_path: str,
     args: List[str],
     expected: str,
 ) -> None:
+    app_path = "tests/test_apps/app_with_callbacks/custom_callback/my_app.py"
 
     cmd = [
         app_path,