Repository: keras-team/keras

Issue:
Cannot get result() since the metric has not yet been built
Basically the heading. I am attaching the colab notebook for reproducibility purpose, and have printed out the tensorflow and keras versions for perusal. [Link here](https://colab.research.google.com/drive/1F5Wzq6AqbYTzlBZHXypgiTn9xUVMOhyt#scrollTo=PUKQ6_Vv3nGk)

I think I'm doing everything the standard way. Cannot figure out why this error occurs. Of course this is a simpler code snippet than the one I'm working on which cannot be shared for obvious purposes, but this code demonstrates the issue.

Any clue on how to fix this/circumvent, or which version of keras/tf can i switch to, to avoid this issue?

If it is any help, I saw another [issue](https://github.com/keras-team/keras-nlp/issues/1716) open on keras-nlp, which discusses the same problem, so i'm guessing it's an issue with CompileMetrics module in the trainers folder, which is where the error points anyway.


Discussion:
Are you satisfied with the resolution of your issue?
<a href="https://docs.google.com/forms/d/e/1FAIpQLSdHag0RVFS7UXzZkKcsFCKOcX8raCupKK9RHSlYxp5U8lSJbQ/viewform?entry.492125872=Yes&entry.243948740=https://github.com/keras-team/keras/issues/20188">Yes</a>
<a href="https://docs.google.com/forms/d/e/1FAIpQLSdHag0RVFS7UXzZkKcsFCKOcX8raCupKK9RHSlYxp5U8lSJbQ/viewform?entry.492125872=No&entry.243948740=https://github.com/keras-team/keras/issues/20188">No</a>




Patch:
diff --git a/keras/src/trainers/trainer.py b/keras/src/trainers/trainer.py
index 4c91cf21e600..7bf0ba8b7953 100644
--- a/keras/src/trainers/trainer.py
+++ b/keras/src/trainers/trainer.py
@@ -134,6 +134,7 @@ def compile(
                 wrapped in a `LossScaleOptimizer`, which will dynamically
                 scale the loss to prevent underflow.
         """
+        self._clear_previous_trainer_metrics()
         optimizer = optimizers.get(optimizer)
         self.optimizer = optimizer
         if (
@@ -246,12 +247,23 @@ def run_eagerly(self, value):
 
     @property
     def metrics(self):
-        metrics = [self._loss_tracker] if self.compiled else []
-        metrics.extend(super().metrics)
-        if self.compiled and self._compile_metrics is not None:
-            metrics += [self._compile_metrics]
-        if self.compiled and self._compile_loss is not None:
-            metrics.extend(self._compile_loss.metrics)
+        # Order: loss tracker, individual loss trackers, compiled metrics,
+        # custom metrcis, sublayer metrics.
+        metrics = []
+        if self.compiled:
+            if self._loss_tracker is not None:
+                metrics.append(self._loss_tracker)
+            if self._compile_metrics is not None:
+                metrics.append(self._compile_metrics)
+            if self._compile_loss is not None:
+                metrics.extend(self._compile_loss.metrics)
+        metrics.extend(self._metrics)
+        for layer in self._layers:
+            if isinstance(layer, Trainer):
+                # All Trainer-related metrics in sublayers should be ignored
+                # because a new Trainer has been instantiated.
+                continue
+            metrics.extend(layer.metrics)
         return metrics
 
     @property
@@ -262,6 +274,35 @@ def reset_metrics(self):
         for m in self.metrics:
             m.reset_state()
 
+    def _get_own_metrics(self):
+        metrics = []
+        if hasattr(self, "_loss_tracker"):
+            metrics.append(self._loss_tracker)
+        if (
+            hasattr(self, "_compile_metrics")
+            and self._compile_metrics is not None
+        ):
+            metrics.append(self._compile_metrics)
+        if hasattr(self, "_compile_loss") and self._compile_loss is not None:
+            metrics.extend(self._compile_loss.metrics)
+        if hasattr(self, "_metrics"):
+            metrics.extend(self._metrics)
+        return metrics
+
+    def _clear_previous_trainer_metrics(self):
+        for layer in self._flatten_layers(include_self=False):
+            if not isinstance(layer, Trainer):
+                continue
+            # A sublayer might be a Trainer. In that case, we need to clear
+            # the Trainer-related metrics, as they are not usable when a
+            # new Trainer is instantiated.
+            for m in self._get_own_metrics():
+                layer._tracker.untrack(m)
+            layer._loss_tracker = None
+            layer._compile_metrics = None
+            layer._compile_loss._metrics = []
+            layer._metrics = []
+
     def compute_loss(
         self,
         x=None,


Test patch:
diff --git a/keras/src/trainers/trainer_test.py b/keras/src/trainers/trainer_test.py
index fee4fa3f282b..c590436f1a9d 100644
--- a/keras/src/trainers/trainer_test.py
+++ b/keras/src/trainers/trainer_test.py
@@ -199,8 +199,8 @@ def __init__(self, units):
         # my_metric.
         self.assertEqual(len(model.metrics), 3)
         self.assertEqual(model.metrics[0], model._loss_tracker)
-        self.assertEqual(model.metrics[1], model.my_metric)
-        self.assertEqual(model.metrics[2], model._compile_metrics)
+        self.assertEqual(model.metrics[1], model._compile_metrics)
+        self.assertEqual(model.metrics[2], model.my_metric)
 
         # All metrics should have their weights created
         self.assertEqual(len(model._loss_tracker.variables), 2)
@@ -227,6 +227,32 @@ def __init__(self, units):
         )
         self.assertEqual(len(model_weighted.metrics), 3)
 
+    @pytest.mark.requires_trainable_backend
+    def test_nested_trainer_metrics(self):
+        # https://github.com/keras-team/keras/issues/20188
+        model = ExampleModel(units=3)
+        model.compile(
+            optimizer=optimizers.SGD(),
+            loss=losses.MeanSquaredError(),
+            metrics=[metrics.MeanSquaredError()],
+        )
+        self.assertLen(model.metrics, 2)
+        self.assertEqual(model.metrics[0], model._loss_tracker)
+        self.assertEqual(model.metrics[1], model._compile_metrics)
+
+        inputs = keras.Input((4,))
+        outputs = model(inputs)
+        outputs = layers.Dense(8)(outputs)
+        new_model = models.Model(inputs, outputs)
+        new_model.compile(
+            optimizer=optimizers.SGD(),
+            loss=losses.MeanSquaredError(),
+            metrics=[metrics.MeanSquaredError()],
+        )
+        self.assertLen(new_model.metrics, 2)
+        self.assertEqual(new_model.metrics[0], new_model._loss_tracker)
+        self.assertEqual(new_model.metrics[1], new_model._compile_metrics)
+
     @pytest.mark.skipif(
         backend.backend() != "torch",
         reason="torch backend runs in eager mode for jit_compile='auto'",