Repository: zurutech/ashpy

Issue:
Precision of comparison when doing model selection
See the following log:

```
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
[960] loss: 0.04510442167520523
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
[970] loss: 0.04019254446029663
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
[980] loss: 0.03933567926287651
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
[990] loss: 0.03875984624028206
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
[1000] loss: 0.03433336690068245
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
[1010] loss: 0.041549138724803925
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
[1020] loss: 0.040606431663036346
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
[1030] loss: 0.041963666677474976
AEAccuracy: validation value: 0.49551415 → 0.4955141544342041
```

As you can see, this is a constant metric, that gets updated every time when model selection is performed, because we compare the saved value with the newly computed value, which is identical but with some decimal digit.

We should fix the number of digits we want to take into consideration

Discussion:

Patch:
diff --git a/src/ashpy/metrics/gan.py b/src/ashpy/metrics/gan.py
index 02eb008..691515a 100644
--- a/src/ashpy/metrics/gan.py
+++ b/src/ashpy/metrics/gan.py
@@ -341,7 +341,7 @@ class InceptionScore(Metric):
             :obj:`tuple` of (:py:class:`numpy.ndarray`, :py:class:`numpy.ndarray`): Mean and STD.
 
         """
-        tf.print("Computing inception score...")
+        print("Computing inception score...")
 
         predictions: tf.Tensor = self._inception_model(images)
 
diff --git a/src/ashpy/metrics/metric.py b/src/ashpy/metrics/metric.py
index 41273e4..b0a20b5 100644
--- a/src/ashpy/metrics/metric.py
+++ b/src/ashpy/metrics/metric.py
@@ -88,11 +88,13 @@ class Metric(ABC):
             self.json_read(self.best_model_sel_file)[self.sanitized_name]
         )
         # Model selection is done ONLY if an operator was passed at __init__
-        if self._model_selection_operator and self._model_selection_operator(
-            current_value, previous_value
+        if (
+            self._model_selection_operator
+            and self._model_selection_operator(current_value, previous_value)
+            and not np.isclose(current_value, previous_value)
         ):
-            tf.print(
-                f"{self.sanitized_name}: validation value: {previous_value} → {current_value}"
+            print(
+                f"{self.sanitized_name}: validation value: {previous_value} → {current_value}",
             )
             self.json_write(
                 self.best_model_sel_file,
diff --git a/src/ashpy/trainers/trainer.py b/src/ashpy/trainers/trainer.py
index b08748d..1f2d978 100644
--- a/src/ashpy/trainers/trainer.py
+++ b/src/ashpy/trainers/trainer.py
@@ -76,7 +76,7 @@ class Trainer(ABC):
         # set and validate metrics
         if metrics is None:
             metrics = ()
-        self._metrics = metrics
+        self._metrics: Tuple[Optional[Metric]] = metrics
         self._validate_metrics()
 
         # set and validate callbacks


Test patch:
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index dfa2f26..3159631 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -23,7 +23,8 @@ import shutil
 from pathlib import Path
 
 import pytest
-from ashpy.metrics import ClassifierLoss
+import tensorflow as tf
+from ashpy.metrics import ClassifierLoss, Metric
 
 from tests.utils.fake_training_loop import FakeAdversarialTraining, FakeTraining
 
@@ -143,3 +144,56 @@ def test_metrics_on_restart(fake_training_fn, tmpdir):
     }
     print(t2_values)
     assert t1_values == t2_values
+
+
+# -------------------------------------------------------------------------------------
+
+
+def test_metric_precision(fake_training_fn, tmpdir, capsys):
+    """
+    Test that we correctly handle float precision issue.
+
+    GIVEN a metric with constant value
+    THEN the metric model selection log ("metric.name: validation value:") should be
+        present exactly once in the captured stdout.
+
+    If the `np.close` clause is removed from `Metric.model_selection()` than working with float
+    will make it so that extremely small values's variances due to floating point precision
+    trigger the model selection multiple times unneccessarily.
+
+    """
+
+    class FakeMetric(Metric):
+        """Fake Metric returning Pi as a constant."""
+
+        def __init__(self, name="fake_metric", model_selection_operator=operator.gt):
+            super().__init__(
+                name=name,
+                model_selection_operator=model_selection_operator,
+                metric=tf.metrics.Mean(name=name, dtype=tf.float32),
+            )
+            self.fake_score = (
+                tf.divide(
+                    tf.exp(tf.random.normal((100,))),
+                    (
+                        tf.add(
+                            tf.exp(tf.random.normal((100,))),
+                            tf.exp(tf.random.normal((100,), 10)),
+                        )
+                    ),
+                )
+                / 10000
+            ).numpy()[0]
+            print("FAKE SCORE: ", self.fake_score)
+
+        def update_state(self, context):
+            updater = lambda value: lambda: self._metric.update_state(value)
+            self._distribute_strategy.experimental_run_v2(updater(self.fake_score))
+
+    fake_training: FakeTraining = fake_training_fn(tmpdir)
+    fake_training.metrics = (*fake_training.metrics, FakeMetric())
+    fake_training.epochs = 5
+    fake_training.build_trainer()
+    assert fake_training()
+    out, _ = capsys.readouterr()
+    assert out.count("fake_metric: validation value:") == 1
diff --git a/tests/utils/fake_training_loop.py b/tests/utils/fake_training_loop.py
index f749056..a8dd3bd 100644
--- a/tests/utils/fake_training_loop.py
+++ b/tests/utils/fake_training_loop.py
@@ -63,8 +63,14 @@ class FakeClassifierTraining(FakeTraining):
         measure_performance_freq=10,
     ):
         """Fake Classifier training loop implementation using an autoencoder as a base model."""
+        self.logdir = logdir
+        self.epochs = epochs
         self.measure_performance_freq = measure_performance_freq
 
+        self.optimizer = optimizer
+
+        self.metrics = metrics
+
         # Model
         self.model: tf.keras.Model = conv_autoencoder(
             layer_spec_input_res,
@@ -82,20 +88,23 @@ class FakeClassifierTraining(FakeTraining):
         )
 
         # Loss
-        reconstruction_error = ashpy.losses.ClassifierLoss(
+        self.reconstruction_error = ashpy.losses.ClassifierLoss(
             tf.keras.losses.MeanSquaredError()
         )
 
         # Trainer
+        self.trainer: ClassifierTrainer
+        self.build_trainer()
+
+    def build_trainer(self):
         self.trainer = ClassifierTrainer(
             model=self.model,
-            optimizer=optimizer,
-            loss=reconstruction_error,
-            logdir=str(logdir),
-            epochs=epochs,
-            metrics=metrics,
+            optimizer=self.optimizer,
+            loss=self.reconstruction_error,
+            logdir=str(self.logdir),
+            epochs=self.epochs,
+            metrics=self.metrics,
         )
-        self.metrics = metrics
 
     def __call__(self) -> bool:
         self.trainer(
@@ -134,6 +143,11 @@ class FakeAdversarialTraining(FakeTraining):
         discriminator=None,
     ):
         """Fake training loop implementation."""
+        self.generator_loss = generator_loss
+        self.discriminator_loss = discriminator_loss
+        self.epochs = epochs
+        self.logdir = logdir
+
         self.measure_performance_freq = measure_performance_freq
 
         # test parameters
@@ -163,18 +177,8 @@ class FakeAdversarialTraining(FakeTraining):
         self.discriminator = discriminator
 
         # Trainer
-        self.trainer = AdversarialTrainer(
-            generator=generator,
-            discriminator=discriminator,
-            generator_optimizer=tf.optimizers.Adam(1e-4),
-            discriminator_optimizer=tf.optimizers.Adam(1e-4),
-            generator_loss=generator_loss,
-            discriminator_loss=discriminator_loss,
-            epochs=epochs,
-            metrics=metrics,
-            callbacks=callbacks,
-            logdir=logdir,
-        )
+        self.trainer: AdversarialTrainer
+        self.build_trainer()
 
         self.dataset = fake_adversarial_dataset(
             image_resolution=image_resolution,
@@ -190,3 +194,17 @@ class FakeAdversarialTraining(FakeTraining):
             self.dataset, measure_performance_freq=self.measure_performance_freq
         )
         return True
+
+    def build_trainer(self):
+        self.trainer = AdversarialTrainer(
+            generator=self.generator,
+            discriminator=self.discriminator,
+            generator_optimizer=tf.optimizers.Adam(1e-4),
+            discriminator_optimizer=tf.optimizers.Adam(1e-4),
+            generator_loss=self.generator_loss,
+            discriminator_loss=self.discriminator_loss,
+            epochs=self.epochs,
+            metrics=self.metrics,
+            callbacks=self.callbacks,
+            logdir=self.logdir,
+        )