Repository: OpenNMT/OpenNMT-tf

Issue:
How to support PPL
Can you add the PPL measurement?


Discussion:

Resolving patch:
diff --git a/opennmt/evaluation.py b/opennmt/evaluation.py
--- a/opennmt/evaluation.py
+++ b/opennmt/evaluation.py
@@ -92,7 +92,7 @@ def _eval(next_fn):
 
     self._eval = _eval
 
-    self._metrics_name = {"loss"}
+    self._metrics_name = {"loss", "perplexity"}
     for scorer in self._scorers:
       self._metrics_name.update(scorer.scores_name)
     model_metrics = self._model.get_metrics()
@@ -173,7 +173,7 @@ def should_stop(self):
     if higher_is_better is None:
       # TODO: the condition below is not always true, find a way to set it
       # correctly for Keras metrics.
-      higher_is_better = target_metric != "loss"
+      higher_is_better = target_metric not in ("loss", "perplexity")
     metrics = [values[target_metric] for _, values in self._metrics_history]
     should_stop = early_stop(
         metrics,
@@ -224,7 +224,7 @@ def __call__(self, step):
       raise RuntimeError("No examples were evaluated")
     loss = loss_num / loss_den
 
-    results = dict(loss=loss)
+    results = dict(loss=loss, perplexity=tf.math.exp(loss))
     if metrics:
       for name, metric in six.iteritems(metrics):
         results[name] = metric.result()


Test patch:
diff --git a/opennmt/tests/evaluation_test.py b/opennmt/tests/evaluation_test.py
--- a/opennmt/tests/evaluation_test.py
+++ b/opennmt/tests/evaluation_test.py
@@ -1,3 +1,4 @@
+import math
 import os
 import six
 
@@ -55,6 +56,12 @@ def compute_loss(self, outputs, labels, training=True):
 
 class EvaluationTest(tf.test.TestCase):
 
+  def _assertMetricsEqual(self, metrics, expected):
+    self.assertLen(metrics, len(expected))
+    for name in six.iterkeys(expected):
+      self.assertIn(name, metrics)
+      self.assertAllClose(metrics[name], expected[name])
+
   def testEvaluationMetric(self):
     features_file = os.path.join(self.get_temp_dir(), "features.txt")
     labels_file = os.path.join(self.get_temp_dir(), "labels.txt")
@@ -72,18 +79,18 @@ def testEvaluationMetric(self):
         batch_size=1,
         early_stopping=early_stopping,
         eval_dir=eval_dir)
-    self.assertSetEqual(evaluator.metrics_name, {"loss", "a", "b"})
+    self.assertSetEqual(evaluator.metrics_name, {"loss", "perplexity", "a", "b"})
     metrics_5 = evaluator(5)
-    self.assertDictEqual(metrics_5, {"loss": 1.0, "a": 2, "b": 3})
+    self._assertMetricsEqual(
+        metrics_5, {"loss": 1.0, "perplexity": math.exp(1.0), "a": 2, "b": 3})
     self.assertFalse(evaluator.should_stop())
     metrics_10 = evaluator(10)
-    self.assertDictEqual(metrics_10, {"loss": 4.0, "a": 5, "b": 6})
+    self._assertMetricsEqual(
+        metrics_10, {"loss": 4.0, "perplexity": math.exp(4.0), "a": 5, "b": 6})
     self.assertTrue(evaluator.should_stop())
     self.assertLen(evaluator.metrics_history, 2)
-    self.assertEqual(evaluator.metrics_history[0][0], 5)
-    self.assertEqual(evaluator.metrics_history[0][1], metrics_5)
-    self.assertEqual(evaluator.metrics_history[1][0], 10)
-    self.assertEqual(evaluator.metrics_history[1][1], metrics_10)
+    self._assertMetricsEqual(evaluator.metrics_history[0][1], metrics_5)
+    self._assertMetricsEqual(evaluator.metrics_history[1][1], metrics_10)
 
     # Recreating the evaluator should load the metrics history from the eval directory.
     evaluator = evaluation.Evaluator(
@@ -93,13 +100,12 @@ def testEvaluationMetric(self):
         batch_size=1,
         eval_dir=eval_dir)
     self.assertLen(evaluator.metrics_history, 2)
-    self.assertEqual(evaluator.metrics_history[0][0], 5)
-    self.assertEqual(evaluator.metrics_history[0][1], metrics_5)
-    self.assertEqual(evaluator.metrics_history[1][0], 10)
-    self.assertEqual(evaluator.metrics_history[1][1], metrics_10)
+    self._assertMetricsEqual(evaluator.metrics_history[0][1], metrics_5)
+    self._assertMetricsEqual(evaluator.metrics_history[1][1], metrics_10)
 
     # Evaluating previous steps should clear future steps in the history.
-    self.assertDictEqual(evaluator(7), {"loss": 7.0, "a": 8, "b": 9})
+    self._assertMetricsEqual(
+        evaluator(7), {"loss": 7.0, "perplexity": math.exp(7.0), "a": 8, "b": 9})
     recorded_steps = list(step for step, _ in evaluator.metrics_history)
     self.assertListEqual(recorded_steps, [5, 7])