Repository: OpenNMT/OpenNMT-tf

Issue:
Add BLEU evaluation metric
Ideally, this metric should be compatible with [`tf.metrics`](https://www.tensorflow.org/api_docs/python/tf/metrics) for a seamless integration in the training flow. Otherwise, we could rely on the `opennmt.utils.hook.SaveEvaluationPredictionHook` hook for external evaluation.


Discussion:
I'm sure you're aware of it, so what's wrong with this implementation:
https://github.com/tensorflow/tensor2tensor/blob/a55c4cf57e509a0caaf2079b11ca3a46c9e17636/tensor2tensor/utils/bleu_hook.py

?
Yes I saw that. Have to look at it more closely but ideally we would like a scorer that gives the same result as `multi-bleu.perl`.

That's why I think it may be better to have an external evaluation by saving the validation translation (which is already implemented) and run `multi-bleu.perl` on it.

Resolving patch:
diff --git a/bin/main.py b/bin/main.py
--- a/bin/main.py
+++ b/bin/main.py
@@ -10,6 +10,7 @@
 import tensorflow as tf
 
 from opennmt.utils import hooks
+from opennmt.utils.evaluator import external_evaluation_fn
 from opennmt.config import load_model_module, load_config
 
 
@@ -92,7 +93,14 @@ def train(estimator, model, config):
     if not os.path.isdir(save_path):
       os.makedirs(save_path)
     eval_hooks.append(hooks.SaveEvaluationPredictionHook(
-        model, os.path.join(save_path, "predictions.txt")))
+        model,
+        os.path.join(save_path, "predictions.txt"),
+        post_evaluation_fn=external_evaluation_fn(
+            config["train"].get("external_evaluators"),
+            config["data"]["eval_labels_file"],
+            output_dir=estimator.model_dir)))
+  elif config["train"].get("external_evaluators") is not None:
+    tf.logging.warning("External evaluators only work when save_eval_predictions is enabled.")
 
   train_spec = tf.estimator.TrainSpec(
       input_fn=model.input_fn(
diff --git a/opennmt/utils/evaluator.py b/opennmt/utils/evaluator.py
new file mode 100644
--- /dev/null
+++ b/opennmt/utils/evaluator.py
@@ -0,0 +1,121 @@
+"""Evaluation related classes and functions."""
+
+import subprocess
+
+import abc
+import re
+import six
+
+import tensorflow as tf
+
+from tensorflow.python.summary.writer.writer_cache import FileWriterCache as SummaryWriterCache
+
+
+@six.add_metaclass(abc.ABCMeta)
+class ExternalEvaluator(object):
+  """Base class for external evaluators."""
+
+  def __init__(self, labels_file=None, output_dir=None):
+    self._labels_file = labels_file
+    self._summary_writer = None
+
+    if output_dir is not None:
+      self._summary_writer = SummaryWriterCache.get(output_dir)
+
+  def __call__(self, step, predictions_path):
+    """Scores the predictions and logs the result.
+
+    Args:
+      step: The step at which this evaluation occurs.
+      predictions_path: The path to the saved predictions.
+    """
+    score = self.score(self._labels_file, predictions_path)
+    if score is None:
+      return
+    if self._summary_writer is not None:
+      self._summarize_score(step, score)
+    self._log_score(score)
+
+  # Some evaluators may return several scores so let them the ability to
+  # define how to log the score result.
+
+  def _summarize_score(self, step, score):
+    summary = tf.Summary(value=[tf.Summary.Value(
+        tag="external_evaluation/{}".format(self.name()), simple_value=score)])
+    self._summary_writer.add_summary(summary, step)
+
+  def _log_score(self, score):
+    tf.logging.info("%s evaluation score: %f", self.name(), score)
+
+  @abc.abstractproperty
+  def name(self):
+    """Returns the name of this evaluator."""
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def score(self, labels_file, predictions_path):
+    """Scores the predictions against the true output labels."""
+    raise NotImplementedError()
+
+
+class BLEUEvaluator(ExternalEvaluator):
+  """Evaluator calling multi-bleu.perl."""
+
+  def name(self):
+    return "BLEU"
+
+  def score(self, labels_file, predictions_path):
+    try:
+      with open(predictions_path, "r") as predictions_file:
+        bleu_out = subprocess.check_output(
+            ["third_party/multi-bleu.perl", labels_file],
+            stdin=predictions_file,
+            stderr=subprocess.STDOUT)
+        bleu_out = bleu_out.decode("utf-8")
+        bleu_score = re.search(r"BLEU = (.+?),", bleu_out).group(1)
+        return float(bleu_score)
+    except subprocess.CalledProcessError as error:
+      if error.output is not None:
+        msg = error.output.strip()
+        tf.logging.warning(
+            "multi-bleu.perl script returned non-zero exit code: {}".format(msg))
+      return None
+
+
+def external_evaluation_fn(evaluators_name, labels_file, output_dir=None):
+  """Returns a callable to be used in
+  :class:`opennmt.utils.hooks.SaveEvaluationPredictionHook` that calls one or
+  more external evaluators.
+
+  Args:
+    evaluators_name: An evaluator name or a list of evaluators name.
+    labels_file: The true output labels.
+    output_dir: The run directory.
+
+  Returns:
+    A callable or ``None`` if :obj:`evaluators_name` is ``None`` or empty.
+
+  Raises:
+    ValueError: if an evaluator name is invalid.
+  """
+  if evaluators_name is None:
+    return None
+  if not isinstance(evaluators_name, list):
+    evaluators_name = [evaluators_name]
+  if not evaluators_name:
+    return None
+
+  evaluators = []
+  for name in evaluators_name:
+    name = name.lower()
+    if name == "bleu":
+      evaluator = BLEUEvaluator(labels_file=labels_file, output_dir=output_dir)
+    else:
+      raise ValueError("No evaluator associated with the name: {}".format(name))
+    evaluators.append(evaluator)
+
+  def _post_evaluation_fn(step, predictions_path):
+    for evaluator in evaluators:
+      evaluator(step, predictions_path)
+
+  return _post_evaluation_fn
diff --git a/opennmt/utils/hooks.py b/opennmt/utils/hooks.py
--- a/opennmt/utils/hooks.py
+++ b/opennmt/utils/hooks.py
@@ -82,8 +82,8 @@ def __init__(self, model, output_file, post_evaluation_fn=None):
       model: The model for which to save the evaluation predictions.
       output_file: The output filename which will be suffixed by the current
         training step.
-      post_evaluation_fn: (optional) A callable that takes as argument the file
-        with the saved predictions.
+      post_evaluation_fn: (optional) A callable that takes as argument the
+        current step and the file with the saved predictions.
     """
     self._model = model
     self._output_file = output_file
@@ -101,8 +101,8 @@ def before_run(self, run_context):  # pylint: disable=unused-argument
     return tf.train.SessionRunArgs([self._predictions, self._global_step])
 
   def after_run(self, run_context, run_values):  # pylint: disable=unused-argument
-    predictions, step = run_values.results
-    self._output_path = "{}.{}".format(self._output_file, step)
+    predictions, self._current_step = run_values.results
+    self._output_path = "{}.{}".format(self._output_file, self._current_step)
     with open(self._output_path, "a") as output_file:
       for prediction in misc.extract_batches(predictions):
         self._model.print_prediction(prediction, stream=output_file)
@@ -110,4 +110,4 @@ def after_run(self, run_context, run_values):  # pylint: disable=unused-argument
   def end(self, session):
     tf.logging.info("Evaluation predictions saved to %s", self._output_path)
     if self._post_evaluation_fn is not None:
-      self._post_evaluation_fn(self._output_path)
+      self._post_evaluation_fn(self._current_step, self._output_path)


Test patch:
diff --git a/opennmt/tests/evaluator_test.py b/opennmt/tests/evaluator_test.py
new file mode 100644
--- /dev/null
+++ b/opennmt/tests/evaluator_test.py
@@ -0,0 +1,15 @@
+import tensorflow as tf
+
+from opennmt.utils import evaluator
+
+
+class EvaluatorTest(tf.test.TestCase):
+
+  def testBLEUEvaluator(self):
+    bleu_evaluator = evaluator.BLEUEvaluator()
+    score = bleu_evaluator.score("data/toy-ende/tgt-val.txt", "data/toy-ende/tgt-val.txt")
+    self.assertEqual(100.0, score)
+
+
+if __name__ == "__main__":
+  tf.test.main()