Repository: OpenNMT/OpenNMT-tf
Pull request #731 (2020-10-20 10:03:05)

Ignore evaluation and checkpoint when computing training throughput


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5f72a62d..5a9d0bec 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,7 +19,7 @@ OpenNMT-tf follows [semantic versioning 2.0.0](https://semver.org/). The API cov
 ### Fixes and improvements
 
 * Raise errors (not warnings) when starting the training but end conditions are met
-* Ignore training initialization time when reporting the first throughput values in the training log
+* Ignore non training time when computing training throughput: ignore initialization, evaluation, and checkpoint saving
 * Print OpenNMT-tf version in the training logs
 
 ## [2.12.1](https://github.com/OpenNMT/OpenNMT-tf/releases/tag/v2.12.1) (2020-09-16)
diff --git a/opennmt/training.py b/opennmt/training.py
index 56e60a6a..ba9a4de1 100644
--- a/opennmt/training.py
+++ b/opennmt/training.py
@@ -108,18 +108,24 @@ class Trainer:
             moving_average.update()
 
         step = iterations.numpy()
+        reset_throughput = False
         self._training_stats.update_on_step(step, loss.numpy())
         if step % report_steps == 0:
           self._training_stats.log(self._is_master)
+          reset_throughput = True
         if step == 1 or (save_steps is not None and step % save_steps == 0):
           self._save_checkpoint(step, moving_average=moving_average)
+          reset_throughput = True
         if eval_steps is not None and step % eval_steps == 0:
           early_stop = self._evaluate(evaluator, step, moving_average=moving_average)
+          reset_throughput = True
           if early_stop:
             tf.get_logger().warning("Early stopping conditions are met. Exiting.")
             break
         if step == max_step:
           break
+        if reset_throughput:
+          self._training_stats.reset_throughput()
 
       if step is None:
         raise RuntimeError("No training steps were executed. This usually means the "
@@ -519,7 +525,7 @@ class TrainingStats:
     self._average_loss = (self._average_loss * self._num_updates + loss) / (self._num_updates + 1)
 
     if self._num_updates < self._warmup_steps:
-      self._reset_accumulation()
+      self.reset_throughput()
     self._num_updates += 1
 
   def get_last_summary(self):
@@ -563,7 +569,6 @@ class TrainingStats:
     # Only the master should log the training statistics but we build the
     # summary on all workers since it may reduce distributed values.
     summary = self.get_last_summary()
-    self._reset_accumulation()
 
     if not is_master:
       return
@@ -589,7 +594,7 @@ class TrainingStats:
     tf.summary.scalar("loss", summary["loss"], description="Training loss")
     tf.summary.scalar("optim/learning_rate", summary["learning_rate"], description="Learning rate")
 
-  def _reset_accumulation(self):
+  def reset_throughput(self):
     """Resets the accumulated values since the last log."""
     self._reset_words_counters()
     self._last_logged_step = self._last_step


Test patch:
diff --git a/opennmt/tests/training_test.py b/opennmt/tests/training_test.py
index 0b552644..f358e5f6 100644
--- a/opennmt/tests/training_test.py
+++ b/opennmt/tests/training_test.py
@@ -98,8 +98,8 @@ class TrainingTest(tf.test.TestCase):
     self.assertNotEqual(summary["words_per_sec"]["target"], 0)
 
     stats.log()
+    stats.reset_throughput()
 
-    # log() should reset accumulated values.
     summary = stats.get_last_summary()
     self.assertEqual(summary["steps_per_sec"], 0)
     self.assertEqual(summary["words_per_sec"]["source"], 0)