Repository: facebookresearch/ParlAI

Issue:
[tensorboard] Resuming model logging causes bad resume in tensorboard
Nothing crashes, but if you train a model a while, and then resume training, the x-axis resets. This should be avoided by giving an explicit global step already.

Here's an example:
![Screen Shot 2019-10-28 at 12 37 32 PM](https://user-images.githubusercontent.com/31896/67697940-c141f980-f97f-11e9-8e06-206251875345.png)


Steps to reproduce:
1. Train a transformer/generator for a little bit. Make sure it has `-veps 0.1 -eps 5` and will be training for a good bit.
2. Wait for it to make a few thousand steps, then Ctrl-C.
3. Run the same training script again.

Solution:
The tensorboard logging code should use an the explicit examples seen parameter to save the log.


Discussion:

Resolving patch:
diff --git a/parlai/core/logs.py b/parlai/core/logs.py
--- a/parlai/core/logs.py
+++ b/parlai/core/logs.py
@@ -70,3 +70,6 @@ def log_metrics(self, setting, step, report):
                 self.writer.add_scalar(f'{setting}/{k}', v, global_step=step)
             else:
                 print(f'k {k} v {v} is not a number')
+
+    def flush(self):
+        self.writer.flush()
diff --git a/parlai/scripts/train_model.py b/parlai/scripts/train_model.py
--- a/parlai/scripts/train_model.py
+++ b/parlai/scripts/train_model.py
@@ -316,15 +316,6 @@ def run_eval(valid_worlds, opt, datatype, max_exs=-1, write_log=False):
     return report
 
 
-def _save_best_valid(model_file, best_valid):
-    """
-    Save the best validation score to disk.
-    """
-    f = open(model_file + '.best_valid', 'w')
-    f.write(str(best_valid))
-    f.close()
-
-
 class TrainLoop:
     """
     TrainLoop contains the core training loop logic.
@@ -407,11 +398,7 @@ def __init__(self, opt):
         self.valid_optim = 1 if opt['validation_metric_mode'] == 'max' else -1
         self.valid_reports = []
         self.best_valid = None
-        if opt.get('model_file') and os.path.isfile(opt['model_file'] + '.best_valid'):
-            with open(opt['model_file'] + ".best_valid", 'r') as f:
-                x = f.readline()
-                self.best_valid = float(x)
-                f.close()
+
         self.impatience = 0
         self.saved = False
         self.valid_worlds = None
@@ -426,10 +413,22 @@ def __init__(self, opt):
             # training stats, etc
             with open(opt['model_file'] + trainstats_suffix) as ts:
                 obj = json.load(ts)
+                self.parleys = obj.get('parleys', 0)
                 self._preempted_epochs = obj.get('total_epochs', 0)
                 self.train_time.total = obj.get('train_time', 0)
                 self.impatience = obj.get('impatience', 0)
                 self.valid_reports = obj.get('valid_reports', [])
+                if 'best_valid' in obj:
+                    self.best_valid = obj['best_valid']
+                else:
+                    # old method
+                    if opt.get('model_file') and os.path.isfile(
+                        opt['model_file'] + '.best_valid'
+                    ):
+                        with open(opt['model_file'] + ".best_valid", 'r') as f:
+                            x = f.readline()
+                            self.best_valid = float(x)
+                            f.close()
 
         if opt['tensorboard_log'] and is_primary_worker():
             self.tb_logger = TensorboardLogger(opt)
@@ -465,6 +464,7 @@ def _save_train_stats(self, suffix=None):
         with open(fn, 'w') as f:
             json.dump(
                 {
+                    'parleys': self.parleys,
                     'train_time': self.train_time.time(),
                     'total_epochs': (
                         self._preempted_epochs
@@ -472,6 +472,7 @@ def _save_train_stats(self, suffix=None):
                     ),
                     'impatience': self.impatience,
                     'valid_reports': self.valid_reports,
+                    'best_valid': self.best_valid,
                 },
                 f,
             )
@@ -499,6 +500,8 @@ def validate(self):
         # logging
         if opt['tensorboard_log'] and is_primary_worker():
             self.tb_logger.log_metrics('valid', self.parleys, valid_report)
+            # flush on a validation
+            self.tb_logger.flush()
         # saving
         if (
             opt.get('model_file')
@@ -542,10 +545,6 @@ def validate(self):
             if opt.get('model_file') and is_primary_worker():
                 print("[ saving best valid model: " + opt['model_file'] + " ]")
                 self.save_model()
-                print(
-                    "[ saving best valid metric: " + opt['model_file'] + ".best_valid ]"
-                )
-                _save_best_valid(opt['model_file'], self.best_valid)
                 self.saved = True
             if (
                 opt['validation_metric'] == 'accuracy'


Test patch: