Repository: dmlc/gluon-nlp

Issue:
training scripts should not evaluate test set
Currently, the MT training scripts do training (with dev set) and evaluation (on the test set) altogether.
Ideally, the training script should not touch test set because reporting numbers on the test set (even if it's not used for model selection) is already leaking test set information.

For example, in [`fairseq`](https://github.com/pytorch/fairseq), train/validation (train.py) and evaluate (generate.py) are in two separate files.


Discussion:
@szha I would like to take this up. Can you give more context about this?
@avinashsai it's about updating the MT training scripts to only evaluate on the dev set to avoid leaking test set information until the users explicitly runs evaluation on the test set

Resolving patch:
diff --git a/scripts/machine_translation/train_gnmt.py b/scripts/machine_translation/train_gnmt.py
--- a/scripts/machine_translation/train_gnmt.py
+++ b/scripts/machine_translation/train_gnmt.py
@@ -100,6 +100,8 @@
                     help='directory path to save the final model and training log')
 parser.add_argument('--gpu', type=int, default=None,
                     help='id of the gpu to use. Set it to empty means to use cpu.')
+parser.add_argument('--validate_on_test_data', type=bool, default=False,
+                    help='To perform validation on test data')
 args = parser.parse_args()
 print(args)
 logging_config(args.save_dir)
@@ -246,16 +248,18 @@ def train():
         valid_bleu_score, _, _, _, _ = compute_bleu([val_tgt_sentences], valid_translation_out)
         logging.info('[Epoch {}] valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'
                      .format(epoch_id, valid_loss, np.exp(valid_loss), valid_bleu_score * 100))
-        test_loss, test_translation_out = evaluate(test_data_loader)
-        test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out)
-        logging.info('[Epoch {}] test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
-                     .format(epoch_id, test_loss, np.exp(test_loss), test_bleu_score * 100))
         dataprocessor.write_sentences(valid_translation_out,
                                       os.path.join(args.save_dir,
                                                    'epoch{:d}_valid_out.txt').format(epoch_id))
-        dataprocessor.write_sentences(test_translation_out,
-                                      os.path.join(args.save_dir,
-                                                   'epoch{:d}_test_out.txt').format(epoch_id))
+        if args.validate_on_test_data:
+            test_loss, test_translation_out = evaluate(test_data_loader)
+            test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out)
+            logging.info('[Epoch {}] test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
+                         .format(epoch_id, test_loss, np.exp(test_loss), test_bleu_score * 100))
+
+            dataprocessor.write_sentences(test_translation_out,
+                                          os.path.join(args.save_dir,
+                                                       'epoch{:d}_test_out.txt').format(epoch_id))
         if valid_bleu_score > best_valid_bleu:
             best_valid_bleu = valid_bleu_score
             save_path = os.path.join(args.save_dir, 'valid_best.params')
diff --git a/scripts/machine_translation/train_transformer.py b/scripts/machine_translation/train_transformer.py
--- a/scripts/machine_translation/train_transformer.py
+++ b/scripts/machine_translation/train_transformer.py
@@ -350,19 +350,9 @@ def train():
                                                     bpe=bpe)
         logging.info('[Epoch {}] valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'
                      .format(epoch_id, valid_loss, np.exp(valid_loss), valid_bleu_score * 100))
-        test_loss, test_translation_out = evaluate(test_data_loader, ctx[0])
-        test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out,
-                                                   tokenized=tokenized, tokenizer=args.bleu,
-                                                   split_compound_word=split_compound_word,
-                                                   bpe=bpe)
-        logging.info('[Epoch {}] test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
-                     .format(epoch_id, test_loss, np.exp(test_loss), test_bleu_score * 100))
         dataprocessor.write_sentences(valid_translation_out,
                                       os.path.join(args.save_dir,
                                                    'epoch{:d}_valid_out.txt').format(epoch_id))
-        dataprocessor.write_sentences(test_translation_out,
-                                      os.path.join(args.save_dir,
-                                                   'epoch{:d}_test_out.txt').format(epoch_id))
         if valid_bleu_score > best_valid_bleu:
             best_valid_bleu = valid_bleu_score
             save_path = os.path.join(args.save_dir, 'valid_best.params')


Test patch: