Repository: huggingface/transformers

Issue:
Incorrect tokenization for MarianNMT models in example script.
## Environment info
<!-- You can run the command `transformers-cli env` and copy-and-paste its output below.
     Don't forget to fill out the missing fields in that output! -->
     
- `transformers` version: 3.0.2
- Platform: Linux-4.4.0-1110-aws-x86_64-with-Ubuntu-16.04-xenial
- Python version: 3.7.8
- PyTorch version (GPU?): 1.5.1+cu101 (True)
- Tensorflow version (GPU?): not installed (NA)
- Using GPU in script?: yes
- Using distributed or parallel set-up in script?: no

### Who can help
 Marian: @sshleifer 

## Information

Model I am using (Bert, XLNet ...): Marian (Helsinki-NLP/opus-mt-nl-fr)

The problem arises when using:
* [x] the official example scripts: (give details below)
* [ ] my own modified scripts: (give details below)

The tasks I am working on is:
* [ ] an official GLUE/SQUaD task: (give the name)
* [x] my own task or dataset: (give details below)

I'm trying to finetune a MarianMT model (Helsinki-NLP/opus-mt-nl-fr), using the example finetuning script `examples/seq2seq/finetune.py`. Marian models have different sentencepiece models for the encoder and decoder, but it appears the script does not take this into account, as the source line and the target line are tokenized in exactly the same way (in `utils.py`, lines 115-116):

```
        source_inputs = encode_line(self.tokenizer, source_line, self.max_source_length)
        target_inputs = encode_line(self.tokenizer, tgt_line, self.max_target_length)
```

I suggest checking whether the tokenizer is a Marian tokenizer and if so, encoding with the tokenizer's `prepare_translation_batch` method. Doing this improves BLEU score on my validation data by >3 points.

## To reproduce

Steps to reproduce the behavior:

1. Simply run the `examples/seq2seq/finetune.sh` with a MarianModel.



Discussion:

Resolving patch:
diff --git a/examples/seq2seq/finetune.py b/examples/seq2seq/finetune.py
--- a/examples/seq2seq/finetune.py
+++ b/examples/seq2seq/finetune.py
@@ -14,7 +14,7 @@
 from torch.utils.data import DataLoader
 
 from lightning_base import BaseTransformer, add_generic_args, generic_train
-from transformers import MBartTokenizer, T5ForConditionalGeneration, get_linear_schedule_with_warmup
+from transformers import MarianTokenizer, MBartTokenizer, T5ForConditionalGeneration, get_linear_schedule_with_warmup
 
 
 try:
@@ -32,7 +32,7 @@
         ROUGE_KEYS,
         calculate_bleu_score,
         Seq2SeqDataset,
-        MBartDataset,
+        TranslationDataset,
         label_smoothed_nll_loss,
     )
 
@@ -40,7 +40,7 @@
 except ImportError:
     from utils import (
         Seq2SeqDataset,
-        MBartDataset,
+        TranslationDataset,
         assert_all_frozen,
         use_task_specific_params,
         lmap,
@@ -108,8 +108,8 @@ def __init__(self, hparams, **kwargs):
         if self.model.config.decoder_start_token_id is None and isinstance(self.tokenizer, MBartTokenizer):
             self.decoder_start_token_id = self.tokenizer.lang_code_to_id[hparams.tgt_lang]
             self.model.config.decoder_start_token_id = self.decoder_start_token_id
-        if isinstance(self.tokenizer, MBartTokenizer):
-            self.dataset_class = MBartDataset
+        if isinstance(self.tokenizer, MBartTokenizer) or isinstance(self.tokenizer, MarianTokenizer):
+            self.dataset_class = TranslationDataset
         else:
             self.dataset_class = Seq2SeqDataset
 
diff --git a/examples/seq2seq/utils.py b/examples/seq2seq/utils.py
--- a/examples/seq2seq/utils.py
+++ b/examples/seq2seq/utils.py
@@ -146,7 +146,9 @@ def make_sortish_sampler(self, batch_size):
         return SortishSampler(self.src_lens, batch_size)
 
 
-class MBartDataset(Seq2SeqDataset):
+class TranslationDataset(Seq2SeqDataset):
+    """A dataset that calls prepare_translation_batch."""
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         if self.max_source_length != self.max_target_length:
diff --git a/src/transformers/tokenization_marian.py b/src/transformers/tokenization_marian.py
--- a/src/transformers/tokenization_marian.py
+++ b/src/transformers/tokenization_marian.py
@@ -127,10 +127,12 @@ def prepare_translation_batch(
         src_texts: List[str],
         tgt_texts: Optional[List[str]] = None,
         max_length: Optional[int] = None,
+        max_target_length: Optional[int] = None,
         pad_to_max_length: bool = True,
         return_tensors: str = "pt",
         truncation_strategy="only_first",
         padding="longest",
+        **unused,
     ) -> BatchEncoding:
         """Prepare model inputs for translation. For best performance, translate one sentence at a time.
         Arguments:
@@ -162,6 +164,9 @@ def prepare_translation_batch(
         if tgt_texts is None:
             return model_inputs
 
+        if max_target_length is not None:
+            tokenizer_kwargs["max_length"] = max_target_length
+
         self.current_spm = self.spm_target
         decoder_inputs: BatchEncoding = self(tgt_texts, **tokenizer_kwargs)
         for k, v in decoder_inputs.items():


Test patch:
diff --git a/examples/seq2seq/test_seq2seq_examples.py b/examples/seq2seq/test_seq2seq_examples.py
--- a/examples/seq2seq/test_seq2seq_examples.py
+++ b/examples/seq2seq/test_seq2seq_examples.py
@@ -14,14 +14,14 @@
 from torch.utils.data import DataLoader
 
 import lightning_base
-from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, MBartTokenizer
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 from transformers.testing_utils import require_multigpu
 
 from .distillation import distill_main, evaluate_checkpoint
 from .finetune import SummarizationModule, main
 from .pack_dataset import pack_data_dir
 from .run_eval import generate_summaries_or_translations, run_generate
-from .utils import MBartDataset, Seq2SeqDataset, label_smoothed_nll_loss, lmap, load_json
+from .utils import Seq2SeqDataset, TranslationDataset, label_smoothed_nll_loss, lmap, load_json
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -406,8 +406,9 @@ def test_pack_dataset():
     assert orig_paths == new_paths
 
 
-def test_mbart_dataset_truncation():
-    tokenizer = MBartTokenizer.from_pretrained(MBART_TINY)
+@pytest.mark.parametrize(["tok_name"], [pytest.param(MBART_TINY), pytest.param(MARIAN_TINY)])
+def test_mbart_dataset_truncation(tok_name):
+    tokenizer = AutoTokenizer.from_pretrained(tok_name)
     tmp_dir = make_test_data_dir()
     max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
     max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
@@ -416,7 +417,7 @@ def test_mbart_dataset_truncation():
     assert max_len_target > max_src_len  # Truncated
     assert max_len_source > max_src_len
     src_lang, tgt_lang = "ro_RO", "de_DE"  # NOT WHAT IT WAS TRAINED ON
-    train_dataset = MBartDataset(
+    train_dataset = TranslationDataset(
         tokenizer,
         data_dir=tmp_dir,
         type_path="train",
@@ -433,6 +434,8 @@ def test_mbart_dataset_truncation():
         assert batch["input_ids"].shape[1] == max_src_len
         # show that targets are the same len
         assert batch["decoder_input_ids"].shape[1] == max_tgt_len
+        if tok_name == MARIAN_TINY:
+            continue
         # check language codes in correct place
         assert batch["decoder_input_ids"][0, 0].item() == tokenizer.lang_code_to_id[tgt_lang]
         assert batch["decoder_input_ids"][0, -1].item() == tokenizer.eos_token_id