Repository: huggingface/trl

Issue:
Suggestion (DPO): throw an error/warn if `tokenizer.pad_token` is `None`
## Description

trl/trainer/utils.py uses `tokenizer.pad_token` to pad the input sequence. 

This is problematic when `tokenizer.pad_token` is `None`, which results in a rather cryptic `TypeError`:

https://github.com/huggingface/trl/blob/911d3658e23786740c29b8042474bc05ce20b466/trl/trainer/utils.py#L314-L315

https://github.com/huggingface/trl/blob/911d3658e23786740c29b8042474bc05ce20b466/trl/trainer/utils.py#L323

```
...

[/usr/local/lib/python3.10/dist-packages/trl/trainer/utils.py](https://localhost:8080/#) in __call__(self, features)
    320                         raise ValueError(f"Unexpected key in batch '{k}'")
    321 
--> 322                     padded_batch[k] = pad_sequence(to_pad, batch_first=True, padding_value=padding_value)
    323                     # for the prompt, flip back so padding is on left side
    324                     if "prompt" in k:

[/usr/local/lib/python3.10/dist-packages/torch/nn/utils/rnn.py](https://localhost:8080/#) in pad_sequence(sequences, batch_first, padding_value)
    398     # assuming trailing dimensions and type of all the Tensors
    399     # in sequences are same and fetching those from sequences[0]
--> 400     return torch._C._nn.pad_sequence(sequences, batch_first, padding_value)
    401 
    402

TypeError: pad_sequence(): argument 'padding_value' (position 3) must be float, not NoneType
```

Perhaps checking whether `tokenizer.pad_token == None` beforehand and raising an error would help users debug the issue.

Any thoughts on this suggestion? Thanks!


Discussion:
Sounds good to me, feel free to open a PR! :) 

Resolving patch:
diff --git a/trl/trainer/utils.py b/trl/trainer/utils.py
--- a/trl/trainer/utils.py
+++ b/trl/trainer/utils.py
@@ -304,6 +304,12 @@ def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
                     to_pad = [torch.LongTensor(ex[k]) for ex in features]
 
                     if (k.startswith("prompt")) and (k.endswith("input_ids")):
+                        if self.pad_token_id is None:
+                            raise ValueError(
+                                "Padding is enabled, but the tokenizer is not configured with a padding token."
+                                " Explicitly set `tokenizer.pad_token` (e.g. `tokenizer.pad_token = tokenizer.eos_token`)"
+                                " before calling the trainer."
+                            )
                         padding_value = self.pad_token_id
                     elif k.endswith("_attention_mask"):
                         padding_value = 0
@@ -319,6 +325,12 @@ def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
                     else:
                         to_pad = [torch.LongTensor(ex[k]) for ex in features]
                     if k.endswith("_input_ids"):
+                        if self.pad_token_id is None:
+                            raise ValueError(
+                                "Padding is enabled, but the tokenizer is not configured with a padding token."
+                                " Explicitly set `tokenizer.pad_token` (e.g. `tokenizer.pad_token = tokenizer.eos_token`)"
+                                " before calling the trainer."
+                            )
                         padding_value = self.pad_token_id
                     elif k.endswith("_labels"):
                         padding_value = self.label_pad_token_id


Test patch:
diff --git a/tests/test_dpo_trainer.py b/tests/test_dpo_trainer.py
--- a/tests/test_dpo_trainer.py
+++ b/tests/test_dpo_trainer.py
@@ -228,6 +228,41 @@ def test_dpo_trainer_without_providing_ref_model_with_lora(self):
                     if param.sum() != 0:
                         self.assertFalse(torch.equal(param, new_param))
 
+    def test_dpo_trainer_padding_token_is_none(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            training_args = TrainingArguments(
+                output_dir=tmp_dir,
+                per_device_train_batch_size=2,
+                max_steps=3,
+                remove_unused_columns=False,
+                gradient_accumulation_steps=1,
+                learning_rate=9e-1,
+                evaluation_strategy="steps",
+            )
+
+            dummy_dataset = self._init_dummy_dataset()
+
+            tokenizer = AutoTokenizer.from_pretrained(self.model_id)
+            tokenizer.pad_token = None
+
+            with self.assertRaisesRegex(
+                ValueError,
+                expected_regex=r"Padding is enabled, but the tokenizer is not configured with a padding token."
+                r" Explicitly set `tokenizer.pad_token` \(e.g. `tokenizer.pad_token = tokenizer.eos_token`\)"
+                r" before calling the trainer.",
+            ):
+                trainer = DPOTrainer(
+                    model=self.model,
+                    ref_model=None,
+                    beta=0.1,
+                    args=training_args,
+                    tokenizer=tokenizer,
+                    train_dataset=dummy_dataset,
+                    eval_dataset=dummy_dataset,
+                )
+
+                trainer.train()
+
     @require_no_wandb
     def test_dpo_trainer_generate_during_eval_no_wandb(self):
         with tempfile.TemporaryDirectory() as tmp_dir: