Repository: pytorch/torchtune

Issue:
Qwen Tokenizer Excludes Last Assistant EOT Token
Qwen Tokenizer does not include an end-of-turn token (`<im_end>`) for the last assistant message.

```python
from torchtune.data import Message
from torchtune.models.qwen2_5 import qwen2_5_tokenizer

messages = [
    Message(role="user", content="user content", masked=True, eot=True),
    Message(role="assistant", content="assistant content", masked=False, eot=True),
]

tokenizer = qwen2_5_tokenizer(
    path="/tmp/Qwen2.5-1.5B-Instruct/vocab.json", 
    merges_file="/tmp/Qwen2.5-1.5B-Instruct/merges.txt", 
    special_tokens_path="/tmp/Qwen2.5-1.5B-Instruct/tokenizer.json",
    max_seq_len=32768
) 

tokens, mask = tokenizer.tokenize_messages(messages)
```

`print(tokenizer.decode(tokens))`:
```

user
user content


assistant
assistant content

```

What I would expect:
```

user
user content


assistant
assistant content


```

Proposal:
- Make use of message.eot in https://github.com/pytorch/torchtune/blob/059cad9c1c0b684ec095634992468eca18bbd395/torchtune/models/qwen2_5/_tokenizer.py#L232C12-L232C70


Discussion:
Thanks for bringing this up! Actually, when I directly compare to the `transformers` library configs for Qwen2.5 and Qwen2, I see that the end of text is set to `
` and `
` is now only used for a padding token!

So the expected response should actually be something like this:
```

user
user content


assistant
assistant content

```

Here's the code I used to compare:
```python
from transformers import AutoTokenizer
qwen2_5_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-14B-Instruct")
messages = [
    {"role": "user", "content": "user content"},
    {"role": "assistant", "content": "assistant content"},
]
tokenized_text = qwen2_5_tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False)
print(qwen2_5_tokenizer.decode(tokenized_text))
```

I'll put up a PR soon to fix this.
Nice catch, and thank you for a prompt fix! 🙇 

I found [a simple recipe that prints the model input](https://github.com/pytorch/torchtune/commit/5231dc67d24259cb73492b9ec23cc54aafbec738) helpful when working on fine-tuning with custom templates.

The output looks like this (and prints only first batch)
```
$ tune run dev/debug_input --config qwen2_5/0.5B_lora_single_device
Printing first batch of inputs:
==================== Sample 1:

Tokens:

user
Below is an instruction that describes a task. Write a response that appropriately completes the request.

### Instruction:
Give three tips for staying healthy.

### Response:



assistant
.... .


```

If this is of any use - I'll be happy to clean it up (e.g. change output format, make number of batches configurable, etc) and send a PR.



Patch:
diff --git a/torchtune/models/qwen2/_tokenizer.py b/torchtune/models/qwen2/_tokenizer.py
index dd6d038003..c3a9794632 100644
--- a/torchtune/models/qwen2/_tokenizer.py
+++ b/torchtune/models/qwen2/_tokenizer.py
@@ -125,9 +125,9 @@ def __init__(
         *,
         prompt_template: Optional[PromptTemplate] = None,
         errors: str = "replace",
-        unk_token: Optional[str] = ENDOFTEXT,
+        unk_token: Optional[str] = None,
         bos_token: Optional[str] = None,
-        eos_token: str = ENDOFTEXT,
+        eos_token: str = IM_END,
         pad_token: Optional[str] = ENDOFTEXT,
         bpe_cache_size: int = DEFAULT_QWEN2_TOKENIZER_BPE_CACHE_SIZE,
     ):
@@ -160,7 +160,7 @@ def __init__(
         self.pad_id = None if pad_token is None else self.special_tokens[pad_token]
         self.im_start_id = self.special_tokens[IM_START]
         self.im_end_id = self.special_tokens[IM_END]
-        self.stop_tokens = [self.eos_id, self.im_end_id]
+        self.stop_tokens = [self.eos_id, self.pad_id]
 
         # Pattern for special tokens.
         self._pattern_split_special_tokens = re.compile(
@@ -168,7 +168,6 @@ def __init__(
         )
 
         self.max_seq_len = max_seq_len
-
         self.prompt_template = prompt_template
 
     def _bpe_without_cache(self, token):
diff --git a/torchtune/models/qwen2_5/_tokenizer.py b/torchtune/models/qwen2_5/_tokenizer.py
index 2d3eb1d01a..30ea34fc86 100644
--- a/torchtune/models/qwen2_5/_tokenizer.py
+++ b/torchtune/models/qwen2_5/_tokenizer.py
@@ -9,6 +9,7 @@
 from torchtune.models.qwen2._tokenizer import (
     DEFAULT_QWEN2_TOKENIZER_BPE_CACHE_SIZE,
     ENDOFTEXT,
+    IM_END,
     QWEN2_SPECIAL_TOKENS,
     Qwen2Tokenizer,
 )
@@ -92,9 +93,9 @@ def __init__(
         *,
         prompt_template: Optional[PromptTemplate] = None,
         errors: str = "replace",
-        unk_token: Optional[str] = ENDOFTEXT,
+        unk_token: Optional[str] = None,
         bos_token: Optional[str] = None,
-        eos_token: str = ENDOFTEXT,
+        eos_token: str = IM_END,
         pad_token: Optional[str] = ENDOFTEXT,
         bpe_cache_size: int = DEFAULT_QWEN2_TOKENIZER_BPE_CACHE_SIZE,
     ):


Test patch:
diff --git a/tests/torchtune/models/qwen2/test_qwen2_tokenizer.py b/tests/torchtune/models/qwen2/test_qwen2_tokenizer.py
index 1d44a1149d..fd9ecec8b1 100644
--- a/tests/torchtune/models/qwen2/test_qwen2_tokenizer.py
+++ b/tests/torchtune/models/qwen2/test_qwen2_tokenizer.py
@@ -59,7 +59,7 @@ def test_tokenize_messages(self, messages):
                            1506, 416, 114, 128, 1429, 182, 253, 82, 120, 163, 330, 105, 262, 13, 223, 1791, 155, 1551,
                            171, 1951, 628, 296, 64, 237, 886, 1390, 130, 883, 1678, 447, 306, 279, 113, 11, 215, 785,
                            215, 1951, 628, 378, 101, 66, 72, 593, 98, 984, 208, 1580, 167, 510, 737, 318, 1278, 13,
-                           2000] # noqa
+                           2002] # noqa
         # fmt: on
 
         expected_mask = [True] * 67 + [False] * 121
@@ -69,8 +69,9 @@ def test_tokenize_messages(self, messages):
         formatted_messages = tokenizer.decode(tokens)
         expected_formatted_messages = (
             f"
user\n{messages[0].text_content}
\n"
-            f"
assistant\n{messages[1].text_content}
"
+            f"
assistant\n{messages[1].text_content}
"
         )
+
         assert expected_formatted_messages == formatted_messages
 
     def test_tokenize_messages_gt_max_seq_len(self, messages):
diff --git a/tests/torchtune/models/qwen2_5/test_tokenizer.py b/tests/torchtune/models/qwen2_5/test_tokenizer.py
index 8649d6d45e..ade07778be 100644
--- a/tests/torchtune/models/qwen2_5/test_tokenizer.py
+++ b/tests/torchtune/models/qwen2_5/test_tokenizer.py
@@ -29,7 +29,7 @@ def test_tokenize_messages(self):
         expected_tokens = [
             151644, 82, 88, 479, 94, 56, 119, 230, 98, 374, 494, 1318, 249, 13, 151645, 94, 151644, 273, 105, 94,
             38, 229, 362, 98, 1695, 310, 1305, 165, 128, 432, 43, 44, 82, 13, 151645, 94, 151644, 397, 251, 249, 94,
-            151643,
+            151645,
         ] # noqa
         # fmt: on
 
@@ -39,7 +39,7 @@ def test_tokenize_messages(self):
             "
user\n"
             "Give me a short introduction to LLMs.
\n"
             "
assistant\n"
-            "
"
+            "
"
         )
         _test_tokenize_messages(
             tokenizer,
@@ -62,7 +62,7 @@ def test_tool_call(self):
             151644, 82, 88, 479, 94, 64, 151645, 94, 151644, 273, 105, 94, 65, 151645, 94, 151644, 397, 251, 249,
             94, 151657, 94, 83, 269, 107, 330, 94, 151658, 151645, 94, 151644, 273, 105, 94, 27, 83, 1364,
             62, 237, 79, 102, 182, 29, 94, 83, 269, 706, 102, 182, 94, 1932, 83, 1364, 62, 237, 79, 102,
-            182, 29, 151645, 94, 151644, 397, 251, 249, 94, 151643,
+            182, 29, 151645, 94, 151644, 397, 251, 249, 94, 151645,
         ] # noqa
         # fmt: on
 
@@ -80,7 +80,7 @@ def test_tool_call(self):
             "test response\n"
             "</tool_response>
\n"
             "
assistant\n"
-            "
"
+            "
"
         )
         _test_tokenize_messages(
             tokenizer,