Repository: huggingface/transformers (Python) — Bug Fix

Issue:
Weird Tokenization when Training New Tokenizer from Llama 2 Tokenizer using `train_new_from_iterator`
### System Info

- `transformers` version: 4.35.2
- Platform: Linux-5.4.0-105-generic-x86_64-with-glibc2.31
- Python version: 3.10.13
- Huggingface_hub version: 0.19.4
- Safetensors version: 0.4.1
- Accelerate version: not installed
- Accelerate config: not found
- PyTorch version (GPU?): not installed (NA)
- Tensorflow version (GPU?): not installed (NA)
- Flax version (CPU?/GPU?/TPU?): not installed (NA)
- Jax version: not installed
- JaxLib version: not installed
- Using GPU in script?: No
- Using distributed or parallel set-up in script?: No

### Who can help?

@ArthurZucker 

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [x] My own task or dataset (give details below)

### Reproduction

```python
import os
import argparse
from datasets import load_dataset
from transformers import (
    AutoTokenizer
)

def python_generator():
    # Load local files for code_search_net/python
    # https://huggingface.co/datasets/code_search_net
    dataset = load_dataset("code_search_net", "python")
    dataset = dataset["train"]
    for start_idx in range(0, len(dataset), 1000):
        samples = dataset[start_idx: start_idx + 1000]
        yield samples["whole_func_string"]

def main(args):
    model_paths = [
        "gpt2",
        "meta-llama/Llama-2-70b-hf",
    ]
    access_token = ""
    for model_path in model_paths:
        print(f"\n\n{model_path}")
        save_dir = (
            f"{model_path}-python-52K_vocab"
        )
        os.makedirs(os.path.join(os.getcwd(), "tokenizers"), exist_ok=True)
        save_path = os.path.join(os.getcwd(), "tokenizers", save_dir)

        old_tokenizer = AutoTokenizer.from_pretrained(
            model_path,
            token=access_token
        )
        assert old_tokenizer.is_fast

        if os.path.exists(save_path):
            new_tokenizer = AutoTokenizer.from_pretrained(save_path)
        else:
            new_tokenizer = old_tokenizer.train_new_from_iterator(
                python_generator(),
                vocab_size=52000
            )
            new_tokenizer.save_pretrained(save_path)

        example_1 = '''
        def add_numbers(a, b):
            """Add the two numbers `a` and `b`."""
            return a + b
        '''
        print(f"\n{example_1}")
        old_tokens = old_tokenizer.tokenize(example_1)
        print(f"old: {old_tokens}")
        new_tokens = new_tokenizer.tokenize(example_1)
        print(f"new: {new_tokens}")

        example_2 = """
        class LinearLayer():
            def __init__(self, input_size, output_size):
                self.weight = torch.randn(input_size, output_size)
                self.bias = torch.zeros(output_size)

            def __call__(self, x):
                return x @ self.weights + self.bias
        """
        print(f"\n{example_2}")
        old_tokens = old_tokenizer.tokenize(example_2)
        print(f"old: {old_tokens}")
        new_tokens = new_tokenizer.tokenize(example_2)
        print(f"new: {new_tokens}")
```

### Expected behavior

The function `train_new_from_iterator` works as expected when training a new tokenizer from a gpt2 tokenizer as demonstrated in the [example](https://huggingface.co/learn/nlp-course/chapter6/2), but does not work for training a new tokenizer from a Llama-2 tokenizer.

With the code snippet above, training a tokenizer from gpt2 gives the output:
```
Example 1:
def add_numbers(a, b):
  """Add the two numbers `a` and `b`."""
  return a + b
        
old: ['Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġdef', 'Ġadd', '_', 'n', 'umbers', '(', 'a', ',', 'Ġb', '):', 'Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ"""', 'Add', 'Ġthe', 'Ġtwo', 'Ġnumbers', 'Ġ`', 'a', '`', 'Ġand', 'Ġ`', 'b', '`', '."', '""', 'Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġreturn', 'Ġa', 'Ġ+', 'Ġb', 'Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ']
new: ['ĊĠĠĠĠĠĠĠ', 'Ġdef', 'Ġadd', '_', 'numbers', '(', 'a', ',', 'Ġb', '):', 'ĊĠĠĠĠĠĠĠĠĠĠĠ', 'Ġ"""', 'Add', 'Ġthe', 'Ġtwo', 'Ġnumbers', 'Ġ`', 'a', '`', 'Ġand', 'Ġ`', 'b', '`."""', 'ĊĠĠĠĠĠĠĠĠĠĠĠ', 'Ġreturn', 'Ġa', 'Ġ+', 'Ġb', 'ĊĠĠĠĠĠĠĠĠ']

Example 2:
class LinearLayer():
  def __init__(self, input_size, output_size):
    self.weight = torch.randn(input_size, output_size)
    self.bias = torch.zeros(output_size)

  def __call__(self, x):
    return x @ self.weights + self.bias
        
old: ['Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġclass', 'ĠLinear', 'Layer', '():', 'Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġdef', 'Ġ__', 'init', '__', '(', 'self', ',', 'Ġinput', '_', 'size', ',', 'Ġoutput', '_', 'size', '):', 'Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġself', '.', 'weight', 'Ġ=', 'Ġtorch', '.', 'rand', 'n', '(', 'input', '_', 'size', ',', 'Ġoutput', '_', 'size', ')', 'Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġself', '.', 'b', 'ias', 'Ġ=', 'Ġtorch', '.', 'zer', 'os', '(', 'output', '_', 'size', ')', 'ĊĊ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġdef', 'Ġ__', 'call', '__', '(', 'self', ',', 'Ġx', '):', 'Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġreturn', 'Ġx', 'Ġ@', 'Ġself', '.', 'weights', 'Ġ+', 'Ġself', '.', 'b', 'ias', 'Ċ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ', 'Ġ']
new: ['ĊĠĠĠĠĠĠĠ', 'Ġclass', 'ĠLinear', 'Layer', '():', 'ĊĠĠĠĠĠĠĠĠĠĠĠ', 'Ġdef', 'Ġ__', 'init', '__(', 'self', ',', 'Ġinput', '_', 'size', ',', 'Ġoutput', '_', 'size', '):', 'ĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ', 'Ġself', '.', 'weight', 'Ġ=', 'Ġtorch', '.', 'randn', '(', 'input', '_', 'size', ',', 'Ġoutput', '_', 'size', ')', 'ĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ', 'Ġself', '.', 'bias', 'Ġ=', 'Ġtorch', '.', 'zeros', '(', 'output', '_', 'size', ')', 'ĊĊĠĠĠĠĠĠĠĠĠĠĠ', 'Ġdef', 'Ġ__', 'call', '__(', 'self', ',', 'Ġx', '):', 'ĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ', 'Ġreturn', 'Ġx', 'Ġ@', 'Ġself', '.', 'weights', 'Ġ+', 'Ġself', '.', 'bias', 'ĊĠĠĠĠĠĠĠĠ']
```

However, training Llama-2's tokenizer gives:
```
Example 1:
def add_numbers(a, b):
  """Add the two numbers `a` and `b`."""
  return a + b
        
old: ['▁', '<0x0A>', '▁▁▁▁▁▁▁', '▁def', '▁add', '_', 'numbers', '(', 'a', ',', '▁b', '):', '<0x0A>', '▁▁▁▁▁▁▁▁▁▁▁', '▁"""', 'Add', '▁the', '▁two', '▁numbers', '▁`', 'a', '`', '▁and', '▁`', 'b', '`', '."', '""', '<0x0A>', '▁▁▁▁▁▁▁▁▁▁▁', '▁return', '▁a', '▁+', '▁b', '<0x0A>', '▁▁▁▁▁▁▁▁']
new: ['▁', '\n▁▁▁▁▁▁▁▁def▁', 'add_', 'number', 's(', 'a,▁b', '):\n▁▁▁▁▁▁▁▁▁▁▁▁"""', 'Add▁the▁', 'two▁', 'number', 's▁`', 'a', '`▁and▁`', 'b', '`', '."""', '\n▁▁▁▁▁▁▁▁▁▁▁▁return▁', 'a▁+▁', 'b', '\n▁▁▁▁▁▁▁▁']

Example 2:
class LinearLayer():
  def __init__(self, input_size, output_size):
    self.weight = torch.randn(input_size, output_size)
    self.bias = torch.zeros(output_size)

  def __call__(self, x):
    return x @ self.weights + self.bias
        
old: ['▁', '<0x0A>', '▁▁▁▁▁▁▁', '▁class', '▁Linear', 'Layer', '():', '<0x0A>', '▁▁▁▁▁▁▁▁▁▁▁', '▁def', '▁__', 'init', '__(', 'self', ',', '▁input', '_', 'size', ',', '▁output', '_', 'size', '):', '<0x0A>', '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁', '▁self', '.', 'weight', '▁=', '▁tor', 'ch', '.', 'rand', 'n', '(', 'input', '_', 'size', ',', '▁output', '_', 'size', ')', '<0x0A>', '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁', '▁self', '.', 'b', 'ias', '▁=', '▁tor', 'ch', '.', 'zer', 'os', '(', 'output', '_', 'size', ')', '<0x0A>', '<0x0A>', '▁▁▁▁▁▁▁▁▁▁▁', '▁def', '▁__', 'call', '__(', 'self', ',', '▁x', '):', '<0x0A>', '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁', '▁return', '▁x', '▁@', '▁self', '.', 'we', 'ights', '▁+', '▁self', '.', 'b', 'ias', '<0x0A>', '▁▁▁▁▁▁▁▁']
new: ['▁', '\n▁▁▁▁▁▁▁▁', 'class▁', 'Linear', 'Layer(', '):\n▁▁▁▁▁▁▁▁▁▁▁▁', 'def▁__init__(self,▁', 'input_', 'size,▁', 'output_', 'size', '):\n▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁self.', 'weight▁=▁', 'torch', '.r', 'and', 'n(', 'input_', 'size,▁', 'output_', 'size', ')\n▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁self.', 'bi', 'as▁=▁', 'torch.', 'zeros(', 'output_', 'size', ')\n\n▁▁▁▁▁▁▁▁▁▁▁▁', 'def▁__', 'call__', '(self,▁x', '):\n▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁return▁', 'x▁', '@▁', 'self.', 'weight', 's▁+▁', 'self.', 'bias', '\n▁▁▁▁▁▁▁▁']
```
The underscores `_` should be prepended at the front of new words, but it seems to be inserted at the back of words or in between words. In fact, it seems like the retrained tokenizer is worse than the original tokenizer on the new data.


Discussion:

Resolving patch:
diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py
--- a/src/transformers/convert_slow_tokenizer.py
+++ b/src/transformers/convert_slow_tokenizer.py
@@ -552,15 +552,22 @@ def tokenizer(self, proto):
 
     def normalizer(self, proto):
         precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
+        _normalizers = [
+            normalizers.Strip(left=False, right=True),  # stripping is important
+            normalizers.Replace(Regex(" {2,}"), "▁"),
+        ]
         if not precompiled_charsmap:
-            return normalizers.Sequence([normalizers.Replace(Regex(" {2,}"), " ")])
+            return normalizers.Sequence(_normalizers)
         else:
-            return normalizers.Sequence(
-                [normalizers.Precompiled(precompiled_charsmap), normalizers.Replace(Regex(" {2,}"), " ")]
-            )
+            return normalizers.Sequence([normalizers.Precompiled(precompiled_charsmap)] + _normalizers)
 
     def pre_tokenizer(self, replacement, add_prefix_space):
-        return pre_tokenizers.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
+        prepend_scheme = "always"
+        if hasattr(self.original_tokenizer, "legacy") and not self.original_tokenizer.legacy:
+            prepend_scheme = "first"
+        return pre_tokenizers.Metaspace(
+            replacement=replacement, add_prefix_space=add_prefix_space, prepend_scheme=prepend_scheme
+        )
 
     def post_processor(self):
         return None


Test patch:
diff --git a/tests/models/t5/test_tokenization_t5.py b/tests/models/t5/test_tokenization_t5.py
--- a/tests/models/t5/test_tokenization_t5.py
+++ b/tests/models/t5/test_tokenization_t5.py
@@ -424,6 +424,41 @@ def test_some_edge_cases(self):
         self.assertEqual(tokens, [])
         self.assertEqual(tokens, tokenizer.sp_model.encode("▁", out_type=str))
 
+    def test_fast_slow_edge_cases(self):
+        # We are testing spaces before and spaces after special tokens + space transformations
+        slow_tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=False)
+        fast_tokenizer = T5TokenizerFast.from_pretrained("t5-base", legacy=False, from_slow=True)
+        slow_tokenizer.add_tokens(AddedToken("<new_token_test_>", rstrip=False, lstrip=False, normalized=False))
+        fast_tokenizer.add_tokens(AddedToken("<new_token_test_>", rstrip=False, lstrip=False, normalized=False))
+
+        edge_case = "Hey!<new_token_test_>. How
Hey <new_token_test_>!"
+        EXPECTED_SLOW = ["▁Hey", "!", "<new_token_test_>", ".", "▁How", "
", "He", "y", "<new_token_test_>", "!"]  # fmt: skip
+        with self.subTest(f"slow {edge_case} normalized = False"):
+            self.assertEqual(slow_tokenizer.tokenize(edge_case), EXPECTED_SLOW)
+        with self.subTest(f"Fast {edge_case} normalized = False"):
+            self.assertEqual(fast_tokenizer.tokenize(edge_case), EXPECTED_SLOW)
+
+        hard_case = "Hey! <new_token_test_>. How
   Hey   <new_token_test_>  !     .     "
+        EXPECTED_SLOW = ["▁Hey", "!", "<new_token_test_>", ".", "▁How", "
", "▁Hey", "<new_token_test_>", "▁", "!", "▁", "."]  # fmt: skip
+        with self.subTest(f"slow {edge_case} normalized = False"):
+            self.assertEqual(slow_tokenizer.tokenize(hard_case), EXPECTED_SLOW)
+        with self.subTest(f"fast {edge_case} normalized = False"):
+            self.assertEqual(fast_tokenizer.tokenize(hard_case), EXPECTED_SLOW)
+
+        fast_tokenizer = T5TokenizerFast.from_pretrained("t5-base", legacy=False, from_slow=True)
+        fast_tokenizer.add_tokens(AddedToken("<new_token_test_>", rstrip=False, lstrip=False, normalized=True))
+
+        # `normalized=True` is the default normalization scheme when adding a token. Normalize -> don't strip the space.
+        # the issue now is that our slow tokenizer should NOT strip the space if we want to simulate sentencepiece token addition.
+
+        EXPECTED_FAST = ["▁Hey", "!", "<new_token_test_>", ".", "▁How", "
", "He", "y", "▁", "<new_token_test_>", "!"]  # fmt: skip
+        with self.subTest(f"fast {edge_case} normalized = True"):
+            self.assertEqual(fast_tokenizer.tokenize(edge_case), EXPECTED_FAST)
+
+        EXPECTED_FAST = ['▁Hey', '!', '▁', '<new_token_test_>', '.', '▁How', '
', '▁Hey','▁', '<new_token_test_>', '▁', '!', '▁', '.']  # fmt: skip
+        with self.subTest(f"fast {edge_case} normalized = False"):
+            self.assertEqual(fast_tokenizer.tokenize(hard_case), EXPECTED_FAST)
+
 
 @require_sentencepiece
 @require_tokenizers