Repository: huggingface/transformers (Python)
Date: 2024-02-27 06:22:32+00:00
Task type: Bug Fix

Issue:
`skip_special_tokens` for `Wav2Vec2CTCTokenizer` does not work expectedly.
### System Info

- `transformers` version: 4.37.2
- Platform: Linux-5.15.0-1042-nvidia-x86_64-with-glibc2.35
- Python version: 3.10.13
- Huggingface_hub version: 0.20.1
- Safetensors version: 0.4.2
- Accelerate version: 0.26.1
- Accelerate config:    not found
- PyTorch version (GPU?): 2.1.2 (True)
- Tensorflow version (GPU?): not installed (NA)
- Flax version (CPU?/GPU?/TPU?): not installed (NA)
- Jax version: not installed
- JaxLib version: not installed
- Using GPU in script?: Yes
- Using distributed or parallel set-up in script?: DDP

### Who can help?

@sanchit-gandhi @ArthurZucker 

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [X] My own task or dataset (give details below)

### Reproduction

```python
processor = Wav2Vec2Processor.from_pretrained(model_args.model_name_or_path)
model = Wav2Vec2ConformerForCTC.from_pretrained(
    model_args.model_name_or_path, ctc_loss_reduction="mean", pad_token_id=processor.tokenizer.pad_token_id
)

metric_wer = evaluate.load("wer")
metric_cer = evaluate.load("cer")

def _compute_metrics(pred):
    pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id

    # ["
": 1, "
": 2, "<unk>": 3, "<pad>": 0]
    preds = processor.batch_decode(pred.predictions, skip_special_tokens=True)
    labels = processor.batch_decode(pred.label_ids, skip_special_tokens=True, group_tokens=False)

    preds = [postprocess(text) if len(text) != 0 else "-" for text in preds]
    labels = [postprocess(sentence) for sentence in labels]

    preds = [re.sub(r"\s+", "", text) for text in preds]
    labels = [re.sub(r"\s+", "", text) for text in labels]

    wer = 100 * metric_wer.compute(predictions=preds, references=labels)
    cer = 100 * metric_cer.compute(predictions=preds, references=labels)

    return {"wer": wer, "cer": cer}

def _preprocess_logits_for_metrics(logits, labels=None):
    return torch.argmax(logits, dim=-1)

trainer = Trainer(
    args=training_args,
    model=model,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    data_collator=data_collator,
    tokenizer=processor,
    compute_metrics=_compute_metrics,
    preprocess_logits_for_metrics=_preprocess_logits_for_metrics,
)
```

### Expected behavior

I want to train ASR model and this issue came out when I evaluate during training.
`preds = processor.batch_decode(pred.predictions, skip_special_tokens=True)`'s results should remove all special tokens (`
`, `<\s>`, `<unk>`, `<pad>`). However, in my experiment, `<unk>` is not removed.

So, I look at the code for `Wav2Vec2CTCTokenizer._decode` (transformers / models / wav2vec2 / tokenization_wav2vec2.py).
```python
 def _decode(
        self,
        token_ids: List[int],
        skip_special_tokens: bool = False,
        clean_up_tokenization_spaces: bool = None,
        group_tokens: bool = True,
        spaces_between_special_tokens: bool = False,
        output_word_offsets: Optional[bool] = False,
        output_char_offsets: Optional[bool] = False,
    ) -> str:
        """
        special _decode function is needed for Wav2Vec2Tokenizer because added tokens should be treated exactly the
        same as tokens of the base vocabulary and therefore the function `convert_tokens_to_string` has to be called on
        the whole token list and not individually on added tokens
        """
        filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)

        result = []
        for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                continue
            result.append(token)
```
This code removes the special tokens in the `filtered_tokens` by `self.all_special_ids`, but when I print the `filtered_tokens`, the outcome looks like `['|', '|', 'token1', 'token2', 'token3', '|', '|', 'token4', '|', '|', '|', 'token5', 'token6', '|', '|', '<unk>', '<unk>', '<unk>', '<unk>', '<unk>']`. 

Since `self.all_special_ids`'s elements are integers for special tokens, `if skip_special_tokens and token in self.all_special_ids:` statement does not work expectedly.

Shouldn’t it be `if skip_special_tokens and token in self.all_special_tokens:`?


Issue discussion:
it could / should but should also be left to the super class IMO! 
Would you like to open a PR for a fix? I don't think that this is intended behaviour

Solution patch:
diff --git a/src/transformers/models/wav2vec2/tokenization_wav2vec2.py b/src/transformers/models/wav2vec2/tokenization_wav2vec2.py
--- a/src/transformers/models/wav2vec2/tokenization_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/tokenization_wav2vec2.py
@@ -125,7 +125,6 @@ class Wav2Vec2CTCTokenizerOutput(ModelOutput):
 
 
 class Wav2Vec2CTCTokenizer(PreTrainedTokenizer):
-
     """
     Constructs a Wav2Vec2CTC tokenizer.
 
@@ -434,7 +433,9 @@ def _decode(
 
         result = []
         for token in filtered_tokens:
-            if skip_special_tokens and token in self.all_special_ids:
+            if skip_special_tokens and (
+                token in self.all_special_ids or (token != self.pad_token and token in self.all_special_tokens)
+            ):
                 continue
             result.append(token)
 
@@ -895,7 +896,9 @@ def _decode(
 
         result = []
         for token in filtered_tokens:
-            if skip_special_tokens and token in self.all_special_ids:
+            if skip_special_tokens and (
+                token in self.all_special_ids or (token != self.pad_token and token in self.all_special_tokens)
+            ):
                 continue
             result.append(token)
 


Test patch:
diff --git a/tests/models/wav2vec2/test_tokenization_wav2vec2.py b/tests/models/wav2vec2/test_tokenization_wav2vec2.py
--- a/tests/models/wav2vec2/test_tokenization_wav2vec2.py
+++ b/tests/models/wav2vec2/test_tokenization_wav2vec2.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Tests for the Wav2Vec2 tokenizer."""
+
 import inspect
 import json
 import os
@@ -145,8 +146,10 @@ def test_tokenizer_decode_added_tokens(self):
             [24, 22, 5, tokenizer.word_delimiter_token_id, 24, 22, 5, 77, tokenizer.pad_token_id, 34, 34],
         ]
         batch_tokens = tokenizer.batch_decode(sample_ids)
+        batch_tokens_2 = tokenizer.batch_decode(sample_ids, skip_special_tokens=True)
 
         self.assertEqual(batch_tokens, ["HELLO<unk>!?!?$$$", "BYE BYE<unk>$$$"])
+        self.assertEqual(batch_tokens_2, ["HELO!?!?", "BYE BYE"])
 
     def test_call(self):
         # Tests that all call wrap to encode_plus and batch_encode_plus
@@ -452,18 +455,20 @@ def test_tokenizer_decode_special(self):
 
     def test_tokenizer_decode_added_tokens(self):
         tokenizer = self.tokenizer_class.from_pretrained("facebook/wav2vec2-base-960h")
-        tokenizer.add_tokens(["!", "?"])
+        tokenizer.add_tokens(["!", "?", "<new_tokens>"])
         tokenizer.add_special_tokens({"cls_token": "$$$"})
 
         # fmt: off
         sample_ids = [
-            [11, 5, 15, tokenizer.pad_token_id, 15, 8, 98, 32, 32, 33, tokenizer.word_delimiter_token_id, 32, 32, 33, 34, 34],
-            [24, 22, 5, tokenizer.word_delimiter_token_id, 24, 22, 5, 77, tokenizer.pad_token_id, 34, 34],
+            [11, 5, 15, tokenizer.pad_token_id, 15, 8, 98, 32, 32, 33, tokenizer.word_delimiter_token_id, 32, 32, 33, 34, 34, 35, 35],
+            [24, 22, 5, tokenizer.word_delimiter_token_id, 24, 22, 5, 77, tokenizer.pad_token_id, 34, 34, 35, 35],
         ]
         # fmt: on
         batch_tokens = tokenizer.batch_decode(sample_ids)
+        batch_tokens_2 = tokenizer.batch_decode(sample_ids, skip_special_tokens=True)
 
-        self.assertEqual(batch_tokens, ["HELLO<unk>!?!?$$$", "BYE BYE<unk>$$$"])
+        self.assertEqual(batch_tokens, ["HELLO<unk>!?!?<new_tokens>$$$", "BYE BYE<unk><new_tokens>$$$"])
+        self.assertEqual(batch_tokens_2, ["HELO!?!?<new_tokens>", "BYE BYE<new_tokens>"])
 
     def test_special_characters_in_vocab(self):
         sent = "ʈʰ æ æ̃ ˧ kʰ"