Repository: huggingface/transformers
Pull request: Try to remove `pickle` - `BloomTokenizerFast`

# What does this PR do?

I can see the effect as the value is changed as expected from

```
(Pdb) self.backend_tokenizer.pre_tokenizer
Sequence(pretokenizers=[Split(pattern=Regex(" ?[^(\s|[.,!?…。，、।۔،])]+"), behavior=Isolated, invert=False), ByteLevel(add_prefix_space=False, trim_offsets=True, use_regex=False)])
```
to
```
(Pdb) self.backend_tokenizer.pre_tokenizer
Sequence(pretokenizers=[Split(pattern=Regex(" ?[^(\s|[.,!?…。，、।۔،])]+"), behavior=Isolated, invert=False), ByteLevel(add_prefix_space=True, trim_offsets=True, use_regex=False)])
```



Changed files:
filename: src/transformers/models/bloom/tokenization_bloom_fast.py
patch: @@ -14,7 +14,6 @@
 # limitations under the License.
 """Tokenization classes for Bloom."""
 
-import pickle
 from typing import Optional
 
 from ...tokenization_utils_base import BatchEncoding
@@ -110,16 +109,11 @@ def __init__(
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
             **kwargs,
         )
-        # TODO @ArthurZucker this can only work one way for now, to update later-on. Tests should also properly
-        # check this as they were green before.
-        pre_tok_state = pickle.dumps(self.backend_tokenizer.pre_tokenizer)
-        decoder_state = pickle.dumps(self.backend_tokenizer.decoder)
-
-        if add_prefix_space:
-            pre_tok_state = pre_tok_state.replace(b'"add_prefix_space":false', b'"add_prefix_space": true')
-            decoder_state = decoder_state.replace(b'"add_prefix_space":false', b'"add_prefix_space": true')
-        self.backend_tokenizer.pre_tokenizer = pickle.loads(pre_tok_state)
-        self.backend_tokenizer.decoder = pickle.loads(decoder_state)
+        # This is a `tokenizers.pre_tokenizers.Sequence`
+        for pre_tokenizer in self.backend_tokenizer.pre_tokenizer:
+            if hasattr(pre_tokenizer, "add_prefix_space"):
+                pre_tokenizer.add_prefix_space = add_prefix_space
+        self.backend_tokenizer.decoder.add_prefix_space = add_prefix_space
 
         self.add_prefix_space = add_prefix_space
 

filename: src/transformers/models/cohere/tokenization_cohere_fast.py
patch: @@ -15,7 +15,6 @@
 
 # This file is based on the tokenization_llama_fast.py file in transformers
 
-import pickle
 from typing import Literal, Union
 
 from tokenizers import processors
@@ -147,16 +146,11 @@ def __init__(
         self.grounded_generation_template = kwargs.pop("grounded_generation_template", None)
         self.tool_use_template = kwargs.pop("tool_use_template", None)
 
-        # TODO @ArthurZucker this can only work one way for now, to update later-on. Tests should also properly
-        # check this as they were green before.
-        pre_tok_state = pickle.dumps(self.backend_tokenizer.pre_tokenizer)
-        decoder_state = pickle.dumps(self.backend_tokenizer.decoder)
-
-        if add_prefix_space:
-            pre_tok_state = pre_tok_state.replace(b'"add_prefix_space":false', b'"add_prefix_space": true')
-            decoder_state = decoder_state.replace(b'"add_prefix_space":false', b'"add_prefix_space": true')
-        self.backend_tokenizer.pre_tokenizer = pickle.loads(pre_tok_state)
-        self.backend_tokenizer.decoder = pickle.loads(decoder_state)
+        # This is a `tokenizers.pre_tokenizers.Sequence`
+        for pre_tokenizer in self.backend_tokenizer.pre_tokenizer:
+            if hasattr(pre_tokenizer, "add_prefix_space"):
+                pre_tokenizer.add_prefix_space = add_prefix_space
+        self.backend_tokenizer.decoder.add_prefix_space = add_prefix_space
 
         self.add_prefix_space = add_prefix_space