File size: 264 Bytes
0280a5a |
1 2 3 4 5 6 7 8 9 |
from transformers import PreTrainedTokenizerFast
class CryptGPTTokenizer(PreTrainedTokenizerFast):
@staticmethod
def clean_up_tokenization(out_string):
return out_string.replace(' ', "")
CryptGPTTokenizer.register_for_auto_class("AutoTokenizer")
|