tokenizer-arena / tokenizer /tokenizer_patcher.py
eson's picture
add compress rate
814ee6b
raw history blame
No virus
211 Bytes
def patch_tokenizer(tokenizer: "PreTrainedTokenizer") -> None:
if "PreTrainedTokenizerBase" not in str(tokenizer._pad.__func__):
tokenizer._pad = MethodType(PreTrainedTokenizerBase._pad, tokenizer)