mamba-370m / tokenization_mamba_fast.py
mjschock's picture
Upload tokenizer
32f2d5a verified
raw
history blame
No virus
234 Bytes
from transformers import GPTNeoXTokenizerFast
class MambaTokenizerFast(GPTNeoXTokenizerFast):
def __init__(
self,
**kwargs,
):
super().__init__(**kwargs)
self.pad_token_id = self.eos_token_id