mamba-1.4b / tokenization_mamba_fast.py
mjschock's picture
Upload tokenizer
d7748da verified
raw
history blame contribute delete
234 Bytes
from transformers import GPTNeoXTokenizerFast
class MambaTokenizerFast(GPTNeoXTokenizerFast):
def __init__(
self,
**kwargs,
):
super().__init__(**kwargs)
self.pad_token_id = self.eos_token_id