File size: 234 Bytes
32f2d5a |
1 2 3 4 5 6 7 8 9 10 11 |
from transformers import GPTNeoXTokenizerFast
class MambaTokenizerFast(GPTNeoXTokenizerFast):
def __init__(
self,
**kwargs,
):
super().__init__(**kwargs)
self.pad_token_id = self.eos_token_id
|