from transformers import GPTNeoXTokenizerFast | |
class MambaTokenizerFast(GPTNeoXTokenizerFast): | |
def __init__( | |
self, | |
**kwargs, | |
): | |
super().__init__(**kwargs) | |
self.pad_token_id = self.eos_token_id | |
from transformers import GPTNeoXTokenizerFast | |
class MambaTokenizerFast(GPTNeoXTokenizerFast): | |
def __init__( | |
self, | |
**kwargs, | |
): | |
super().__init__(**kwargs) | |
self.pad_token_id = self.eos_token_id | |