orionweller
/

test-flex-gpt

Model card Files Files and versions Community

oweller2 commited on 30 days ago

Commit

970954b

•

1 Parent(s): 6d20d8a

update

Files changed (1) hide show

tokenizer.py +4 -0

tokenizer.py CHANGED Viewed

@@ -5,3 +5,7 @@ class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
         if token_ids_1 is None:
             return [id for id in token_ids_0 if id != self.eos_token_id]
         return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]

         if token_ids_1 is None:
             return [id for id in token_ids_0 if id != self.eos_token_id]
         return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]
+# Register the class
+from transformers import AutoTokenizer
+AutoTokenizer.register(ModernDecoderBERTTokenizer)