update

tokenizer.py

@@ -1,10 +1,15 @@
-from transformers import PreTrainedTokenizerFast
+from transformers import PreTrainedTokenizer
 
-class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
+class ModernDecoderBERTTokenizer(PreTrainedTokenizer):
     def __init__(self, *args, **kwargs):
+        breakpoint()
         super().__init__(*args, **kwargs)
         
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        breakpoint()
         if token_ids_1 is None:
             return [id for id in token_ids_0 if id != self.eos_token_id]
-        return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]
+        return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]
+
+
+AutoTokenizer.register("ModernDecoderBERTTokenizer", ModernDecoderBERTTokenizer)

tokenizer_config.json

@@ -937,10 +937,10 @@
   "sep_token": "[SEP]",
   "unk_token": "[UNK]",
   "eos_token": "[SEP]",
-  "tokenizer_class": "ModernDecoderBERTTokenizer",
+  "tokenizer_class": "orionweller/test-flex-gpt--tokenizer.ModernDecoderBERTTokenizer",
   "truncation": "right",
   "auto_map": {
-    "AutoConfig": "orionweller/test-flex-gpt--configuration_bert.FlexBertConfig",
+    "AutoConfig": "orionweller/test-flex-gpt--tokenizer.ModernDecoderBERTTokenizer",
     "AutoTokenizer": [
       "orionweller/test-flex-gpt--tokenizer.ModernDecoderBERTTokenizer",
       "orionweller/test-flex-gpt--tokenizer.ModernDecoderBERTTokenizer"