update

tokenizer.py

@@ -7,14 +7,13 @@ class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
     def _batch_encode_plus(self, *args, **kwargs):
         outputs = super()._batch_encode_plus(*args, **kwargs)
         del outputs["token_type_ids"]
-        # if the last token is eos, remove it
-        # for key in ['input_ids', 'attention_mask']:
-        #     if isinstance(outputs[key], torch.Tensor):
-        #         outputs[key] = outputs[key][..., :-1]
-        #     elif isinstance(outputs[key], numpy.ndarray): 
-        #         outputs[key] = outputs[key][..., :-1]
-        #     elif isinstance(outputs[key], list):
-        #         outputs[key] = [sequence[:-1] for sequence in outputs[key]]
+        for key in ['input_ids', 'attention_mask']:
+            if isinstance(outputs[key], torch.Tensor):
+                outputs[key] = outputs[key][..., :-1]
+            elif isinstance(outputs[key], numpy.ndarray): 
+                outputs[key] = outputs[key][..., :-1]
+            elif isinstance(outputs[key], list):
+                outputs[key] = [sequence[:-1] for sequence in outputs[key]]
         return outputs
 
 # Register the class

tokenizer_config.json

@@ -938,7 +938,6 @@
   "unk_token": "[UNK]",
   "eos_token": "[SEP]",
   "tokenizer_class": "ModernDecoderBERTTokenizer",
-  "truncation": "right",
   "auto_map": {
     "AutoConfig": "orionweller/test-flex-gpt--configuration_bert.FlexBertConfig",
     "AutoTokenizer": [