Removed CustomTokenizer class

precious3_gpt_multi_modal.py

@@ -28,14 +28,6 @@ class Custom_MPTConfig(MPTConfig):
     def __init__(self):
         super().__init__()
 
-class CustomTokenizer(PreTrainedTokenizerFast):
-    def __init__(self, **kwargs):
-        super().__init__( tokenizer_file="../tokenizer.json",
-        unk_token="[UNK]",
-        pad_token="[PAD]",
-        eos_token="[EOS]",
-        bos_token="[BOS]", **kwargs)
-
 
 class Custom_MptModel(MPTModel): # MptModel
     def __init__(self, config: MPTConfig, modality0_dim=128, modality2_dim=1536):