ADD: add fast tokenizer

added_tokens.json

@@ -1 +1 @@
-{"<pad>": 40000, "<mask>": 40001}
+{"<mask>": 40001, "<pad>": 40000}

save_tokenizer.py

@@ -1,9 +0,0 @@
-from transformers import T5Tokenizer
-
-tokenizer = T5Tokenizer.from_pretrained(
-    '/cognitive_comp/common_data/tokenizers/sentence_piece_bpe/bpe_v40000_s42_cov0.9995_max6_corpus1M.model',
-    additional_special_tokens=['<s>', '<mask>'],
-    extra_ids=0)
-tokenizer.bos_token = '<s>'
-tokenizer.mask_token = '<mask>'
-tokenizer.save_pretrained('/cognitive_comp/gaoxinyu/pretrained_model/bart-base')

tokenizer_config.json

@@ -8,6 +8,7 @@
         "<mask>"
     ],
     "sp_model_kwargs": {},
-    "name_or_path": "/cognitive_comp/common_data/tokenizers/sentence_piece_bpe/bpe_v40000_s42_cov0.9995_max6_corpus1M.model",
+    "name_or_path": "/cognitive_comp/gaoxinyu/hf_hub/Randeng-BART-139M",
+    "special_tokens_map_file": "/cognitive_comp/gaoxinyu/hf_hub/Randeng-BART-139M/special_tokens_map.json",
     "tokenizer_class": "T5Tokenizer"
 }