add special_tokens

README.md

@@ -15,8 +15,7 @@ from transformers import PreTrainedTokenizerFast
 from transformers import BartForConditionalGeneration
 
 tokenizer = PreTrainedTokenizerFast.from_pretrained(
-    'gogamza/kobart-summarization',
-    bos_token='<s>', eos_token='</s>', unk_token='<unk>', pad_token='<pad>', mask_token='<mask>')
+    'gogamza/kobart-summarization')
 
 model = BartForConditionalGeneration.from_pretrained('gogamza/kobart-summarization')
 

added_tokens.json

@@ -0,0 +1,2 @@
+{}
+

special_tokens_map.json

@@ -0,0 +1,2 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": "<mask>"}
+