byeongal commited on
Commit
f589442
1 Parent(s): 8606d4f

update tokenizer

Browse files
Files changed (3) hide show
  1. merges.txt +0 -0
  2. tokenizer_config.json +1 -1
  3. vocab.json +0 -0
merges.txt ADDED
The diff for this file is too large to render. See raw diff
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "special_tokens_map_file": null, "name_or_path": "bart-base"}
1
+ {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "special_tokens_map_file": null}
vocab.json ADDED
The diff for this file is too large to render. See raw diff