Commit
•
6e6f39d
1
Parent(s):
3c5b601
update tokenizer
Browse files- config.json +2 -1
- tokenizer_config.json +1 -1
config.json
CHANGED
|
@@ -35,5 +35,6 @@
|
|
| 35 |
"use_cache": true,
|
| 36 |
"vocab_size": 96103,
|
| 37 |
"num_beams": 5,
|
| 38 |
-
"length_penalty": 0.8
|
|
|
|
| 39 |
}
|
|
|
|
| 35 |
"use_cache": true,
|
| 36 |
"vocab_size": 96103,
|
| 37 |
"num_beams": 5,
|
| 38 |
+
"length_penalty": 0.8,
|
| 39 |
+
"tokenizer_class": "PegasusTokenizer"
|
| 40 |
}
|
tokenizer_config.json
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
{"
|
|
|
|
| 1 |
+
{"pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": "[MASK]", "mask_token_sent": null, "offset": 0, "additional_special_tokens": [], "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sep_token": {"content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "special_tokens_map_file": "/home/patrick/.cache/huggingface/transformers/b548e984b09823ed5cea0e622c0ec194a7e07b3c5d2e6b48a7cde84bd179cc7c.4eda581f816a0a941629106f0338c957910ce4839ecf7e3e743bb79523bf7249", "name_or_path": "./bigbird-pegasus-large-arxiv", "model_max_length": 4096}
|