Upload tokenizer
Browse files- README.md +1 -1
- tokenizer_config.json +1 -1
README.md
CHANGED
@@ -1,8 +1,8 @@
|
|
1 |
---
|
2 |
license: other
|
3 |
-
base_model: facebook/opt-125m
|
4 |
tags:
|
5 |
- generated_from_trainer
|
|
|
6 |
model-index:
|
7 |
- name: opt_125_wiki_k10
|
8 |
results: []
|
|
|
1 |
---
|
2 |
license: other
|
|
|
3 |
tags:
|
4 |
- generated_from_trainer
|
5 |
+
base_model: facebook/opt-125m
|
6 |
model-index:
|
7 |
- name: opt_125_wiki_k10
|
8 |
results: []
|
tokenizer_config.json
CHANGED
@@ -23,7 +23,7 @@
|
|
23 |
"clean_up_tokenization_spaces": true,
|
24 |
"eos_token": "</s>",
|
25 |
"errors": "replace",
|
26 |
-
"model_max_length":
|
27 |
"pad_token": "</s>",
|
28 |
"tokenizer_class": "GPT2Tokenizer",
|
29 |
"unk_token": "</s>"
|
|
|
23 |
"clean_up_tokenization_spaces": true,
|
24 |
"eos_token": "</s>",
|
25 |
"errors": "replace",
|
26 |
+
"model_max_length": 1000000000000000019884624838656,
|
27 |
"pad_token": "</s>",
|
28 |
"tokenizer_class": "GPT2Tokenizer",
|
29 |
"unk_token": "</s>"
|