ellen625 commited on
Commit
b6d7006
1 Parent(s): 014d1b8

Upload tokenizer

Browse files
Files changed (2) hide show
  1. README.md +1 -1
  2. tokenizer_config.json +1 -1
README.md CHANGED
@@ -1,8 +1,8 @@
1
  ---
2
  license: other
3
- base_model: facebook/opt-125m
4
  tags:
5
  - generated_from_trainer
 
6
  model-index:
7
  - name: opt_125_wiki_k10
8
  results: []
 
1
  ---
2
  license: other
 
3
  tags:
4
  - generated_from_trainer
5
+ base_model: facebook/opt-125m
6
  model-index:
7
  - name: opt_125_wiki_k10
8
  results: []
tokenizer_config.json CHANGED
@@ -23,7 +23,7 @@
23
  "clean_up_tokenization_spaces": true,
24
  "eos_token": "</s>",
25
  "errors": "replace",
26
- "model_max_length": 256,
27
  "pad_token": "</s>",
28
  "tokenizer_class": "GPT2Tokenizer",
29
  "unk_token": "</s>"
 
23
  "clean_up_tokenization_spaces": true,
24
  "eos_token": "</s>",
25
  "errors": "replace",
26
+ "model_max_length": 1000000000000000019884624838656,
27
  "pad_token": "</s>",
28
  "tokenizer_class": "GPT2Tokenizer",
29
  "unk_token": "</s>"