IlyaGusev commited on
Commit
9fe4929
1 Parent(s): 03b13f7

Tokenizer config fix

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
  "tokenizer_class": "LlamaTokenizer",
3
- "model_max_length": 2048,
4
  "padding_side": "left",
5
  "bos_token": "<s>",
6
  "eos_token": "</s>",
7
  "unk_token": "<unk>",
8
  "clean_up_tokenization_spaces": false,
9
  "special_tokens_map_file": "special_tokens_map.json"
10
- }
 
1
  {
2
  "tokenizer_class": "LlamaTokenizer",
3
+ "model_max_length": 4096,
4
  "padding_side": "left",
5
  "bos_token": "<s>",
6
  "eos_token": "</s>",
7
  "unk_token": "<unk>",
8
  "clean_up_tokenization_spaces": false,
9
  "special_tokens_map_file": "special_tokens_map.json"
10
+ }