rishiraj commited on
Commit
935a84c
1 Parent(s): a1ea2d8

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +0 -1
  2. tokenizer_config.json +2 -2
special_tokens_map.json CHANGED
@@ -17,7 +17,6 @@
17
  "rstrip": false,
18
  "single_word": false
19
  },
20
- "pad_token": "<|end_of_turn|>",
21
  "unk_token": {
22
  "content": "<unk>",
23
  "lstrip": false,
 
17
  "rstrip": false,
18
  "single_word": false
19
  },
 
20
  "unk_token": {
21
  "content": "<unk>",
22
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -50,8 +50,8 @@
50
  "clean_up_tokenization_spaces": false,
51
  "eos_token": "<|end_of_turn|>",
52
  "legacy": true,
53
- "model_max_length": 2048,
54
- "pad_token": "<|end_of_turn|>",
55
  "sp_model_kwargs": {},
56
  "spaces_between_special_tokens": false,
57
  "tokenizer_class": "LlamaTokenizer",
 
50
  "clean_up_tokenization_spaces": false,
51
  "eos_token": "<|end_of_turn|>",
52
  "legacy": true,
53
+ "model_max_length": 1000000000000000019884624838656,
54
+ "pad_token": null,
55
  "sp_model_kwargs": {},
56
  "spaces_between_special_tokens": false,
57
  "tokenizer_class": "LlamaTokenizer",