perlthoughts commited on
Commit
5313b45
1 Parent(s): 87e0c5b

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +6 -1
tokenizer_config.json CHANGED
@@ -1,4 +1,6 @@
1
  {
 
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "<unk>",
@@ -28,9 +30,12 @@
28
  "additional_special_tokens": [],
29
  "bos_token": "<s>",
30
  "clean_up_tokenization_spaces": false,
31
- "device_map": "auto",
 
 
32
  "eos_token": "</s>",
33
  "legacy": true,
 
34
  "model_max_length": 1000000000000000019884624838656,
35
  "pad_token": null,
36
  "sp_model_kwargs": {},
 
1
  {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
 
30
  "additional_special_tokens": [],
31
  "bos_token": "<s>",
32
  "clean_up_tokenization_spaces": false,
33
+ "device_map": {
34
+ "": "cuda"
35
+ },
36
  "eos_token": "</s>",
37
  "legacy": true,
38
+ "low_cpu_mem_usage": true,
39
  "model_max_length": 1000000000000000019884624838656,
40
  "pad_token": null,
41
  "sp_model_kwargs": {},