Jeronymous commited on
Commit
a776682
·
verified ·
1 Parent(s): e6cf6f4

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -59,7 +59,7 @@
59
  "special": true
60
  }
61
  },
62
- "additional_special_tokens": ["<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>", "</s>"],
63
  "bos_token": "<s>",
64
  "clean_up_tokenization_spaces": false,
65
  "eos_token": "<|eot_id|>",
 
59
  "special": true
60
  }
61
  },
62
+ "additional_special_tokens": ["<|start_header_id|>", "<|end_header_id|>", "</s>"],
63
  "bos_token": "<s>",
64
  "clean_up_tokenization_spaces": false,
65
  "eos_token": "<|eot_id|>",