emrgnt-cmplxty commited on
Commit
20ce201
1 Parent(s): a128f3c

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +5 -7
tokenizer_config.json CHANGED
@@ -1,4 +1,6 @@
1
  {
 
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "<unk>",
@@ -25,11 +27,7 @@
25
  "special": true
26
  }
27
  },
28
- "additional_special_tokens": [
29
- "<unk>",
30
- "<s>",
31
- "</s>"
32
- ],
33
  "bos_token": "<s>",
34
  "clean_up_tokenization_spaces": false,
35
  "eos_token": "</s>",
@@ -40,5 +38,5 @@
40
  "spaces_between_special_tokens": false,
41
  "tokenizer_class": "LlamaTokenizer",
42
  "unk_token": "<unk>",
43
- "use_default_system_prompt": true
44
- }
 
1
  {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
 
27
  "special": true
28
  }
29
  },
30
+ "additional_special_tokens": [],
 
 
 
 
31
  "bos_token": "<s>",
32
  "clean_up_tokenization_spaces": false,
33
  "eos_token": "</s>",
 
38
  "spaces_between_special_tokens": false,
39
  "tokenizer_class": "LlamaTokenizer",
40
  "unk_token": "<unk>",
41
+ "use_default_system_prompt": false
42
+ }