teknium commited on
Commit
98986c8
1 Parent(s): d455e34

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +6 -6
tokenizer_config.json CHANGED
@@ -28,19 +28,19 @@
28
  },
29
  "32000": {
30
  "content": "<|im_end|>",
31
- "lstrip": true,
32
- "normalized": true,
33
- "rstrip": true,
34
  "single_word": false,
35
- "special": false
36
  },
37
  "32001": {
38
  "content": "<|im_start|>",
39
  "lstrip": true,
40
- "normalized": true,
41
  "rstrip": true,
42
  "single_word": false,
43
- "special": false
44
  }
45
  },
46
  "additional_special_tokens": [],
 
28
  },
29
  "32000": {
30
  "content": "<|im_end|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
  "single_word": false,
35
+ "special": true
36
  },
37
  "32001": {
38
  "content": "<|im_start|>",
39
  "lstrip": true,
40
+ "normalized": false,
41
  "rstrip": true,
42
  "single_word": false,
43
+ "special": true
44
  }
45
  },
46
  "additional_special_tokens": [],