beomi commited on
Commit
eb9527e
1 Parent(s): 837f45c

add DPS based tokens

Browse files
Files changed (1) hide show
  1. tokenizer.json +5 -5
tokenizer.json CHANGED
@@ -64144,11 +64144,11 @@
64144
  "댜": 63987,
64145
  "<|im_start|>": 63988,
64146
  "<|im_end|>": 63989,
64147
- "<|reserved_0|>": 63990,
64148
- "<|reserved_1|>": 63991,
64149
- "<|reserved_2|>": 63992,
64150
- "<|reserved_3|>": 63993,
64151
- "<|reserved_4|>": 63994,
64152
  "<|reserved_5|>": 63995,
64153
  "<|reserved_6|>": 63996,
64154
  "<|reserved_7|>": 63997,
 
64144
  "댜": 63987,
64145
  "<|im_start|>": 63988,
64146
  "<|im_end|>": 63989,
64147
+ "<|sep|>": 63990,
64148
+ "<|endoftext|>": 63991,
64149
+ "<|acc|>": 63992,
64150
+ "<|rrn|>": 63993,
64151
+ "<|tel|>": 63994,
64152
  "<|reserved_5|>": 63995,
64153
  "<|reserved_6|>": 63996,
64154
  "<|reserved_7|>": 63997,