add DPS based tokens
Browse files- tokenizer.json +5 -5
tokenizer.json
CHANGED
@@ -64144,11 +64144,11 @@
|
|
64144 |
"댜": 63987,
|
64145 |
"<|im_start|>": 63988,
|
64146 |
"<|im_end|>": 63989,
|
64147 |
-
"<|
|
64148 |
-
"<|
|
64149 |
-
"<|
|
64150 |
-
"<|
|
64151 |
-
"<|
|
64152 |
"<|reserved_5|>": 63995,
|
64153 |
"<|reserved_6|>": 63996,
|
64154 |
"<|reserved_7|>": 63997,
|
|
|
64144 |
"댜": 63987,
|
64145 |
"<|im_start|>": 63988,
|
64146 |
"<|im_end|>": 63989,
|
64147 |
+
"<|sep|>": 63990,
|
64148 |
+
"<|endoftext|>": 63991,
|
64149 |
+
"<|acc|>": 63992,
|
64150 |
+
"<|rrn|>": 63993,
|
64151 |
+
"<|tel|>": 63994,
|
64152 |
"<|reserved_5|>": 63995,
|
64153 |
"<|reserved_6|>": 63996,
|
64154 |
"<|reserved_7|>": 63997,
|