Upload tokenizer.json
Browse files- tokenizer.json +2 -0
tokenizer.json
CHANGED
@@ -32165,6 +32165,8 @@
|
|
32165 |
"벨": 31997,
|
32166 |
"ゼ": 31998,
|
32167 |
"梦": 31999,
|
|
|
|
|
32168 |
"ㅡㅡ": 32002,
|
32169 |
"ㅜㅜ": 32003,
|
32170 |
"ㅠㅠ": 32004,
|
|
|
32165 |
"벨": 31997,
|
32166 |
"ゼ": 31998,
|
32167 |
"梦": 31999,
|
32168 |
+
"<|im_end|>": 32000,
|
32169 |
+
"<|im_start|>": 32001,
|
32170 |
"ㅡㅡ": 32002,
|
32171 |
"ㅜㅜ": 32003,
|
32172 |
"ㅠㅠ": 32004,
|