Upload tokenizer/added_tokens.json with huggingface_hub
Browse files- tokenizer/added_tokens.json +30 -0
tokenizer/added_tokens.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"[new_id_0]": 32100,
|
| 3 |
+
"[new_id_10]": 32110,
|
| 4 |
+
"[new_id_11]": 32111,
|
| 5 |
+
"[new_id_12]": 32112,
|
| 6 |
+
"[new_id_13]": 32113,
|
| 7 |
+
"[new_id_14]": 32114,
|
| 8 |
+
"[new_id_15]": 32115,
|
| 9 |
+
"[new_id_16]": 32116,
|
| 10 |
+
"[new_id_17]": 32117,
|
| 11 |
+
"[new_id_18]": 32118,
|
| 12 |
+
"[new_id_19]": 32119,
|
| 13 |
+
"[new_id_1]": 32101,
|
| 14 |
+
"[new_id_20]": 32120,
|
| 15 |
+
"[new_id_21]": 32121,
|
| 16 |
+
"[new_id_22]": 32122,
|
| 17 |
+
"[new_id_23]": 32123,
|
| 18 |
+
"[new_id_24]": 32124,
|
| 19 |
+
"[new_id_25]": 32125,
|
| 20 |
+
"[new_id_26]": 32126,
|
| 21 |
+
"[new_id_27]": 32127,
|
| 22 |
+
"[new_id_2]": 32102,
|
| 23 |
+
"[new_id_3]": 32103,
|
| 24 |
+
"[new_id_4]": 32104,
|
| 25 |
+
"[new_id_5]": 32105,
|
| 26 |
+
"[new_id_6]": 32106,
|
| 27 |
+
"[new_id_7]": 32107,
|
| 28 |
+
"[new_id_8]": 32108,
|
| 29 |
+
"[new_id_9]": 32109
|
| 30 |
+
}
|