Upload tokenizer
Browse files- tokenizer.json +0 -0
- vocab.txt +15 -1
tokenizer.json
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
vocab.txt
CHANGED
@@ -5,11 +5,17 @@
|
|
5 |
[MASK]
|
6 |
,
|
7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
π
|
9 |
—
|
10 |
…
|
11 |
、
|
12 |
-
。
|
13 |
〇
|
14 |
ㄚ
|
15 |
一
|
@@ -9020,6 +9026,14 @@
|
|
9020 |
:
|
9021 |
;
|
9022 |
?
|
|
|
|
|
9023 |
##〇
|
|
|
|
|
|
|
|
|
|
|
|
|
9024 |
〇〇
|
9025 |
〇〇〇
|
|
|
5 |
[MASK]
|
6 |
,
|
7 |
-
|
8 |
+
C
|
9 |
+
E
|
10 |
+
L
|
11 |
+
P
|
12 |
+
S
|
13 |
+
[
|
14 |
+
]
|
15 |
π
|
16 |
—
|
17 |
…
|
18 |
、
|
|
|
19 |
〇
|
20 |
ㄚ
|
21 |
一
|
|
|
9026 |
:
|
9027 |
;
|
9028 |
?
|
9029 |
+
##L
|
9030 |
+
##S
|
9031 |
##〇
|
9032 |
+
##E
|
9033 |
+
##P
|
9034 |
+
CL
|
9035 |
+
SE
|
9036 |
+
CLS
|
9037 |
+
SEP
|
9038 |
〇〇
|
9039 |
〇〇〇
|