Delete data/token_list
Browse files
data/token_list/bpe_unigram32/bpe.model
DELETED
@@ -1,3 +0,0 @@
|
|
1 |
-
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:8b6b8c0c690d78825417ed02f63172d66941ccd0979b666c9e8986ac90805c1e
|
3 |
-
size 238013
|
|
|
|
|
|
|
|
data/token_list/bpe_unigram32/tokens.txt
DELETED
@@ -1,32 +0,0 @@
|
|
1 |
-
<blank>
|
2 |
-
<unk>
|
3 |
-
▁
|
4 |
-
a
|
5 |
-
r
|
6 |
-
e
|
7 |
-
o
|
8 |
-
n
|
9 |
-
l
|
10 |
-
t
|
11 |
-
d
|
12 |
-
m
|
13 |
-
i
|
14 |
-
2
|
15 |
-
s
|
16 |
-
0
|
17 |
-
9
|
18 |
-
y
|
19 |
-
ñ
|
20 |
-
1
|
21 |
-
u
|
22 |
-
h
|
23 |
-
p
|
24 |
-
c
|
25 |
-
3
|
26 |
-
4
|
27 |
-
7
|
28 |
-
8
|
29 |
-
z
|
30 |
-
b
|
31 |
-
f
|
32 |
-
<sos/eos>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|