add tokenizer
Browse files
runs/Aug04_16-52-02_927bdb83f4b9/1659632020.6624546/events.out.tfevents.1659632020.927bdb83f4b9.88.5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ca5cef0b6488aaedd66218fd86cf309f3d7cfb1bae5a4cbe54638091358ac101
|
| 3 |
+
size 5515
|
runs/Aug04_16-52-02_927bdb83f4b9/events.out.tfevents.1659632020.927bdb83f4b9.88.4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:824931834914a133525778dd570d780eddf0630676a53e45d6f929e73076dd8b
|
| 3 |
+
size 5395
|
vocab.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
-
"[PAD]":
|
| 3 |
-
"[UNK]":
|
| 4 |
"|": 20,
|
| 5 |
"ㄱ": 19,
|
| 6 |
"ㄲ": 38,
|
|
|
|
| 1 |
{
|
| 2 |
+
"[PAD]": 50,
|
| 3 |
+
"[UNK]": 49,
|
| 4 |
"|": 20,
|
| 5 |
"ㄱ": 19,
|
| 6 |
"ㄲ": 38,
|