add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Dec03_11-00-05_2627df782b8c/1638529295.762905/events.out.tfevents.1638529295.2627df782b8c.34.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16050063c7ea8a2911e80e34f6a66b16a8f80853fc4351f39c9f4ee5d5b3ccd1
+size 4573

runs/Dec03_11-00-05_2627df782b8c/events.out.tfevents.1638529295.2627df782b8c.34.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fb0a4fd33ea1610cf1e96d821848e776806d01fa8796a28563225ff9d332d52
+size 4487

runs/Dec03_11-02-54_2627df782b8c/1638529462.5650914/events.out.tfevents.1638529462.2627df782b8c.34.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61cb73e5dd1b35f1d2c98d4893a495d7c2e46b02edb208a5339a5034db2268b0
+size 4573

runs/Dec03_11-02-54_2627df782b8c/events.out.tfevents.1638529462.2627df782b8c.34.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6ae125d14e7530fd83e343d8a5c4742d8dc2c60d9b198978a4ae5cb2544a2b7
+size 4487

runs/Dec03_11-09-57_2627df782b8c/1638529861.2182262/events.out.tfevents.1638529861.2627df782b8c.34.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c8a047eec1edb9c96834f56bd840cee86904608d3b76bf2b5723f887b52ec00
+size 4573

runs/Dec03_11-09-57_2627df782b8c/events.out.tfevents.1638529861.2627df782b8c.34.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b9a9aa0126f3b816c755617b99444e1dc4a3a91f871d6f5d081776cc3a51721
+size 4487

runs/Dec03_11-17-15_2627df782b8c/1638530299.4366481/events.out.tfevents.1638530299.2627df782b8c.34.7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5f1b692e9e608cdc37cf2c1c4f88a3eb999549f739ccf5043df92ec011edda1
+size 4573

runs/Dec03_11-17-15_2627df782b8c/events.out.tfevents.1638530299.2627df782b8c.34.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:906a447dffb2f772e4e2794cec033185594127958b44113957b94b7cdadfd538
+size 4486

vocab.json

@@ -1 +1 @@
-{"б": 0, "ф": 2, "м": 3, "н": 4, "т": 5, "а": 6, "ю": 7, "г": 8, "у": 9, "э": 10, "к": 11, "ы": 12, "в": 13, "ь": 14, "я": 15, "ж": 16, "ё": 17, "ъ": 18, "х": 19, "щ": 20, "й": 21, "з": 22, "ш": 23, "п": 24, "ч": 25, "е": 26, "д": 27, "р": 28, "о": 29, "с": 30, "и": 31, "л": 32, "ц": 33, "|": 1, "[UNK]": 34, "[PAD]": 35}
+{"с": 0, "р": 1, "ю": 2, "ж": 3, "щ": 4, "ё": 5, "и": 6, "я": 7, "й": 8, "д": 9, "ч": 10, "ь": 12, "г": 13, "ш": 14, "е": 15, "х": 16, "т": 17, "н": 18, "ы": 19, "б": 20, "у": 21, "л": 22, "в": 23, "к": 24, "э": 25, "з": 26, "о": 27, "ф": 28, "ъ": 29, "м": 30, "ц": 31, "а": 32, "п": 33, "|": 11, "[UNK]": 34, "[PAD]": 35}