tuanio
/

w2v2-vi-en-char-based

Model card Files Files and versions Community

tuanio commited on Oct 6, 2023

Commit

6015fd8

•

1 Parent(s): faf4d9c

Upload tokenizer

Files changed (3) hide show

added_tokens.json +2 -3
special_tokens_map.json +0 -7
vocab.json +3 -4

added_tokens.json CHANGED Viewed

@@ -1,5 +1,4 @@
 {
-  "</s>": 146,
-  "<s>": 145,
-  "[PAD]": 147
 }

 {
+  "</s>": 145,
+  "<s>": 144
 }

special_tokens_map.json CHANGED Viewed

@@ -13,13 +13,6 @@
       "normalized": true,
       "rstrip": false,
       "single_word": false
-    },
-    {
-      "content": "[PAD]",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false
     }
   ],
   "bos_token": "<s>",

       "normalized": true,
       "rstrip": false,
       "single_word": false
     }
   ],
   "bos_token": "<s>",

vocab.json CHANGED Viewed

@@ -1,5 +1,4 @@
 {
-  "!": 1,
   "#": 2,
   "&": 3,
   "'": 4,
@@ -24,6 +23,8 @@
   ">": 23,
   "?": 24,
   "@": 25,
   "_": 26,
   "a": 27,
   "b": 28,
@@ -136,12 +137,10 @@
   "ỵ": 134,
   "ỷ": 135,
   "ỹ": 136,
-  "‎": 137,
   "–": 138,
   "‘": 139,
   "’": 140,
   "“": 141,
   "”": 142,
-  "…": 143,
-  "": 144
 }

 {
   "#": 2,
   "&": 3,
   "'": 4,
   ">": 23,
   "?": 24,
   "@": 25,
+  "[PAD]": 146,
+  "[UNK]": 145,
   "_": 26,
   "a": 27,
   "b": 28,
   "ỵ": 134,
   "ỷ": 135,
   "ỹ": 136,
   "–": 138,
   "‘": 139,
   "’": 140,
   "“": 141,
   "”": 142,
+  "…": 143
 }