Add special character tokens for broken words

added_tokens.json

@@ -166,27 +166,59 @@
   "[SUBJECT_PREFIX:2ps]": 32130,
   "[SUBJECT_PREFIX:3p]": 32171,
   "[SUB_VERB]": 32197,
+  "[w0q]": 32216,
+  "[w10q]": 32224,
+  "[w11q]": 32225,
+  "[w12q]": 32226,
+  "[w13q]": 32227,
+  "[w14q]": 32228,
+  "[w15q]": 32229,
+  "[w1q]": 32204,
+  "[w1q]a": 32211,
+  "[w1q]and[w4q]": 32200,
+  "[w1q]e": 32209,
+  "[w1q]ep[w4q]": 32205,
+  "[w1q]o": 32207,
+  "[w1q]os": 32199,
+  "[w2q]": 32217,
+  "[w3q]": 32218,
+  "[w4q]": 32219,
+  "[w5q]": 32214,
+  "[w6q]": 32220,
+  "[w7q]": 32221,
+  "[w8q]": 32222,
+  "[w9q]": 32223,
   "a'e": 32127,
+  "a[w15q]e": 32213,
   "amo": 32104,
   "bo": 32110,
   "e'ym": 32139,
+  "e[w15q]ym": 32201,
+  "end[w4q]": 32215,
   "endé": 32128,
   "ere": 32157,
+  "g[w0q]i": 32212,
   "gûi": 32185,
+  "ix[w4q]": 32208,
   "ixé": 32165,
   "mo": 32164,
   "n'": 32126,
+  "n[w15q]": 32202,
   "na": 32155,
   "namo": 32142,
   "nde": 32181,
   "opo": 32183,
+  "or[w4q]": 32210,
   "oro": 32105,
   "oré": 32169,
   "pa": 32136,
+  "pe[w10q]": 32206,
+  "pe[w1q]ep[w4q]": 32198,
   "peîepé": 32117,
   "peẽ": 32138,
   "ramo": 32161,
   "ta": 32129,
+  "um[w10q]": 32203,
   "umẽ": 32191,
   "xe": 32182,
   "í": 32121,

config.json

@@ -57,5 +57,5 @@
   "torch_dtype": "float32",
   "transformers_version": "4.38.1",
   "use_cache": true,
-  "vocab_size": 32198
+  "vocab_size": 32230
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d80dfecd26b9f6c7e6fecc204679f434def1ff66f0c81494fdbb784d1234b6a9
-size 242185256
+oid sha256:e50d118ba9f820756e704f94946c0589e8e6def3296aafae22476dfd1eb1fcc5
+size 242250792

special_tokens_map.json

@@ -21,20 +21,6 @@
       "rstrip": false,
       "single_word": false
     },
-    {
-      "content": "î",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "îepé",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
     {
       "content": "e",
       "lstrip": false,
@@ -50,21 +36,21 @@
       "single_word": false
     },
     {
-      "content": "ta",
+      "content": "pe[w1q]ep[w4q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "n'",
+      "content": "ta",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "peẽ",
+      "content": "[w1q]os",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -78,7 +64,7 @@
       "single_word": false
     },
     {
-      "content": "îe",
+      "content": "[w1q]and[w4q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -112,13 +98,6 @@
       "rstrip": false,
       "single_word": false
     },
-    {
-      "content": "îos",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
     {
       "content": "[MAIN_VERB]",
       "lstrip": false,
@@ -141,7 +120,7 @@
       "single_word": false
     },
     {
-      "content": "û",
+      "content": "e[w15q]ym",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -183,14 +162,14 @@
       "single_word": false
     },
     {
-      "content": "[GERUND_SUFFIX:CLASS_1:IYU]",
+      "content": "n[w15q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "îo",
+      "content": "[GERUND_SUFFIX:CLASS_1:IYU]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -239,14 +218,21 @@
       "single_word": false
     },
     {
-      "content": "xe",
+      "content": "um[w10q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "a'e",
+      "content": "[w1q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "xe",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -280,6 +266,13 @@
       "rstrip": false,
       "single_word": false
     },
+    {
+      "content": "[w1q]ep[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
     {
       "content": "[SUBJECT_PREFIX:1ppe]",
       "lstrip": false,
@@ -301,6 +294,13 @@
       "rstrip": false,
       "single_word": false
     },
+    {
+      "content": "pe[w10q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
     {
       "content": "[OBJECT:1ppe]",
       "lstrip": false,
@@ -329,6 +329,13 @@
       "rstrip": false,
       "single_word": false
     },
+    {
+      "content": "[w1q]o",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
     {
       "content": "[NEGATION_PREFIX]",
       "lstrip": false,
@@ -336,6 +343,13 @@
       "rstrip": false,
       "single_word": false
     },
+    {
+      "content": "ix[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
     {
       "content": "[CIRCUMSTANTIAL_SUFFIX:NULL_ENDING]",
       "lstrip": false,
@@ -357,6 +371,13 @@
       "rstrip": false,
       "single_word": false
     },
+    {
+      "content": "[w1q]e",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
     {
       "content": "pe",
       "lstrip": false,
@@ -386,70 +407,70 @@
       "single_word": false
     },
     {
-      "content": "[GERUND_SUBJECT_PREFIX:2ps]",
+      "content": "or[w4q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[OBJECT:3p]",
+      "content": "[GERUND_SUBJECT_PREFIX:2ps]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "îandé",
+      "content": "[w1q]a",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "gûi",
+      "content": "g[w0q]i",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[OBJECT:2pp:SUBJECT_1P]",
+      "content": "[OBJECT:3p]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[CIRCUMSTANTIAL_SUFFIX:VOWEL_ENDING]",
+      "content": "[OBJECT:2pp:SUBJECT_1P]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[OBJECT_MARKER:3p:MONOSYLLABIC]",
+      "content": "[CIRCUMSTANTIAL_SUFFIX:VOWEL_ENDING]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "peîepé",
+      "content": "[OBJECT_MARKER:3p:MONOSYLLABIC]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[GERUND_SUFFIX:CLASS_1:R]",
+      "content": "a[w15q]e",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "endé",
+      "content": "[GERUND_SUFFIX:CLASS_1:R]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -511,13 +532,6 @@
       "rstrip": false,
       "single_word": false
     },
-    {
-      "content": "e'ym",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
     {
       "content": "[SUBJECT_PREFIX:3p]",
       "lstrip": false,
@@ -547,42 +561,42 @@
       "single_word": false
     },
     {
-      "content": "ixé",
+      "content": "[GERUND_SUFFIX:CLASS_1:ORAL_VOWEL]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[GERUND_SUFFIX:CLASS_1:ORAL_VOWEL]",
+      "content": "pa",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "pa",
+      "content": "ramo",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "ramo",
+      "content": "ere",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "ere",
+      "content": "[OBJECT:REFLEXIVE]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[OBJECT:REFLEXIVE]",
+      "content": "[w5q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -652,105 +666,203 @@
       "single_word": false
     },
     {
-      "content": "í",
+      "content": "[PLURIFORM_PREFIX:R]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "oré",
+      "content": "[SUBJECT:2ps]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "umẽ",
+      "content": "o",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[PLURIFORM_PREFIX:R]",
+      "content": "[SUBJECT:2ps:OBJECT_1P]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[SUBJECT:2ps]",
+      "content": "end[w4q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "o",
+      "content": "[OBJECT:1ps]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[SUBJECT:2ps:OBJECT_1P]",
+      "content": "[w0q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "îa",
+      "content": "[GERUND_SUFFIX:CLASS_2:DEFAULT]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[OBJECT:1ps]",
+      "content": "[GERUND_SUFFIX:CLASS_2:NASAL_VOWEL_ENDING]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[GERUND_SUFFIX:CLASS_2:DEFAULT]",
+      "content": "[SUBJECT_PREFIX:1ppi]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[GERUND_SUFFIX:CLASS_2:NASAL_VOWEL_ENDING]",
+      "content": "[SUBJECT_PREFIX:2ps]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[SUBJECT_PREFIX:1ppi]",
+      "content": "s",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[SUBJECT_PREFIX:2ps]",
+      "content": "[SPACE]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "s",
+      "content": "[w0q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "[SPACE]",
+      "content": "[w1q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w2q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w3q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w5q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w6q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w7q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w8q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w9q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w10q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w11q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w12q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w13q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w14q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[w15q]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,

tokenizer_config.json

@@ -1696,74 +1696,333 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "32198": {
+      "content": "pe[w1q]ep[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32199": {
+      "content": "[w1q]os",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32200": {
+      "content": "[w1q]and[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32201": {
+      "content": "e[w15q]ym",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32202": {
+      "content": "n[w15q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32203": {
+      "content": "um[w10q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32204": {
+      "content": "[w1q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32205": {
+      "content": "[w1q]ep[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32206": {
+      "content": "pe[w10q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32207": {
+      "content": "[w1q]o",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32208": {
+      "content": "ix[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32209": {
+      "content": "[w1q]e",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32210": {
+      "content": "or[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32211": {
+      "content": "[w1q]a",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32212": {
+      "content": "g[w0q]i",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32213": {
+      "content": "a[w15q]e",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32214": {
+      "content": "[w5q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32215": {
+      "content": "end[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32216": {
+      "content": "[w0q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32217": {
+      "content": "[w2q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32218": {
+      "content": "[w3q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32219": {
+      "content": "[w4q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32220": {
+      "content": "[w6q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32221": {
+      "content": "[w7q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32222": {
+      "content": "[w8q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32223": {
+      "content": "[w9q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32224": {
+      "content": "[w10q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32225": {
+      "content": "[w11q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32226": {
+      "content": "[w12q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32227": {
+      "content": "[w13q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32228": {
+      "content": "[w14q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32229": {
+      "content": "[w15q]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [
     "[SUBJECT:3p:DIRECT]",
     "[GERUND_SUBJECT_PREFIX:1ppi]",
     "[OBJECT:DIRECT]",
-    "î",
-    "îepé",
     "e",
     "[OBJECT:2pp]",
+    "pe[w1q]ep[w4q]",
     "ta",
-    "n'",
-    "peẽ",
+    "[w1q]os",
     "[SUBJECT:2pp]",
-    "îe",
+    "[w1q]and[w4q]",
     "[GERUND_SUBJECT_PREFIX:3p]",
     "[GERUND_SUBJECT_PREFIX:2pp]",
     "[NEGATION_PARTICLE:NA]",
     "[IMPERATIVE_PREFIX:2ps]",
-    "îos",
     "[MAIN_VERB]",
     "[GERUND_SUFFIX:CLASS_1:NASAL_IYU]",
     "[PLURIFORM_PREFIX:S]",
-    "û",
+    "e[w15q]ym",
     "[OBJECT_MARKER:3p:PLURIFORM_PREFIX:MONOSYLLABIC]",
     "[OBJECT:1ppi]",
     "abo",
     "[GERUND_SUFFIX:CLASS_1]",
     "[SUB_VERB]",
+    "n[w15q]",
     "[GERUND_SUFFIX:CLASS_1:IYU]",
-    "îo",
     "[NEGATION_SUFFIX]",
     "[GERUND_SUBJECT_PREFIX:1ppe]",
     "i",
     "[SUBJECT:3p]",
     "[IMPERATIVE_PREFIX:2pp]",
     "́",
+    "um[w10q]",
+    "[w1q]",
     "xe",
-    "a'e",
     "t",
     "[SUBJECT:1ppe]",
     "[SUBJECT:1ps]",
     "a",
+    "[w1q]ep[w4q]",
     "[SUBJECT_PREFIX:1ppe]",
     "amo",
     "[OBJECT:2ps]",
+    "pe[w10q]",
     "[OBJECT:1ppe]",
     "[NEGATION_SUFFIX:CONSONANT_ENDING]",
     "[OBJECT:MUTUAL]",
     "[GERUND_SUFFIX:CLASS_2:ORAL_VOWEL_ENDING]",
+    "[w1q]o",
     "[NEGATION_PREFIX]",
+    "ix[w4q]",
     "[CIRCUMSTANTIAL_SUFFIX:NULL_ENDING]",
     "[SUBJECT:1ppi]",
     "[PERMISSIVE_PREFIX:VOWEL]",
+    "[w1q]e",
     "pe",
     "nde",
     "[OBJECT_MARKER:3p:DEFAULT]",
     "opo",
+    "or[w4q]",
     "[GERUND_SUBJECT_PREFIX:2ps]",
+    "[w1q]a",
+    "g[w0q]i",
     "[OBJECT:3p]",
-    "îandé",
-    "gûi",
     "[OBJECT:2pp:SUBJECT_1P]",
     "[CIRCUMSTANTIAL_SUFFIX:VOWEL_ENDING]",
     "[OBJECT_MARKER:3p:MONOSYLLABIC]",
-    "peîepé",
+    "a[w15q]e",
     "[GERUND_SUFFIX:CLASS_1:R]",
-    "endé",
     "namo",
     "[SUBJECT:2pp:OBJECT_1P]",
     "mo",
@@ -1772,17 +2031,16 @@
     "[CIRCUMSTANTIAL_SUFFIX:CONSONANT_ENDING]",
     "[ROOT]",
     "[NEGATION_PARTICLE:UME]",
-    "e'ym",
     "[SUBJECT_PREFIX:3p]",
     "r",
     "[NEGATION_SUFFIX:VOWEL_ENDING]",
     "[SUBJECT_PREFIX:1ps]",
-    "ixé",
     "[GERUND_SUFFIX:CLASS_1:ORAL_VOWEL]",
     "pa",
     "ramo",
     "ere",
     "[OBJECT:REFLEXIVE]",
+    "[w5q]",
     "[GERUND_SUFFIX:CLASS_1:NASAL_VOWEL]",
     "na",
     "[GERUND_SUFFIX:CLASS_1:B]",
@@ -1792,21 +2050,35 @@
     "[OBJECT:2ps:SUBJECT_1P]",
     "[GERUND_SUBJECT_PREFIX:1ps]",
     "[PERMISSIVE_PREFIX:CONSONANT]",
-    "í",
-    "oré",
-    "umẽ",
     "[PLURIFORM_PREFIX:R]",
     "[SUBJECT:2ps]",
     "o",
     "[SUBJECT:2ps:OBJECT_1P]",
-    "îa",
+    "end[w4q]",
     "[OBJECT:1ps]",
+    "[w0q]",
     "[GERUND_SUFFIX:CLASS_2:DEFAULT]",
     "[GERUND_SUFFIX:CLASS_2:NASAL_VOWEL_ENDING]",
     "[SUBJECT_PREFIX:1ppi]",
     "[SUBJECT_PREFIX:2ps]",
     "s",
-    "[SPACE]"
+    "[SPACE]",
+    "[w0q]",
+    "[w1q]",
+    "[w2q]",
+    "[w3q]",
+    "[w4q]",
+    "[w5q]",
+    "[w6q]",
+    "[w7q]",
+    "[w8q]",
+    "[w9q]",
+    "[w10q]",
+    "[w11q]",
+    "[w12q]",
+    "[w13q]",
+    "[w14q]",
+    "[w15q]"
   ],
   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",