Upload folder using huggingface_hub

Browse files

Files changed (5) hide show

README.md +5 -1
special_tokens_map.json +0 -4
tokenizer.json +29 -47
tokenizer_config.json +0 -20
vocab.json +310 -310

README.md CHANGED Viewed

@@ -47,4 +47,8 @@ tokens = tokenizer.encode("Hello, world!")
 ## Sample Encoding
 | Text | Tokens | Token IDs |
 |------|--------|-----------|
-| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 3, 70, 81, 77, 87, 92, 4, 81, 86, 87, 92` |

 ## Sample Encoding
 | Text | Tokens | Token IDs |
 |------|--------|-----------|
+| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 13, 70, 81, 77, 87, 92, 31, 81, 86, 87, 92` |
+Command used to create this tokenizer:
+```bash
+['/home/gsa/tokenizers2/flexitok/tokenizer_training/train_tokenizers.py', 'algorithm=bpe', 'vocab_size=2000', 'langs=[fas_Arab]', 'data_dir=/scratch/gsa/data/multilingual-addition/', 'output_dir=/scratch/gsa/trained_tokenizers/multilingual_addition', 'pretokenizer=custom:addition', 'number_handling=ltr_3digit', 'add_numbers=false', 'handle_contractions=false', 'unicode_normalization=nfc', 'use_byte_level_regex=false', 'byte_fallback=false', 'strip_zero_width=false', 'cjk_char_split=false', 'add_cjk_chars=false', 'max_lines=-1', 'test_string=yirmi iki+dokuz=otuz bir\\ntwenty two+nine=thirty one', 'hf.publish_to_hf=true', 'hf_repo_prefix=flexitok/', 'hf.hf_repo_id=flexitok/maddition_fas_Arab_2000', 'hf.collections=[flexitok/multilingual_addition_tokenizers]']

special_tokens_map.json CHANGED Viewed

@@ -1,8 +1,4 @@
 {
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>"

 {
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>"

tokenizer.json CHANGED Viewed

@@ -29,24 +29,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 3,
-      "content": "+",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 4,
-      "content": "=",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {
@@ -99,35 +81,35 @@
       "<s>": 0,
       "</s>": 1,
       "<pad>": 2,
-      "+": 3,
-      "=": 4,
-      "!": 5,
-      "\"": 6,
-      "#": 7,
-      "$": 8,
-      "%": 9,
-      "&": 10,
-      "'": 11,
-      "(": 12,
-      ")": 13,
-      "*": 14,
-      ",": 15,
-      "-": 16,
-      ".": 17,
-      "/": 18,
-      "0": 19,
-      "1": 20,
-      "2": 21,
-      "3": 22,
-      "4": 23,
-      "5": 24,
-      "6": 25,
-      "7": 26,
-      "8": 27,
-      "9": 28,
-      ":": 29,
-      ";": 30,
-      "<": 31,
       ">": 32,
       "?": 33,
       "@": 34,

       "rstrip": false,
       "normalized": false,
       "special": true
     }
   ],
   "normalizer": {
       "<s>": 0,
       "</s>": 1,
       "<pad>": 2,
+      "!": 3,
+      "\"": 4,
+      "#": 5,
+      "$": 6,
+      "%": 7,
+      "&": 8,
+      "'": 9,
+      "(": 10,
+      ")": 11,
+      "*": 12,
+      "+": 13,
+      ",": 14,
+      "-": 15,
+      ".": 16,
+      "/": 17,
+      "0": 18,
+      "1": 19,
+      "2": 20,
+      "3": 21,
+      "4": 22,
+      "5": 23,
+      "6": 24,
+      "7": 25,
+      "8": 26,
+      "9": 27,
+      ":": 28,
+      ";": 29,
+      "<": 30,
+      "=": 31,
       ">": 32,
       "?": 33,
       "@": 34,

tokenizer_config.json CHANGED Viewed

@@ -23,28 +23,8 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "3": {
-      "content": "+",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "=",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",

       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",

vocab.json CHANGED Viewed

@@ -1,342 +1,342 @@
 {
-  "Ø¯ÙĪÛĮØ³Øª": 310,
-  "ô": 179,
-  "ÿ": 190,
-  "µ": 116,
-  "5": 24,
   "ą": 196,
-  "ü": 187,
-  "ý": 188,
-  "Ġ": 223,
   "î": 173,
-  "ÙĩØ²": 292,
-  "ó": 178,
-  "Ě": 217,
-  "Ĩ": 231,
-  "Æ": 133,
-  "Ą": 195,
-  "Ø³ÛĮØµØ¯": 309,
-  "ÙĨØµØ¯": 302,
-  "Į": 237,
-  "ë": 170,
-  "!": 5,
-  "=": 4,
   "j": 76,
-  "i": 75,
-  "ĝ": 220,
-  "ÙĨÙĪØ²Ø¯Ùĩ": 334,
-  "£": 99,
-  "®": 109,
-  "u": 87,
-  "Ù¾Ø§": 295,
-  "½": 124,
-  "ĭ": 236,
-  "Ĥ": 227,
   "]": 63,
-  "U": 55,
-  "^": 64,
-  "Ø´Ø´": 287,
-  "Ø¯ÙĪ": 282,
-  "8": 27,
-  "ı": 240,
-  "Ó": 146,
-  "[": 61,
-  "1": 20,
-  "Ľ": 252,
-  "ł": 257,
   "å": 164,
-  "é": 168,
-  "Ú©": 317,
-  "ĵ": 244,
   "¦": 102,
-  "Ù¾ÙĨØ¬Ø§Ùĩ": 314,
-  "Ă": 193,
-  "f": 72,
   "Ø´": 266,
-  "Ĕ": 211,
-  "Ė": 213,
-  "Ļ": 250,
-  "ö": 181,
-  "²Ø¯Ùĩ": 320,
-  "N": 48,
-  "ÙĨÙĪØ¯": 315,
-  "V": 56,
-  "ÙĨ": 267,
-  "v": 88,
-  "č": 204,
-  "º": 121,
-  "I": 43,
-  "Û": 154,
-  "ğ": 222,
-  "Ē": 209,
   "ÙĩÙģØª": 277,
-  "_": 65,
-  "ÙĩØ§Ø±": 281,
-  "Õ": 148,
   "Ü": 155,
-  "ú": 185,
-  "Ø¯": 261,
-  "Â": 129,
-  "ÙĪ": 259,
-  "Y": 59,
-  "ÙĩØ´ØªØ§Ø¯": 308,
-  "ÚĨ": 274,
-  "\"": 6,
-  "ľ": 253,
-  "Ù¾Ø§ÙĨØµØ¯": 316,
-  "ß": 158,
-  "¬Ø¯Ùĩ": 324,
-  "o": 81,
-  ">": 32,
-  "Ø´Ø§ÙĨØ²Ø¯Ùĩ": 328,
-  "Ø³ÛĮ": 280,
-  "h": 74,
-  "T": 54,
-  "Ń": 258,
-  "ð": 175,
-  "ï": 174,
-  "¹": 120,
-  "p": 82,
   "c": 69,
-  "ù": 184,
-  "Ø±": 336,
-  "W": 57,
-  "ĸ": 247,
   "ī": 234,
-  "ÛĮØ³Øª": 286,
-  "&": 10,
-  "ÚĨÙĩØ§Ø±Ø¯Ùĩ": 333,
-  "ŀ": 255,
-  "ċ": 202,
-  "¥": 101,
-  "Ċ": 201,
-  "ę": 216,
   "Ğ": 221,
-  "°": 111,
-  "C": 37,
-  "ÙĨØ": 279,
-  "ÙĩÙģØªØµØ¯": 305,
   "ļ": 251,
-  "`": 66,
-  "b": 68,
-  "į": 238,
-  "ĥ": 228,
-  "3": 22,
   "S": 53,
-  "Ø´ØµØª": 301,
-  "y": 91,
   "ø": 183,
-  "·": 118,
-  "đ": 208,
-  "ª": 106,
-  "Ī": 233,
-  "¯": 110,
-  "ÛĮØ³": 284,
-  "ä": 163,
-  "Ķ": 245,
-  "P": 50,
-  "ÛĮÚ©": 318,
-  "2": 21,
-  "#": 7,
-  "Ĵ": 243,
-  "À": 127,
-  "ď": 206,
-  "D": 38,
-  ".": 17,
-  "Ď": 205,
-  "ò": 177,
-  "È": 135,
-  "Đ": 207,
-  "Ĭ": 235,
-  "ÛĮ": 268,
-  "è": 167,
   "Ù": 152,
-  "Ùģ": 337,
-  "§": 103,
-  "ġ": 224,
-  "ĉ": 200,
-  "¾": 125,
   "ă": 194,
-  "ĳ": 242,
-  "ÙĩÙĦ": 303,
-  "R": 52,
-  "Ç": 134,
-  "«": 107,
-  "Ø²Ø¯Ùĩ": 323,
-  "$": 8,
-  "Ø²": 291,
-  "Øª": 263,
-  "B": 36,
-  "Ø§Ø±": 272,
-  ";": 30,
-  "Ħ": 229,
-  "Ì": 139,
-  "~": 96,
-  "É": 136,
-  "{": 93,
-  "ê": 169,
-  "Ø§Ùĩ": 300,
-  "L": 46,
-  "©": 105,
-  "\\": 62,
-  "Q": 51,
-  "s": 85,
-  "ÚĨÙĩÙĦ": 304,
-  "ÙĩØ´Øª": 278,
-  "Ù¾ÙĨØ¬": 290,
-  "Ø¨ÛĮØ³Øª": 297,
   "³": 114,
-  "G": 41,
-  "ÙĩÙģØªØ§Ø¯": 306,
-  "ÙĩÙ": 270,
   "a": 67,
-  "X": 58,
-  "Ø¯Ùĩ": 298,
-  "Ā": 191,
-  "J": 44,
-  "E": 39,
-  "ÚĨÙĩØ§Ø±ØµØ¯": 311,
-  "Z": 60,
-  "r": 84,
-  "ç": 166,
-  "Ø§ÙĨØ²Ø¯Ùĩ": 327,
-  "M": 47,
-  "ÙĨØ²Ø¯Ùĩ": 322,
-  "<": 31,
-  "ÙĩØ¬Ø¯Ùĩ": 326,
-  "ě": 218,
-  "}": 95,
-  "¤": 100,
-  "Þ": 157,
-  "ÙĨÙĩ": 288,
-  "¼": 123,
-  "Ò": 145,
-  ")": 13,
-  "ØµÙģØ±": 339,
-  "ÙĩØ´ØªØµØ¯": 307,
-  "O": 49,
-  "ÙĨÙĩØµØ¯": 313,
-  "z": 92,
-  "H": 42,
-  "±": 112,
-  "ØµØª": 299,
-  "Ï": 142,
-  "Ê": 137,
-  "Ä": 131,
-  "Ù¾Ø§ÙĨØ²Ø¯Ùĩ": 335,
-  "d": 70,
-  "¨ÛĮØ³Øª": 296,
-  "Ŀ": 254,
-  "ÙĨÙĪ": 294,
-  "/": 18,
-  ":": 29,
-  "<pad>": 2,
-  "Å": 132,
   "Ø§Ø²Ø¯Ùĩ": 321,
-  "+": 3,
-  "ì": 171,
-  "Ð": 143,
-  "à": 159,
-  "x": 90,
-  "Ø³": 269,
-  "Ę": 215,
-  "û": 186,
-  "q": 83,
-  "k": 77,
-  "Č": 203,
-  "<s>": 0,
-  "Ñ": 144,
-  "ÛĮØ§Ø²Ø¯Ùĩ": 329,
-  "Ø¬Ø¯Ùĩ": 325,
-  "¢": 98,
-  "¡": 97,
-  "Øµ": 262,
-  "ã": 162,
-  "Ùĩ": 260,
   "Ø³Ùĩ": 319,
-  "%": 9,
-  "Ö": 149,
-  "'": 11,
-  "*": 14,
-  "Ë": 138,
-  "ć": 198,
-  "Í": 140,
   "F": 40,
-  "â": 161,
-  "9": 28,
-  "Ý": 156,
-  "ÙĩØ´": 276,
-  "õ": 180,
-  "ē": 210,
-  "¿": 126,
   "þ": 189,
-  "m": 79,
-  "w": 89,
-  "ĕ": 212,
-  "-": 16,
-  "(": 12,
-  "á": 160,
-  "ā": 192,
-  "ĺ": 249,
-  "Ù¾": 273,
   "Ø³ÛĮØ²Ø¯Ùĩ": 331,
-  "Ú": 153,
-  "×": 150,
-  "n": 80,
-  "Ã": 130,
-  "ñ": 176,
   "Ø´Ø´ØµØ¯": 312,
-  "»": 122,
-  "l": 78,
-  "ė": 214,
-  "Ø": 151,
-  "¨": 104,
-  "ħ": 230,
-  "¶": 117,
-  ",": 15,
-  "ÚĨÙĩØ§Ø±": 285,
-  "Ĳ": 241,
-  "ĩ": 232,
-  "Ĺ": 248,
-  "0": 19,
-  "Ć": 197,
-  "ØµØ¯": 265,
-  "Ø¯ÙĪØ§Ø²Ø¯Ùĩ": 332,
-  "Ø§Ø¯": 283,
-  "í": 172,
-  "ķ": 246,
-  "÷": 182,
   "¸": 119,
-  "6": 25,
-  "İ": 239,
-  "¬": 108,
-  "?": 33,
-  "æ": 165,
-  "²": 113,
-  "ģ": 226,
-  "</s>": 1,
-  "ÙĩØ²Ø§Ø±": 293,
-  "Ĉ": 199,
-  "Ô": 147,
-  "4": 23,
-  "g": 73,
-  "@": 34,
-  "Ł": 256,
-  "e": 71,
-  "Ĝ": 219,
-  "|": 94,
-  "ÙĩÙģ": 275,
   "A": 35,
   "Î": 141,
-  "K": 45,
-  "Ø§": 264,
-  "Ù¾ÙĨØ": 289,
   "Á": 128,
-  "7": 26,
-  "t": 86,
-  "Ø§Ø": 271,
   "ÙĩÙģØ¯Ùĩ": 330,
-  "´": 115,
-  "Ģ": 225,
-  "ØµÙģ": 338
 }

 {
+  "z": 92,
+  "Ī": 233,
+  "Ĺ": 248,
+  "Ý": 156,
+  "¯": 110,
+  "ÛĮØ§Ø²Ø¯Ùĩ": 329,
+  "C": 37,
+  "Ľ": 252,
+  "Ø´ØµØª": 301,
+  "ÙĩØ´": 276,
+  "À": 127,
+  "ä": 163,
+  "Í": 140,
   "ą": 196,
+  "3": 21,
+  "D": 38,
+  "ē": 210,
+  "\\": 62,
+  "y": 91,
+  "¬": 108,
+  "ÚĨ": 274,
+  "ª": 106,
+  "±": 112,
+  "ÛĮØ³Øª": 286,
+  "Þ": 157,
   "î": 173,
+  "ć": 198,
+  "µ": 116,
+  ":": 28,
   "j": 76,
+  "h": 74,
+  "ØµÙģØ±": 339,
+  "Ø§": 264,
+  "8": 26,
+  "v": 88,
+  "Ø§Ø¯": 283,
+  "Ê": 137,
+  "Ģ": 225,
+  "¶": 117,
+  "</s>": 1,
+  "Č": 203,
+  "Ë": 138,
+  "ó": 178,
+  "P": 50,
   "]": 63,
+  "Ä": 131,
+  "4": 22,
+  "O": 49,
+  "Ø²Ø¯Ùĩ": 323,
+  "Ù¾ÙĨØ¬": 290,
+  "Ø¬Ø¯Ùĩ": 325,
+  "@": 34,
+  "*": 12,
   "å": 164,
+  "ý": 188,
+  "Øµ": 262,
+  "Å": 132,
+  "Ùĩ": 260,
+  "Z": 60,
   "¦": 102,
+  "ñ": 176,
+  "d": 70,
+  "Ĵ": 243,
+  "¢": 98,
+  "R": 52,
+  "Ā": 191,
+  "%": 7,
+  "l": 78,
+  "È": 135,
+  "ð": 175,
+  "Ĳ": 241,
+  "¿": 126,
   "Ø´": 266,
+  "U": 55,
+  "Q": 51,
+  "Ô": 147,
+  "Ø": 151,
+  "5": 23,
+  "¤": 100,
+  "&": 8,
+  "Ď": 205,
+  "à": 159,
+  "x": 90,
+  "Ì": 139,
+  "ĸ": 247,
+  "[": 61,
+  "ĳ": 242,
+  "Ø³ÛĮ": 280,
+  "Į": 237,
+  ".": 16,
+  "ÙĨÙĩØµØ¯": 313,
+  "ĝ": 220,
+  "ċ": 202,
+  "¨": 104,
+  "ı": 240,
   "ÙĩÙģØª": 277,
+  "ã": 162,
+  "ë": 170,
+  "Ñ": 144,
+  "0": 18,
+  "Ħ": 229,
+  "ě": 218,
+  "ÙĩØ´Øª": 278,
+  "Ù¾Ø§ÙĨØ²Ø¯Ùĩ": 335,
+  "í": 172,
+  "Ú": 153,
+  "Ù¾": 273,
+  "÷": 182,
+  "?": 33,
+  "ā": 192,
+  "J": 44,
+  "è": 167,
+  "ę": 216,
+  "ò": 177,
+  "ÙĨÙĩ": 288,
+  "ĺ": 249,
+  "r": 84,
+  "Ċ": 201,
+  "Ø¨ÛĮØ³Øª": 297,
+  "æ": 165,
+  "X": 58,
   "Ü": 155,
+  "e": 71,
+  "Ã": 130,
   "c": 69,
+  "ł": 257,
+  ",": 14,
+  "ğ": 222,
+  "ÙĩÙĦ": 303,
   "ī": 234,
+  "Ø³ÛĮØµØ¯": 309,
+  ";": 29,
+  "Ă": 193,
+  "Ø§Ø±": 272,
+  "Û": 154,
+  "ĩ": 232,
+  "ÚĨÙĩÙĦ": 304,
+  "Ø¯Ùĩ": 298,
+  "ķ": 246,
+  "Ð": 143,
+  "Ø§ÙĨØ²Ø¯Ùĩ": 327,
+  "ĵ": 244,
+  "Ć": 197,
+  "Ĭ": 235,
+  "Ø¯": 261,
+  "Ė": 213,
+  "â": 161,
+  "á": 160,
+  "Ùģ": 337,
+  "đ": 208,
   "Ğ": 221,
+  "K": 45,
+  "Ù¾Ø§ÙĨØµØ¯": 316,
+  "2": 20,
   "ļ": 251,
+  "7": 25,
+  "č": 204,
+  "G": 41,
+  ")": 11,
+  "ÙĩÙģØªØ§Ø¯": 306,
+  "İ": 239,
+  "1": 19,
+  "Ø¯ÙĪÛĮØ³Øª": 310,
+  "ï": 174,
+  "ß": 158,
+  "é": 168,
   "S": 53,
+  "/": 17,
+  "Ï": 142,
+  "ÙĩØ²": 292,
+  "+": 13,
+  "k": 77,
+  "ü": 187,
+  "6": 24,
+  "E": 39,
+  "ÙĨØ": 279,
+  "Ö": 149,
+  "ÙĨÙĪ": 294,
+  "»": 122,
+  "½": 124,
+  "b": 68,
+  "Ě": 217,
+  "ÙĩØ¬Ø¯Ùĩ": 326,
   "ø": 183,
+  "^": 64,
   "Ù": 152,
+  "×": 150,
+  "ØµØ¯": 265,
+  "9": 27,
+  "Ò": 145,
+  "M": 47,
+  "į": 238,
+  "(": 10,
   "ă": 194,
+  "-": 15,
   "³": 114,
+  "Ø¯ÙĪØ§Ø²Ø¯Ùĩ": 332,
+  "º": 121,
+  "õ": 180,
   "a": 67,
+  "ÙĩÙģ": 275,
+  "ÙĩØ§Ø±": 281,
+  "ÙĨØµØ¯": 302,
+  "ÙĪ": 259,
+  "p": 82,
+  "ÙĩÙ": 270,
+  "s": 85,
+  "ÚĨÙĩØ§Ø±": 285,
+  "ģ": 226,
+  "n": 80,
   "Ø§Ø²Ø¯Ùĩ": 321,
+  "¨ÛĮØ³Øª": 296,
+  "Ĕ": 211,
+  "ú": 185,
+  "ď": 206,
+  "Ø¯ÙĪ": 282,
+  "ÙĩÙģØªØµØ¯": 305,
+  "Ķ": 245,
+  "Ē": 209,
+  "ė": 214,
+  "ÙĩØ´ØªØµØ¯": 307,
+  "ŀ": 255,
+  "Ù¾Ø§": 295,
+  "~": 96,
+  "Ġ": 223,
+  "V": 56,
   "Ø³Ùĩ": 319,
+  "Ú©": 317,
+  "I": 43,
+  "ç": 166,
+  "{": 93,
+  "´": 115,
+  "²": 113,
+  "ĭ": 236,
   "F": 40,
   "þ": 189,
   "Ø³ÛĮØ²Ø¯Ùĩ": 331,
+  "B": 36,
+  ">": 32,
+  "w": 89,
+  "ÙĨÙĪØ¯": 315,
+  "Ù¾ÙĨØ¬Ø§Ùĩ": 314,
+  "Ĝ": 219,
+  "L": 46,
+  "ÿ": 190,
+  "Õ": 148,
+  "Ç": 134,
+  "$": 6,
   "Ø´Ø´ØµØ¯": 312,
+  "ĕ": 212,
+  "ÙĩØ´ØªØ§Ø¯": 308,
+  "Ø§Ø": 271,
   "¸": 119,
+  "¼": 123,
+  "ĥ": 228,
+  "Ù¾ÙĨØ": 289,
+  "=": 31,
   "A": 35,
+  "ħ": 230,
+  "ê": 169,
+  "ì": 171,
+  "|": 94,
+  "ÚĨÙĩØ§Ø±ØµØ¯": 311,
+  "²Ø¯Ùĩ": 320,
+  "}": 95,
+  "t": 86,
+  "!": 3,
+  "ġ": 224,
+  "ÛĮÚ©": 318,
+  "¥": 101,
+  "Ø´Ø´": 287,
+  "Ø³": 269,
+  "¬Ø¯Ùĩ": 324,
   "Î": 141,
+  "Ø´Ø§ÙĨØ²Ø¯Ùĩ": 328,
+  "û": 186,
+  "ù": 184,
+  "«": 107,
+  "Đ": 207,
   "Á": 128,
+  "ÙĨÙĪØ²Ø¯Ùĩ": 334,
+  "Ļ": 250,
+  "ľ": 253,
+  "i": 75,
+  "H": 42,
+  "q": 83,
+  "\"": 4,
+  "'": 9,
+  "ÙĩØ²Ø§Ø±": 293,
+  "Øª": 263,
+  "<": 30,
+  "£": 99,
+  "T": 54,
+  "Æ": 133,
+  "g": 73,
   "ÙĩÙģØ¯Ùĩ": 330,
+  "<pad>": 2,
+  "Ó": 146,
+  "Y": 59,
+  "·": 118,
+  "Â": 129,
+  "§": 103,
+  "¹": 120,
+  "_": 65,
+  "©": 105,
+  "Ń": 258,
+  "o": 81,
+  "ĉ": 200,
+  "Ĩ": 231,
+  "N": 48,
+  "Ĥ": 227,
+  "Ĉ": 199,
+  "m": 79,
+  "ÛĮØ³": 284,
+  "ÚĨÙĩØ§Ø±Ø¯Ùĩ": 333,
+  "`": 66,
+  "f": 72,
+  "Ł": 256,
+  "ØµØª": 299,
+  "ÙĨ": 267,
+  "ÛĮ": 268,
+  "Ŀ": 254,
+  "ÙĨØ²Ø¯Ùĩ": 322,
+  "¾": 125,
+  "Ø²": 291,
+  "ô": 179,
+  "#": 5,
+  "u": 87,
+  "É": 136,
+  "¡": 97,
+  "W": 57,
+  "Ą": 195,
+  "®": 109,
+  "°": 111,
+  "Ø±": 336,
+  "<s>": 0,
+  "Ę": 215,
+  "ØµÙģ": 338,
+  "ö": 181,
+  "Ø§Ùĩ": 300
 }