v3

README.md

@@ -1,9 +1,9 @@
 ---
+language:
+- en
 license: apache-2.0
 datasets:
 - BEE-spoke-data/bees-internal
-language:
-- en
 ---
 
 # BeeTokenizer

tokenizer_config.json

@@ -64,7 +64,7 @@
       "single_word": false,
       "special": true
     },
-    "32100": {
+    "32072": {
       "content": "  ",
       "lstrip": false,
       "normalized": true,
@@ -72,7 +72,7 @@
       "single_word": false,
       "special": false
     },
-    "32101": {
+    "32073": {
       "content": "   ",
       "lstrip": false,
       "normalized": true,
@@ -80,7 +80,7 @@
       "single_word": false,
       "special": false
     },
-    "32102": {
+    "32074": {
       "content": "    ",
       "lstrip": false,
       "normalized": true,
@@ -88,7 +88,7 @@
       "single_word": false,
       "special": false
     },
-    "32103": {
+    "32075": {
       "content": "     ",
       "lstrip": false,
       "normalized": true,
@@ -96,7 +96,7 @@
       "single_word": false,
       "special": false
     },
-    "32104": {
+    "32076": {
       "content": "      ",
       "lstrip": false,
       "normalized": true,
@@ -104,7 +104,7 @@
       "single_word": false,
       "special": false
     },
-    "32105": {
+    "32077": {
       "content": "       ",
       "lstrip": false,
       "normalized": true,
@@ -112,7 +112,7 @@
       "single_word": false,
       "special": false
     },
-    "32106": {
+    "32078": {
       "content": "        ",
       "lstrip": false,
       "normalized": true,
@@ -120,7 +120,7 @@
       "single_word": false,
       "special": false
     },
-    "32107": {
+    "32079": {
       "content": "         ",
       "lstrip": false,
       "normalized": true,
@@ -128,7 +128,7 @@
       "single_word": false,
       "special": false
     },
-    "32108": {
+    "32080": {
       "content": "          ",
       "lstrip": false,
       "normalized": true,
@@ -136,7 +136,7 @@
       "single_word": false,
       "special": false
     },
-    "32109": {
+    "32081": {
       "content": "           ",
       "lstrip": false,
       "normalized": true,
@@ -144,7 +144,7 @@
       "single_word": false,
       "special": false
     },
-    "32110": {
+    "32082": {
       "content": "            ",
       "lstrip": false,
       "normalized": true,
@@ -152,7 +152,7 @@
       "single_word": false,
       "special": false
     },
-    "32111": {
+    "32083": {
       "content": "             ",
       "lstrip": false,
       "normalized": true,
@@ -160,7 +160,7 @@
       "single_word": false,
       "special": false
     },
-    "32112": {
+    "32084": {
       "content": "              ",
       "lstrip": false,
       "normalized": true,
@@ -168,7 +168,7 @@
       "single_word": false,
       "special": false
     },
-    "32113": {
+    "32085": {
       "content": "               ",
       "lstrip": false,
       "normalized": true,
@@ -176,7 +176,7 @@
       "single_word": false,
       "special": false
     },
-    "32114": {
+    "32086": {
       "content": "                ",
       "lstrip": false,
       "normalized": true,
@@ -184,7 +184,7 @@
       "single_word": false,
       "special": false
     },
-    "32115": {
+    "32087": {
       "content": "                 ",
       "lstrip": false,
       "normalized": true,
@@ -192,7 +192,7 @@
       "single_word": false,
       "special": false
     },
-    "32116": {
+    "32088": {
       "content": "                  ",
       "lstrip": false,
       "normalized": true,
@@ -200,7 +200,7 @@
       "single_word": false,
       "special": false
     },
-    "32117": {
+    "32089": {
       "content": "                   ",
       "lstrip": false,
       "normalized": true,
@@ -208,7 +208,7 @@
       "single_word": false,
       "special": false
     },
-    "32118": {
+    "32090": {
       "content": "                    ",
       "lstrip": false,
       "normalized": true,
@@ -216,7 +216,7 @@
       "single_word": false,
       "special": false
     },
-    "32119": {
+    "32091": {
       "content": "                     ",
       "lstrip": false,
       "normalized": true,
@@ -224,7 +224,7 @@
       "single_word": false,
       "special": false
     },
-    "32120": {
+    "32092": {
       "content": "                      ",
       "lstrip": false,
       "normalized": true,
@@ -232,7 +232,7 @@
       "single_word": false,
       "special": false
     },
-    "32121": {
+    "32093": {
       "content": "                       ",
       "lstrip": false,
       "normalized": true,
@@ -240,7 +240,7 @@
       "single_word": false,
       "special": false
     },
-    "32122": {
+    "32094": {
       "content": "                        ",
       "lstrip": false,
       "normalized": true,
@@ -248,7 +248,7 @@
       "single_word": false,
       "special": false
     },
-    "32123": {
+    "32095": {
       "content": "                         ",
       "lstrip": false,
       "normalized": true,
@@ -256,7 +256,7 @@
       "single_word": false,
       "special": false
     },
-    "32124": {
+    "32096": {
       "content": "                          ",
       "lstrip": false,
       "normalized": true,
@@ -264,7 +264,7 @@
       "single_word": false,
       "special": false
     },
-    "32125": {
+    "32097": {
       "content": "                           ",
       "lstrip": false,
       "normalized": true,
@@ -272,7 +272,7 @@
       "single_word": false,
       "special": false
     },
-    "32126": {
+    "32098": {
       "content": "                            ",
       "lstrip": false,
       "normalized": true,
@@ -280,7 +280,7 @@
       "single_word": false,
       "special": false
     },
-    "32127": {
+    "32099": {
       "content": "                             ",
       "lstrip": false,
       "normalized": true,