Upload tokenizer

added_tokens.json

@@ -1,17 +1,18 @@
 {
-  "<ADULT>": 50273,
-  "<BLACK>": 50267,
+  "<ADULT>": 50276,
+  "<ASIAN>": 50268,
+  "<BLACK>": 50272,
   "<CHILD>": 50274,
-  "<HISPANIC>": 50278,
-  "<MAN>": 50279,
-  "<MIDDLE_AGED>": 50272,
-  "<NATIVE_AMERICAN>": 50268,
-  "<NON_BINARY>": 50269,
-  "<PACIFIC_ISLANDER>": 50271,
+  "<HISPANIC>": 50277,
+  "<MAN>": 50267,
+  "<MIDDLE_AGED>": 50275,
+  "<NATIVE_AMERICAN>": 50273,
+  "<NON_BINARY>": 50280,
+  "<PACIFIC_ISLANDER>": 50269,
   "<PERT_SEP>": 50266,
-  "<SENIOR>": 50270,
+  "<SENIOR>": 50278,
   "<SEP>": 50265,
-  "<WHITE>": 50277,
-  "<WOMAN>": 50275,
-  "<YOUNG>": 50276
+  "<WHITE>": 50270,
+  "<WOMAN>": 50271,
+  "<YOUNG>": 50279
 }

tokenizer.json

@@ -68,7 +68,7 @@
     },
     {
       "id": 50267,
-      "content": "<BLACK>",
+      "content": "<MAN>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -77,7 +77,7 @@
     },
     {
       "id": 50268,
-      "content": "<NATIVE_AMERICAN>",
+      "content": "<ASIAN>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -86,7 +86,7 @@
     },
     {
       "id": 50269,
-      "content": "<NON_BINARY>",
+      "content": "<PACIFIC_ISLANDER>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 50270,
-      "content": "<SENIOR>",
+      "content": "<WHITE>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 50271,
-      "content": "<PACIFIC_ISLANDER>",
+      "content": "<WOMAN>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
     },
     {
       "id": 50272,
-      "content": "<MIDDLE_AGED>",
+      "content": "<BLACK>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
     },
     {
       "id": 50273,
-      "content": "<ADULT>",
+      "content": "<NATIVE_AMERICAN>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -140,7 +140,7 @@
     },
     {
       "id": 50275,
-      "content": "<WOMAN>",
+      "content": "<MIDDLE_AGED>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -149,7 +149,7 @@
     },
     {
       "id": 50276,
-      "content": "<YOUNG>",
+      "content": "<ADULT>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -158,7 +158,7 @@
     },
     {
       "id": 50277,
-      "content": "<WHITE>",
+      "content": "<HISPANIC>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -167,7 +167,7 @@
     },
     {
       "id": 50278,
-      "content": "<HISPANIC>",
+      "content": "<SENIOR>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -176,7 +176,16 @@
     },
     {
       "id": 50279,
-      "content": "<MAN>",
+      "content": "<YOUNG>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50280,
+      "content": "<NON_BINARY>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,

tokenizer_config.json

@@ -58,7 +58,7 @@
       "special": true
     },
     "50267": {
-      "content": "<BLACK>",
+      "content": "<MAN>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -66,7 +66,7 @@
       "special": true
     },
     "50268": {
-      "content": "<NATIVE_AMERICAN>",
+      "content": "<ASIAN>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -74,7 +74,7 @@
       "special": true
     },
     "50269": {
-      "content": "<NON_BINARY>",
+      "content": "<PACIFIC_ISLANDER>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -82,7 +82,7 @@
       "special": true
     },
     "50270": {
-      "content": "<SENIOR>",
+      "content": "<WHITE>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -90,7 +90,7 @@
       "special": true
     },
     "50271": {
-      "content": "<PACIFIC_ISLANDER>",
+      "content": "<WOMAN>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -98,7 +98,7 @@
       "special": true
     },
     "50272": {
-      "content": "<MIDDLE_AGED>",
+      "content": "<BLACK>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -106,7 +106,7 @@
       "special": true
     },
     "50273": {
-      "content": "<ADULT>",
+      "content": "<NATIVE_AMERICAN>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
       "special": true
     },
     "50275": {
-      "content": "<WOMAN>",
+      "content": "<MIDDLE_AGED>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -130,7 +130,7 @@
       "special": true
     },
     "50276": {
-      "content": "<YOUNG>",
+      "content": "<ADULT>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -138,7 +138,7 @@
       "special": true
     },
     "50277": {
-      "content": "<WHITE>",
+      "content": "<HISPANIC>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -146,7 +146,7 @@
       "special": true
     },
     "50278": {
-      "content": "<HISPANIC>",
+      "content": "<SENIOR>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -154,7 +154,15 @@
       "special": true
     },
     "50279": {
-      "content": "<MAN>",
+      "content": "<YOUNG>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50280": {
+      "content": "<NON_BINARY>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,