Upload tokenizer

added_tokens.json

@@ -0,0 +1,106 @@
+{
+  "<|NLG|>": 50260,
+  "<|NLU|>": 50259,
+  "<|beginoftext|>": 50257,
+  "<|extra_id_0|>": 50261,
+  "<|extra_id_10|>": 50271,
+  "<|extra_id_11|>": 50272,
+  "<|extra_id_12|>": 50273,
+  "<|extra_id_13|>": 50274,
+  "<|extra_id_14|>": 50275,
+  "<|extra_id_15|>": 50276,
+  "<|extra_id_16|>": 50277,
+  "<|extra_id_17|>": 50278,
+  "<|extra_id_18|>": 50279,
+  "<|extra_id_19|>": 50280,
+  "<|extra_id_1|>": 50262,
+  "<|extra_id_20|>": 50281,
+  "<|extra_id_21|>": 50282,
+  "<|extra_id_22|>": 50283,
+  "<|extra_id_23|>": 50284,
+  "<|extra_id_24|>": 50285,
+  "<|extra_id_25|>": 50286,
+  "<|extra_id_26|>": 50287,
+  "<|extra_id_27|>": 50288,
+  "<|extra_id_28|>": 50289,
+  "<|extra_id_29|>": 50290,
+  "<|extra_id_2|>": 50263,
+  "<|extra_id_30|>": 50291,
+  "<|extra_id_31|>": 50292,
+  "<|extra_id_32|>": 50293,
+  "<|extra_id_33|>": 50294,
+  "<|extra_id_34|>": 50295,
+  "<|extra_id_35|>": 50296,
+  "<|extra_id_36|>": 50297,
+  "<|extra_id_37|>": 50298,
+  "<|extra_id_38|>": 50299,
+  "<|extra_id_39|>": 50300,
+  "<|extra_id_3|>": 50264,
+  "<|extra_id_40|>": 50301,
+  "<|extra_id_41|>": 50302,
+  "<|extra_id_42|>": 50303,
+  "<|extra_id_43|>": 50304,
+  "<|extra_id_44|>": 50305,
+  "<|extra_id_45|>": 50306,
+  "<|extra_id_46|>": 50307,
+  "<|extra_id_47|>": 50308,
+  "<|extra_id_48|>": 50309,
+  "<|extra_id_49|>": 50310,
+  "<|extra_id_4|>": 50265,
+  "<|extra_id_50|>": 50311,
+  "<|extra_id_51|>": 50312,
+  "<|extra_id_52|>": 50313,
+  "<|extra_id_53|>": 50314,
+  "<|extra_id_54|>": 50315,
+  "<|extra_id_55|>": 50316,
+  "<|extra_id_56|>": 50317,
+  "<|extra_id_57|>": 50318,
+  "<|extra_id_58|>": 50319,
+  "<|extra_id_59|>": 50320,
+  "<|extra_id_5|>": 50266,
+  "<|extra_id_60|>": 50321,
+  "<|extra_id_61|>": 50322,
+  "<|extra_id_62|>": 50323,
+  "<|extra_id_63|>": 50324,
+  "<|extra_id_64|>": 50325,
+  "<|extra_id_65|>": 50326,
+  "<|extra_id_66|>": 50327,
+  "<|extra_id_67|>": 50328,
+  "<|extra_id_68|>": 50329,
+  "<|extra_id_69|>": 50330,
+  "<|extra_id_6|>": 50267,
+  "<|extra_id_70|>": 50331,
+  "<|extra_id_71|>": 50332,
+  "<|extra_id_72|>": 50333,
+  "<|extra_id_73|>": 50334,
+  "<|extra_id_74|>": 50335,
+  "<|extra_id_75|>": 50336,
+  "<|extra_id_76|>": 50337,
+  "<|extra_id_77|>": 50338,
+  "<|extra_id_78|>": 50339,
+  "<|extra_id_79|>": 50340,
+  "<|extra_id_7|>": 50268,
+  "<|extra_id_80|>": 50341,
+  "<|extra_id_81|>": 50342,
+  "<|extra_id_82|>": 50343,
+  "<|extra_id_83|>": 50344,
+  "<|extra_id_84|>": 50345,
+  "<|extra_id_85|>": 50346,
+  "<|extra_id_86|>": 50347,
+  "<|extra_id_87|>": 50348,
+  "<|extra_id_88|>": 50349,
+  "<|extra_id_89|>": 50350,
+  "<|extra_id_8|>": 50269,
+  "<|extra_id_90|>": 50351,
+  "<|extra_id_91|>": 50352,
+  "<|extra_id_92|>": 50353,
+  "<|extra_id_93|>": 50354,
+  "<|extra_id_94|>": 50355,
+  "<|extra_id_95|>": 50356,
+  "<|extra_id_96|>": 50357,
+  "<|extra_id_97|>": 50358,
+  "<|extra_id_98|>": 50359,
+  "<|extra_id_99|>": 50360,
+  "<|extra_id_9|>": 50270,
+  "<|unk|>": 50258
+}

special_tokens_map.json

@@ -0,0 +1,110 @@
+{
+  "additional_special_tokens": [
+    "<|NLU|>",
+    "<|NLG|>",
+    "<|extra_id_0|>",
+    "<|extra_id_1|>",
+    "<|extra_id_2|>",
+    "<|extra_id_3|>",
+    "<|extra_id_4|>",
+    "<|extra_id_5|>",
+    "<|extra_id_6|>",
+    "<|extra_id_7|>",
+    "<|extra_id_8|>",
+    "<|extra_id_9|>",
+    "<|extra_id_10|>",
+    "<|extra_id_11|>",
+    "<|extra_id_12|>",
+    "<|extra_id_13|>",
+    "<|extra_id_14|>",
+    "<|extra_id_15|>",
+    "<|extra_id_16|>",
+    "<|extra_id_17|>",
+    "<|extra_id_18|>",
+    "<|extra_id_19|>",
+    "<|extra_id_20|>",
+    "<|extra_id_21|>",
+    "<|extra_id_22|>",
+    "<|extra_id_23|>",
+    "<|extra_id_24|>",
+    "<|extra_id_25|>",
+    "<|extra_id_26|>",
+    "<|extra_id_27|>",
+    "<|extra_id_28|>",
+    "<|extra_id_29|>",
+    "<|extra_id_30|>",
+    "<|extra_id_31|>",
+    "<|extra_id_32|>",
+    "<|extra_id_33|>",
+    "<|extra_id_34|>",
+    "<|extra_id_35|>",
+    "<|extra_id_36|>",
+    "<|extra_id_37|>",
+    "<|extra_id_38|>",
+    "<|extra_id_39|>",
+    "<|extra_id_40|>",
+    "<|extra_id_41|>",
+    "<|extra_id_42|>",
+    "<|extra_id_43|>",
+    "<|extra_id_44|>",
+    "<|extra_id_45|>",
+    "<|extra_id_46|>",
+    "<|extra_id_47|>",
+    "<|extra_id_48|>",
+    "<|extra_id_49|>",
+    "<|extra_id_50|>",
+    "<|extra_id_51|>",
+    "<|extra_id_52|>",
+    "<|extra_id_53|>",
+    "<|extra_id_54|>",
+    "<|extra_id_55|>",
+    "<|extra_id_56|>",
+    "<|extra_id_57|>",
+    "<|extra_id_58|>",
+    "<|extra_id_59|>",
+    "<|extra_id_60|>",
+    "<|extra_id_61|>",
+    "<|extra_id_62|>",
+    "<|extra_id_63|>",
+    "<|extra_id_64|>",
+    "<|extra_id_65|>",
+    "<|extra_id_66|>",
+    "<|extra_id_67|>",
+    "<|extra_id_68|>",
+    "<|extra_id_69|>",
+    "<|extra_id_70|>",
+    "<|extra_id_71|>",
+    "<|extra_id_72|>",
+    "<|extra_id_73|>",
+    "<|extra_id_74|>",
+    "<|extra_id_75|>",
+    "<|extra_id_76|>",
+    "<|extra_id_77|>",
+    "<|extra_id_78|>",
+    "<|extra_id_79|>",
+    "<|extra_id_80|>",
+    "<|extra_id_81|>",
+    "<|extra_id_82|>",
+    "<|extra_id_83|>",
+    "<|extra_id_84|>",
+    "<|extra_id_85|>",
+    "<|extra_id_86|>",
+    "<|extra_id_87|>",
+    "<|extra_id_88|>",
+    "<|extra_id_89|>",
+    "<|extra_id_90|>",
+    "<|extra_id_91|>",
+    "<|extra_id_92|>",
+    "<|extra_id_93|>",
+    "<|extra_id_94|>",
+    "<|extra_id_95|>",
+    "<|extra_id_96|>",
+    "<|extra_id_97|>",
+    "<|extra_id_98|>",
+    "<|extra_id_99|>"
+  ],
+  "bos_token": "<|beginoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|pad|>",
+  "unk_token": "<|unk|>"
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "add_prefix_space": false,
+  "additional_special_tokens": [
+    "<|pad|>",
+    "<|endoftext|>"
+  ],
+  "bos_token": "<|pad|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|pad|>",
+  "model_max_length": 1024,
+  "special_tokens_map_file": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|pad|>"
+}