Upload tokenizer

Browse files

Files changed (3) hide show

special_tokens_map.json +9 -0
tokenizer.json +316 -0
tokenizer_config.json +5 -0

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+  "bos_token": "<cls>",
+  "cls_token": "<cls>",
+  "eos_token": "<sep>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "<sep>",
+  "unk_token": "<unk>"
+}

tokenizer.json ADDED Viewed

	@@ -0,0 +1,316 @@

+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<cls>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<sep>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "<mask>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "<cls>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "<cls>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "<cls>": {
+        "id": "<cls>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "<cls>"
+        ]
+      },
+      "<sep>": {
+        "id": "<sep>",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "<sep>"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "BPEDecoder",
+    "suffix": "</w>"
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "vocab": {
+      "<pad>": 0,
+      "<cls>": 1,
+      "<sep>": 2,
+      "<unk>": 3,
+      "<mask>": 4,
+      "A": 5,
+      "B": 6,
+      "C": 7,
+      "D": 8,
+      "E": 9,
+      "F": 10,
+      "G": 11,
+      "H": 12,
+      "I": 13,
+      "K": 14,
+      "L": 15,
+      "M": 16,
+      "N": 17,
+      "O": 18,
+      "P": 19,
+      "Q": 20,
+      "R": 21,
+      "S": 22,
+      "T": 23,
+      "U": 24,
+      "V": 25,
+      "W": 26,
+      "X": 27,
+      "Y": 28,
+      "Z": 29,
+      "AA": 30,
+      "LL": 31,
+      "SS": 32,
+      "LA": 33,
+      "LS": 34,
+      "GG": 35,
+      "LV": 36,
+      "LR": 37,
+      "LE": 38,
+      "LG": 39,
+      "LT": 40,
+      "AV": 41,
+      "AG": 42,
+      "LP": 43,
+      "LD": 44,
+      "LI": 45,
+      "AS": 46,
+      "EE": 47,
+      "RR": 48,
+      "LK": 49,
+      "VV": 50,
+      "AT": 51,
+      "AE": 52,
+      "SG": 53,
+      "DG": 54,
+      "AR": 55,
+      "KK": 56,
+      "AI": 57,
+      "AP": 58,
+      "TT": 59,
+      "LN": 60,
+      "LF": 61,
+      "LQ": 62,
+      "AD": 63,
+      "SV": 64,
+      "ST": 65,
+      "SI": 66,
+      "SP": 67,
+      "DE": 68,
+      "TG": 69,
+      "TV": 70,
+      "SR": 71,
+      "SE": 72,
+      "DV": 73,
+      "PP": 74,
+      "KE": 75,
+      "KI": 76,
+      "DI": 77,
+      "GV": 78,
+      "AQ": 79,
+      "AF": 80,
+      "RV": 81,
+      "TI": 82,
+      "NI": 83,
+      "PV": 84,
+      "RE": 85,
+      "SF": 86,
+      "KV": 87,
+      "NG": 88,
+      "LY": 89,
+      "RI": 90,
+      "PG": 91,
+      "RG": 92,
+      "DD": 93,
+      "PE": 94,
+      "AK": 95,
+      "NV": 96,
+      "SD": 97,
+      "TE": 98,
+      "QQ": 99
+    },
+    "merges": [
+      "A A",
+      "L L",
+      "S S",
+      "L A",
+      "L S",
+      "G G",
+      "L V",
+      "L R",
+      "L E",
+      "L G",
+      "L T",
+      "A V",
+      "A G",
+      "L P",
+      "L D",
+      "L I",
+      "A S",
+      "E E",
+      "R R",
+      "L K",
+      "V V",
+      "A T",
+      "A E",
+      "S G",
+      "D G",
+      "A R",
+      "K K",
+      "A I",
+      "A P",
+      "T T",
+      "L N",
+      "L F",
+      "L Q",
+      "A D",
+      "S V",
+      "S T",
+      "S I",
+      "S P",
+      "D E",
+      "T G",
+      "T V",
+      "S R",
+      "S E",
+      "D V",
+      "P P",
+      "K E",
+      "K I",
+      "D I",
+      "G V",
+      "A Q",
+      "A F",
+      "R V",
+      "T I",
+      "N I",
+      "P V",
+      "R E",
+      "S F",
+      "K V",
+      "N G",
+      "L Y",
+      "R I",
+      "P G",
+      "R G",
+      "D D",
+      "P E",
+      "A K",
+      "N V",
+      "S D",
+      "T E",
+      "Q Q"
+    ]
+  }
+}

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,5 @@

+{
+  "clean_up_tokenization_spaces": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}