Add tokenizer

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,425 @@
+{
+  "version": "1.0",
+  "truncation": {
+    "direction": "Right",
+    "max_length": 256,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": {
+    "strategy": {
+      "Fixed": 256
+    },
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 0,
+    "pad_type_id": 0,
+    "pad_token": "[PAD]"
+  },
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "BertNormalizer",
+    "clean_text": true,
+    "handle_chinese_chars": true,
+    "strip_accents": null,
+    "lowercase": true
+  },
+  "pre_tokenizer": {
+    "type": "BertPreTokenizer"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[CLS]": {
+        "id": "[CLS]",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "[CLS]"
+        ]
+      },
+      "[SEP]": {
+        "id": "[SEP]",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "[SEP]"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": true
+  },
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 100,
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[CLS]": 2,
+      "[SEP]": 3,
+      "[MASK]": 4,
+      "0": 5,
+      "1": 6,
+      "2": 7,
+      "3": 8,
+      "4": 9,
+      "5": 10,
+      "6": 11,
+      "7": 12,
+      "8": 13,
+      "9": 14,
+      "10": 15,
+      "11": 16,
+      "12": 17,
+      "13": 18,
+      "14": 19,
+      "15": 20,
+      "16": 21,
+      "17": 22,
+      "18": 23,
+      "19": 24,
+      "20": 25,
+      "21": 26,
+      "22": 27,
+      "23": 28,
+      "24": 29,
+      "25": 30,
+      "26": 31,
+      "27": 32,
+      "28": 33,
+      "29": 34,
+      "30": 35,
+      "31": 36,
+      "32": 37,
+      "33": 38,
+      "34": 39,
+      "35": 40,
+      "36": 41,
+      "37": 42,
+      "38": 43,
+      "39": 44,
+      "40": 45,
+      "41": 46,
+      "42": 47,
+      "43": 48,
+      "44": 49,
+      "45": 50,
+      "46": 51,
+      "47": 52,
+      "48": 53,
+      "49": 54,
+      "50": 55,
+      "51": 56,
+      "52": 57,
+      "53": 58,
+      "54": 59,
+      "55": 60,
+      "56": 61,
+      "57": 62,
+      "58": 63,
+      "59": 64,
+      "60": 65,
+      "61": 66,
+      "62": 67,
+      "63": 68,
+      "64": 69,
+      "65": 70,
+      "66": 71,
+      "67": 72,
+      "68": 73,
+      "69": 74,
+      "70": 75,
+      "71": 76,
+      "72": 77,
+      "73": 78,
+      "74": 79,
+      "75": 80,
+      "76": 81,
+      "77": 82,
+      "78": 83,
+      "79": 84,
+      "80": 85,
+      "81": 86,
+      "82": 87,
+      "83": 88,
+      "84": 89,
+      "85": 90,
+      "86": 91,
+      "87": 92,
+      "88": 93,
+      "89": 94,
+      "90": 95,
+      "91": 96,
+      "92": 97,
+      "93": 98,
+      "94": 99,
+      "95": 100,
+      "96": 101,
+      "97": 102,
+      "98": 103,
+      "99": 104,
+      "100": 105,
+      "101": 106,
+      "102": 107,
+      "103": 108,
+      "104": 109,
+      "105": 110,
+      "106": 111,
+      "107": 112,
+      "108": 113,
+      "109": 114,
+      "110": 115,
+      "111": 116,
+      "112": 117,
+      "113": 118,
+      "114": 119,
+      "115": 120,
+      "116": 121,
+      "117": 122,
+      "118": 123,
+      "119": 124,
+      "120": 125,
+      "121": 126,
+      "122": 127,
+      "123": 128,
+      "124": 129,
+      "125": 130,
+      "126": 131,
+      "127": 132,
+      "128": 133,
+      "129": 134,
+      "130": 135,
+      "131": 136,
+      "132": 137,
+      "133": 138,
+      "134": 139,
+      "135": 140,
+      "136": 141,
+      "137": 142,
+      "138": 143,
+      "139": 144,
+      "140": 145,
+      "141": 146,
+      "142": 147,
+      "143": 148,
+      "144": 149,
+      "145": 150,
+      "146": 151,
+      "147": 152,
+      "148": 153,
+      "149": 154,
+      "150": 155,
+      "151": 156,
+      "152": 157,
+      "153": 158,
+      "154": 159,
+      "155": 160,
+      "156": 161,
+      "157": 162,
+      "158": 163,
+      "159": 164,
+      "160": 165,
+      "161": 166,
+      "162": 167,
+      "163": 168,
+      "164": 169,
+      "165": 170,
+      "166": 171,
+      "167": 172,
+      "168": 173,
+      "169": 174,
+      "170": 175,
+      "171": 176,
+      "172": 177,
+      "173": 178,
+      "174": 179,
+      "175": 180,
+      "176": 181,
+      "177": 182,
+      "178": 183,
+      "179": 184,
+      "180": 185,
+      "181": 186,
+      "182": 187,
+      "183": 188,
+      "184": 189,
+      "185": 190,
+      "186": 191,
+      "187": 192,
+      "188": 193,
+      "189": 194,
+      "190": 195,
+      "191": 196,
+      "192": 197,
+      "193": 198,
+      "194": 199,
+      "195": 200,
+      "196": 201,
+      "197": 202,
+      "198": 203,
+      "199": 204,
+      "200": 205,
+      "201": 206,
+      "202": 207,
+      "203": 208,
+      "204": 209,
+      "205": 210,
+      "206": 211,
+      "207": 212,
+      "208": 213,
+      "209": 214,
+      "210": 215,
+      "211": 216,
+      "212": 217,
+      "213": 218,
+      "214": 219,
+      "215": 220,
+      "216": 221,
+      "217": 222,
+      "218": 223,
+      "219": 224,
+      "220": 225,
+      "221": 226,
+      "222": 227,
+      "223": 228,
+      "224": 229,
+      "225": 230,
+      "226": 231,
+      "227": 232,
+      "228": 233,
+      "229": 234,
+      "230": 235,
+      "231": 236,
+      "232": 237,
+      "233": 238,
+      "234": 239,
+      "235": 240,
+      "236": 241,
+      "237": 242,
+      "238": 243,
+      "239": 244,
+      "240": 245,
+      "241": 246,
+      "242": 247,
+      "243": 248,
+      "244": 249,
+      "245": 250,
+      "246": 251,
+      "247": 252,
+      "248": 253,
+      "249": 254,
+      "250": 255,
+      "251": 256,
+      "252": 257,
+      "253": 258,
+      "254": 259,
+      "255": 260
+    }
+  }
+}

tokenizer_config.json

@@ -0,0 +1,66 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "max_len": 256,
+  "max_length": 256,
+  "model_max_length": 256,
+  "never_split": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}

vocab.txt

@@ -0,0 +1,261 @@
+[PAD]
+[UNK]
+[CLS]
+[SEP]
+[MASK]
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255