Upload tokenizer

added_tokens.json

@@ -0,0 +1,502 @@
+{
+  "CLASS_0": 30922,
+  "CLASS_1": 30923,
+  "CLASS_10": 30932,
+  "CLASS_11": 30933,
+  "CLASS_12": 30934,
+  "CLASS_13": 30935,
+  "CLASS_14": 30936,
+  "CLASS_15": 30937,
+  "CLASS_16": 30938,
+  "CLASS_17": 30939,
+  "CLASS_18": 30940,
+  "CLASS_19": 30941,
+  "CLASS_2": 30924,
+  "CLASS_20": 30942,
+  "CLASS_21": 30943,
+  "CLASS_22": 30944,
+  "CLASS_23": 30945,
+  "CLASS_24": 30946,
+  "CLASS_25": 30947,
+  "CLASS_26": 30948,
+  "CLASS_27": 30949,
+  "CLASS_28": 30950,
+  "CLASS_29": 30951,
+  "CLASS_3": 30925,
+  "CLASS_30": 30952,
+  "CLASS_31": 30953,
+  "CLASS_32": 30954,
+  "CLASS_33": 30955,
+  "CLASS_34": 30956,
+  "CLASS_35": 30957,
+  "CLASS_36": 30958,
+  "CLASS_37": 30959,
+  "CLASS_38": 30960,
+  "CLASS_39": 30961,
+  "CLASS_4": 30926,
+  "CLASS_40": 30962,
+  "CLASS_41": 30963,
+  "CLASS_42": 30964,
+  "CLASS_43": 30965,
+  "CLASS_44": 30966,
+  "CLASS_45": 30967,
+  "CLASS_46": 30968,
+  "CLASS_47": 30969,
+  "CLASS_48": 30970,
+  "CLASS_49": 30971,
+  "CLASS_5": 30927,
+  "CLASS_50": 30972,
+  "CLASS_51": 30973,
+  "CLASS_52": 30974,
+  "CLASS_53": 30975,
+  "CLASS_54": 30976,
+  "CLASS_55": 30977,
+  "CLASS_56": 30978,
+  "CLASS_57": 30979,
+  "CLASS_58": 30980,
+  "CLASS_59": 30981,
+  "CLASS_6": 30928,
+  "CLASS_60": 30982,
+  "CLASS_61": 30983,
+  "CLASS_62": 30984,
+  "CLASS_63": 30985,
+  "CLASS_64": 30986,
+  "CLASS_65": 30987,
+  "CLASS_66": 30988,
+  "CLASS_67": 30989,
+  "CLASS_68": 30990,
+  "CLASS_69": 30991,
+  "CLASS_7": 30929,
+  "CLASS_70": 30992,
+  "CLASS_71": 30993,
+  "CLASS_72": 30994,
+  "CLASS_73": 30995,
+  "CLASS_74": 30996,
+  "CLASS_75": 30997,
+  "CLASS_76": 30998,
+  "CLASS_77": 30999,
+  "CLASS_78": 31000,
+  "CLASS_79": 31001,
+  "CLASS_8": 30930,
+  "CLASS_80": 31002,
+  "CLASS_81": 31003,
+  "CLASS_82": 31004,
+  "CLASS_83": 31005,
+  "CLASS_84": 31006,
+  "CLASS_85": 31007,
+  "CLASS_86": 31008,
+  "CLASS_87": 31009,
+  "CLASS_88": 31010,
+  "CLASS_89": 31011,
+  "CLASS_9": 30931,
+  "CLASS_90": 31012,
+  "CLASS_91": 31013,
+  "CLASS_92": 31014,
+  "CLASS_93": 31015,
+  "CLASS_94": 31016,
+  "CLASS_95": 31017,
+  "CLASS_96": 31018,
+  "CLASS_97": 31019,
+  "CLASS_98": 31020,
+  "CLASS_99": 31021,
+  "FUNC_0": 30822,
+  "FUNC_1": 30823,
+  "FUNC_10": 30832,
+  "FUNC_11": 30833,
+  "FUNC_12": 30834,
+  "FUNC_13": 30835,
+  "FUNC_14": 30836,
+  "FUNC_15": 30837,
+  "FUNC_16": 30838,
+  "FUNC_17": 30839,
+  "FUNC_18": 30840,
+  "FUNC_19": 30841,
+  "FUNC_2": 30824,
+  "FUNC_20": 30842,
+  "FUNC_21": 30843,
+  "FUNC_22": 30844,
+  "FUNC_23": 30845,
+  "FUNC_24": 30846,
+  "FUNC_25": 30847,
+  "FUNC_26": 30848,
+  "FUNC_27": 30849,
+  "FUNC_28": 30850,
+  "FUNC_29": 30851,
+  "FUNC_3": 30825,
+  "FUNC_30": 30852,
+  "FUNC_31": 30853,
+  "FUNC_32": 30854,
+  "FUNC_33": 30855,
+  "FUNC_34": 30856,
+  "FUNC_35": 30857,
+  "FUNC_36": 30858,
+  "FUNC_37": 30859,
+  "FUNC_38": 30860,
+  "FUNC_39": 30861,
+  "FUNC_4": 30826,
+  "FUNC_40": 30862,
+  "FUNC_41": 30863,
+  "FUNC_42": 30864,
+  "FUNC_43": 30865,
+  "FUNC_44": 30866,
+  "FUNC_45": 30867,
+  "FUNC_46": 30868,
+  "FUNC_47": 30869,
+  "FUNC_48": 30870,
+  "FUNC_49": 30871,
+  "FUNC_5": 30827,
+  "FUNC_50": 30872,
+  "FUNC_51": 30873,
+  "FUNC_52": 30874,
+  "FUNC_53": 30875,
+  "FUNC_54": 30876,
+  "FUNC_55": 30877,
+  "FUNC_56": 30878,
+  "FUNC_57": 30879,
+  "FUNC_58": 30880,
+  "FUNC_59": 30881,
+  "FUNC_6": 30828,
+  "FUNC_60": 30882,
+  "FUNC_61": 30883,
+  "FUNC_62": 30884,
+  "FUNC_63": 30885,
+  "FUNC_64": 30886,
+  "FUNC_65": 30887,
+  "FUNC_66": 30888,
+  "FUNC_67": 30889,
+  "FUNC_68": 30890,
+  "FUNC_69": 30891,
+  "FUNC_7": 30829,
+  "FUNC_70": 30892,
+  "FUNC_71": 30893,
+  "FUNC_72": 30894,
+  "FUNC_73": 30895,
+  "FUNC_74": 30896,
+  "FUNC_75": 30897,
+  "FUNC_76": 30898,
+  "FUNC_77": 30899,
+  "FUNC_78": 30900,
+  "FUNC_79": 30901,
+  "FUNC_8": 30830,
+  "FUNC_80": 30902,
+  "FUNC_81": 30903,
+  "FUNC_82": 30904,
+  "FUNC_83": 30905,
+  "FUNC_84": 30906,
+  "FUNC_85": 30907,
+  "FUNC_86": 30908,
+  "FUNC_87": 30909,
+  "FUNC_88": 30910,
+  "FUNC_89": 30911,
+  "FUNC_9": 30831,
+  "FUNC_90": 30912,
+  "FUNC_91": 30913,
+  "FUNC_92": 30914,
+  "FUNC_93": 30915,
+  "FUNC_94": 30916,
+  "FUNC_95": 30917,
+  "FUNC_96": 30918,
+  "FUNC_97": 30919,
+  "FUNC_98": 30920,
+  "FUNC_99": 30921,
+  "VAR_0": 30522,
+  "VAR_1": 30523,
+  "VAR_10": 30532,
+  "VAR_100": 30622,
+  "VAR_101": 30623,
+  "VAR_102": 30624,
+  "VAR_103": 30625,
+  "VAR_104": 30626,
+  "VAR_105": 30627,
+  "VAR_106": 30628,
+  "VAR_107": 30629,
+  "VAR_108": 30630,
+  "VAR_109": 30631,
+  "VAR_11": 30533,
+  "VAR_110": 30632,
+  "VAR_111": 30633,
+  "VAR_112": 30634,
+  "VAR_113": 30635,
+  "VAR_114": 30636,
+  "VAR_115": 30637,
+  "VAR_116": 30638,
+  "VAR_117": 30639,
+  "VAR_118": 30640,
+  "VAR_119": 30641,
+  "VAR_12": 30534,
+  "VAR_120": 30642,
+  "VAR_121": 30643,
+  "VAR_122": 30644,
+  "VAR_123": 30645,
+  "VAR_124": 30646,
+  "VAR_125": 30647,
+  "VAR_126": 30648,
+  "VAR_127": 30649,
+  "VAR_128": 30650,
+  "VAR_129": 30651,
+  "VAR_13": 30535,
+  "VAR_130": 30652,
+  "VAR_131": 30653,
+  "VAR_132": 30654,
+  "VAR_133": 30655,
+  "VAR_134": 30656,
+  "VAR_135": 30657,
+  "VAR_136": 30658,
+  "VAR_137": 30659,
+  "VAR_138": 30660,
+  "VAR_139": 30661,
+  "VAR_14": 30536,
+  "VAR_140": 30662,
+  "VAR_141": 30663,
+  "VAR_142": 30664,
+  "VAR_143": 30665,
+  "VAR_144": 30666,
+  "VAR_145": 30667,
+  "VAR_146": 30668,
+  "VAR_147": 30669,
+  "VAR_148": 30670,
+  "VAR_149": 30671,
+  "VAR_15": 30537,
+  "VAR_150": 30672,
+  "VAR_151": 30673,
+  "VAR_152": 30674,
+  "VAR_153": 30675,
+  "VAR_154": 30676,
+  "VAR_155": 30677,
+  "VAR_156": 30678,
+  "VAR_157": 30679,
+  "VAR_158": 30680,
+  "VAR_159": 30681,
+  "VAR_16": 30538,
+  "VAR_160": 30682,
+  "VAR_161": 30683,
+  "VAR_162": 30684,
+  "VAR_163": 30685,
+  "VAR_164": 30686,
+  "VAR_165": 30687,
+  "VAR_166": 30688,
+  "VAR_167": 30689,
+  "VAR_168": 30690,
+  "VAR_169": 30691,
+  "VAR_17": 30539,
+  "VAR_170": 30692,
+  "VAR_171": 30693,
+  "VAR_172": 30694,
+  "VAR_173": 30695,
+  "VAR_174": 30696,
+  "VAR_175": 30697,
+  "VAR_176": 30698,
+  "VAR_177": 30699,
+  "VAR_178": 30700,
+  "VAR_179": 30701,
+  "VAR_18": 30540,
+  "VAR_180": 30702,
+  "VAR_181": 30703,
+  "VAR_182": 30704,
+  "VAR_183": 30705,
+  "VAR_184": 30706,
+  "VAR_185": 30707,
+  "VAR_186": 30708,
+  "VAR_187": 30709,
+  "VAR_188": 30710,
+  "VAR_189": 30711,
+  "VAR_19": 30541,
+  "VAR_190": 30712,
+  "VAR_191": 30713,
+  "VAR_192": 30714,
+  "VAR_193": 30715,
+  "VAR_194": 30716,
+  "VAR_195": 30717,
+  "VAR_196": 30718,
+  "VAR_197": 30719,
+  "VAR_198": 30720,
+  "VAR_199": 30721,
+  "VAR_2": 30524,
+  "VAR_20": 30542,
+  "VAR_200": 30722,
+  "VAR_201": 30723,
+  "VAR_202": 30724,
+  "VAR_203": 30725,
+  "VAR_204": 30726,
+  "VAR_205": 30727,
+  "VAR_206": 30728,
+  "VAR_207": 30729,
+  "VAR_208": 30730,
+  "VAR_209": 30731,
+  "VAR_21": 30543,
+  "VAR_210": 30732,
+  "VAR_211": 30733,
+  "VAR_212": 30734,
+  "VAR_213": 30735,
+  "VAR_214": 30736,
+  "VAR_215": 30737,
+  "VAR_216": 30738,
+  "VAR_217": 30739,
+  "VAR_218": 30740,
+  "VAR_219": 30741,
+  "VAR_22": 30544,
+  "VAR_220": 30742,
+  "VAR_221": 30743,
+  "VAR_222": 30744,
+  "VAR_223": 30745,
+  "VAR_224": 30746,
+  "VAR_225": 30747,
+  "VAR_226": 30748,
+  "VAR_227": 30749,
+  "VAR_228": 30750,
+  "VAR_229": 30751,
+  "VAR_23": 30545,
+  "VAR_230": 30752,
+  "VAR_231": 30753,
+  "VAR_232": 30754,
+  "VAR_233": 30755,
+  "VAR_234": 30756,
+  "VAR_235": 30757,
+  "VAR_236": 30758,
+  "VAR_237": 30759,
+  "VAR_238": 30760,
+  "VAR_239": 30761,
+  "VAR_24": 30546,
+  "VAR_240": 30762,
+  "VAR_241": 30763,
+  "VAR_242": 30764,
+  "VAR_243": 30765,
+  "VAR_244": 30766,
+  "VAR_245": 30767,
+  "VAR_246": 30768,
+  "VAR_247": 30769,
+  "VAR_248": 30770,
+  "VAR_249": 30771,
+  "VAR_25": 30547,
+  "VAR_250": 30772,
+  "VAR_251": 30773,
+  "VAR_252": 30774,
+  "VAR_253": 30775,
+  "VAR_254": 30776,
+  "VAR_255": 30777,
+  "VAR_256": 30778,
+  "VAR_257": 30779,
+  "VAR_258": 30780,
+  "VAR_259": 30781,
+  "VAR_26": 30548,
+  "VAR_260": 30782,
+  "VAR_261": 30783,
+  "VAR_262": 30784,
+  "VAR_263": 30785,
+  "VAR_264": 30786,
+  "VAR_265": 30787,
+  "VAR_266": 30788,
+  "VAR_267": 30789,
+  "VAR_268": 30790,
+  "VAR_269": 30791,
+  "VAR_27": 30549,
+  "VAR_270": 30792,
+  "VAR_271": 30793,
+  "VAR_272": 30794,
+  "VAR_273": 30795,
+  "VAR_274": 30796,
+  "VAR_275": 30797,
+  "VAR_276": 30798,
+  "VAR_277": 30799,
+  "VAR_278": 30800,
+  "VAR_279": 30801,
+  "VAR_28": 30550,
+  "VAR_280": 30802,
+  "VAR_281": 30803,
+  "VAR_282": 30804,
+  "VAR_283": 30805,
+  "VAR_284": 30806,
+  "VAR_285": 30807,
+  "VAR_286": 30808,
+  "VAR_287": 30809,
+  "VAR_288": 30810,
+  "VAR_289": 30811,
+  "VAR_29": 30551,
+  "VAR_290": 30812,
+  "VAR_291": 30813,
+  "VAR_292": 30814,
+  "VAR_293": 30815,
+  "VAR_294": 30816,
+  "VAR_295": 30817,
+  "VAR_296": 30818,
+  "VAR_297": 30819,
+  "VAR_298": 30820,
+  "VAR_299": 30821,
+  "VAR_3": 30525,
+  "VAR_30": 30552,
+  "VAR_31": 30553,
+  "VAR_32": 30554,
+  "VAR_33": 30555,
+  "VAR_34": 30556,
+  "VAR_35": 30557,
+  "VAR_36": 30558,
+  "VAR_37": 30559,
+  "VAR_38": 30560,
+  "VAR_39": 30561,
+  "VAR_4": 30526,
+  "VAR_40": 30562,
+  "VAR_41": 30563,
+  "VAR_42": 30564,
+  "VAR_43": 30565,
+  "VAR_44": 30566,
+  "VAR_45": 30567,
+  "VAR_46": 30568,
+  "VAR_47": 30569,
+  "VAR_48": 30570,
+  "VAR_49": 30571,
+  "VAR_5": 30527,
+  "VAR_50": 30572,
+  "VAR_51": 30573,
+  "VAR_52": 30574,
+  "VAR_53": 30575,
+  "VAR_54": 30576,
+  "VAR_55": 30577,
+  "VAR_56": 30578,
+  "VAR_57": 30579,
+  "VAR_58": 30580,
+  "VAR_59": 30581,
+  "VAR_6": 30528,
+  "VAR_60": 30582,
+  "VAR_61": 30583,
+  "VAR_62": 30584,
+  "VAR_63": 30585,
+  "VAR_64": 30586,
+  "VAR_65": 30587,
+  "VAR_66": 30588,
+  "VAR_67": 30589,
+  "VAR_68": 30590,
+  "VAR_69": 30591,
+  "VAR_7": 30529,
+  "VAR_70": 30592,
+  "VAR_71": 30593,
+  "VAR_72": 30594,
+  "VAR_73": 30595,
+  "VAR_74": 30596,
+  "VAR_75": 30597,
+  "VAR_76": 30598,
+  "VAR_77": 30599,
+  "VAR_78": 30600,
+  "VAR_79": 30601,
+  "VAR_8": 30530,
+  "VAR_80": 30602,
+  "VAR_81": 30603,
+  "VAR_82": 30604,
+  "VAR_83": 30605,
+  "VAR_84": 30606,
+  "VAR_85": 30607,
+  "VAR_86": 30608,
+  "VAR_87": 30609,
+  "VAR_88": 30610,
+  "VAR_89": 30611,
+  "VAR_9": 30531,
+  "VAR_90": 30612,
+  "VAR_91": 30613,
+  "VAR_92": 30614,
+  "VAR_93": 30615,
+  "VAR_94": 30616,
+  "VAR_95": 30617,
+  "VAR_96": 30618,
+  "VAR_97": 30619,
+  "VAR_98": 30620,
+  "VAR_99": 30621
+}

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "name_or_path": "/root/data/thesis/data/mpnet_GAT_open_redirect_v3_masked",
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": "/root/.cache/huggingface/hub/models--sentence-transformers--all-MiniLM-L6-v2/snapshots/7dbbc90392e2f80f3d3c277d6e90027e55de9125/special_tokens_map.json",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}