2023.09.26

Browse files

Files changed (5) hide show

added_tokens.json +214 -0
special_tokens_map.json +221 -0
tokenizer.json +0 -0
tokenizer_config.json +16 -0
vocab.txt +0 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,214 @@

+{
+  "[BOS]": 32000,
+  "[EOS]": 32001,
+  "[UNK0]": 32002,
+  "[UNK1]": 32003,
+  "[UNK2]": 32004,
+  "[UNK3]": 32005,
+  "[UNK4]": 32006,
+  "[UNK5]": 32007,
+  "[UNK6]": 32008,
+  "[UNK7]": 32009,
+  "[UNK8]": 32010,
+  "[UNK9]": 32011,
+  "[unused0]": 32012,
+  "[unused100]": 32112,
+  "[unused101]": 32113,
+  "[unused102]": 32114,
+  "[unused103]": 32115,
+  "[unused104]": 32116,
+  "[unused105]": 32117,
+  "[unused106]": 32118,
+  "[unused107]": 32119,
+  "[unused108]": 32120,
+  "[unused109]": 32121,
+  "[unused10]": 32022,
+  "[unused110]": 32122,
+  "[unused111]": 32123,
+  "[unused112]": 32124,
+  "[unused113]": 32125,
+  "[unused114]": 32126,
+  "[unused115]": 32127,
+  "[unused116]": 32128,
+  "[unused117]": 32129,
+  "[unused118]": 32130,
+  "[unused119]": 32131,
+  "[unused11]": 32023,
+  "[unused120]": 32132,
+  "[unused121]": 32133,
+  "[unused122]": 32134,
+  "[unused123]": 32135,
+  "[unused124]": 32136,
+  "[unused125]": 32137,
+  "[unused126]": 32138,
+  "[unused127]": 32139,
+  "[unused128]": 32140,
+  "[unused129]": 32141,
+  "[unused12]": 32024,
+  "[unused130]": 32142,
+  "[unused131]": 32143,
+  "[unused132]": 32144,
+  "[unused133]": 32145,
+  "[unused134]": 32146,
+  "[unused135]": 32147,
+  "[unused136]": 32148,
+  "[unused137]": 32149,
+  "[unused138]": 32150,
+  "[unused139]": 32151,
+  "[unused13]": 32025,
+  "[unused140]": 32152,
+  "[unused141]": 32153,
+  "[unused142]": 32154,
+  "[unused143]": 32155,
+  "[unused144]": 32156,
+  "[unused145]": 32157,
+  "[unused146]": 32158,
+  "[unused147]": 32159,
+  "[unused148]": 32160,
+  "[unused149]": 32161,
+  "[unused14]": 32026,
+  "[unused150]": 32162,
+  "[unused151]": 32163,
+  "[unused152]": 32164,
+  "[unused153]": 32165,
+  "[unused154]": 32166,
+  "[unused155]": 32167,
+  "[unused156]": 32168,
+  "[unused157]": 32169,
+  "[unused158]": 32170,
+  "[unused159]": 32171,
+  "[unused15]": 32027,
+  "[unused160]": 32172,
+  "[unused161]": 32173,
+  "[unused162]": 32174,
+  "[unused163]": 32175,
+  "[unused164]": 32176,
+  "[unused165]": 32177,
+  "[unused166]": 32178,
+  "[unused167]": 32179,
+  "[unused168]": 32180,
+  "[unused169]": 32181,
+  "[unused16]": 32028,
+  "[unused170]": 32182,
+  "[unused171]": 32183,
+  "[unused172]": 32184,
+  "[unused173]": 32185,
+  "[unused174]": 32186,
+  "[unused175]": 32187,
+  "[unused176]": 32188,
+  "[unused177]": 32189,
+  "[unused178]": 32190,
+  "[unused179]": 32191,
+  "[unused17]": 32029,
+  "[unused180]": 32192,
+  "[unused181]": 32193,
+  "[unused182]": 32194,
+  "[unused183]": 32195,
+  "[unused184]": 32196,
+  "[unused185]": 32197,
+  "[unused186]": 32198,
+  "[unused187]": 32199,
+  "[unused188]": 32200,
+  "[unused189]": 32201,
+  "[unused18]": 32030,
+  "[unused190]": 32202,
+  "[unused191]": 32203,
+  "[unused192]": 32204,
+  "[unused193]": 32205,
+  "[unused194]": 32206,
+  "[unused195]": 32207,
+  "[unused196]": 32208,
+  "[unused197]": 32209,
+  "[unused198]": 32210,
+  "[unused199]": 32211,
+  "[unused19]": 32031,
+  "[unused1]": 32013,
+  "[unused20]": 32032,
+  "[unused21]": 32033,
+  "[unused22]": 32034,
+  "[unused23]": 32035,
+  "[unused24]": 32036,
+  "[unused25]": 32037,
+  "[unused26]": 32038,
+  "[unused27]": 32039,
+  "[unused28]": 32040,
+  "[unused29]": 32041,
+  "[unused2]": 32014,
+  "[unused30]": 32042,
+  "[unused31]": 32043,
+  "[unused32]": 32044,
+  "[unused33]": 32045,
+  "[unused34]": 32046,
+  "[unused35]": 32047,
+  "[unused36]": 32048,
+  "[unused37]": 32049,
+  "[unused38]": 32050,
+  "[unused39]": 32051,
+  "[unused3]": 32015,
+  "[unused40]": 32052,
+  "[unused41]": 32053,
+  "[unused42]": 32054,
+  "[unused43]": 32055,
+  "[unused44]": 32056,
+  "[unused45]": 32057,
+  "[unused46]": 32058,
+  "[unused47]": 32059,
+  "[unused48]": 32060,
+  "[unused49]": 32061,
+  "[unused4]": 32016,
+  "[unused50]": 32062,
+  "[unused51]": 32063,
+  "[unused52]": 32064,
+  "[unused53]": 32065,
+  "[unused54]": 32066,
+  "[unused55]": 32067,
+  "[unused56]": 32068,
+  "[unused57]": 32069,
+  "[unused58]": 32070,
+  "[unused59]": 32071,
+  "[unused5]": 32017,
+  "[unused60]": 32072,
+  "[unused61]": 32073,
+  "[unused62]": 32074,
+  "[unused63]": 32075,
+  "[unused64]": 32076,
+  "[unused65]": 32077,
+  "[unused66]": 32078,
+  "[unused67]": 32079,
+  "[unused68]": 32080,
+  "[unused69]": 32081,
+  "[unused6]": 32018,
+  "[unused70]": 32082,
+  "[unused71]": 32083,
+  "[unused72]": 32084,
+  "[unused73]": 32085,
+  "[unused74]": 32086,
+  "[unused75]": 32087,
+  "[unused76]": 32088,
+  "[unused77]": 32089,
+  "[unused78]": 32090,
+  "[unused79]": 32091,
+  "[unused7]": 32019,
+  "[unused80]": 32092,
+  "[unused81]": 32093,
+  "[unused82]": 32094,
+  "[unused83]": 32095,
+  "[unused84]": 32096,
+  "[unused85]": 32097,
+  "[unused86]": 32098,
+  "[unused87]": 32099,
+  "[unused88]": 32100,
+  "[unused89]": 32101,
+  "[unused8]": 32020,
+  "[unused90]": 32102,
+  "[unused91]": 32103,
+  "[unused92]": 32104,
+  "[unused93]": 32105,
+  "[unused94]": 32106,
+  "[unused95]": 32107,
+  "[unused96]": 32108,
+  "[unused97]": 32109,
+  "[unused98]": 32110,
+  "[unused99]": 32111,
+  "[unused9]": 32021
+}

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,221 @@

+{
+  "additional_special_tokens": [
+    "[BOS]",
+    "[EOS]",
+    "[UNK0]",
+    "[UNK1]",
+    "[UNK2]",
+    "[UNK3]",
+    "[UNK4]",
+    "[UNK5]",
+    "[UNK6]",
+    "[UNK7]",
+    "[UNK8]",
+    "[UNK9]",
+    "[unused0]",
+    "[unused1]",
+    "[unused2]",
+    "[unused3]",
+    "[unused4]",
+    "[unused5]",
+    "[unused6]",
+    "[unused7]",
+    "[unused8]",
+    "[unused9]",
+    "[unused10]",
+    "[unused11]",
+    "[unused12]",
+    "[unused13]",
+    "[unused14]",
+    "[unused15]",
+    "[unused16]",
+    "[unused17]",
+    "[unused18]",
+    "[unused19]",
+    "[unused20]",
+    "[unused21]",
+    "[unused22]",
+    "[unused23]",
+    "[unused24]",
+    "[unused25]",
+    "[unused26]",
+    "[unused27]",
+    "[unused28]",
+    "[unused29]",
+    "[unused30]",
+    "[unused31]",
+    "[unused32]",
+    "[unused33]",
+    "[unused34]",
+    "[unused35]",
+    "[unused36]",
+    "[unused37]",
+    "[unused38]",
+    "[unused39]",
+    "[unused40]",
+    "[unused41]",
+    "[unused42]",
+    "[unused43]",
+    "[unused44]",
+    "[unused45]",
+    "[unused46]",
+    "[unused47]",
+    "[unused48]",
+    "[unused49]",
+    "[unused50]",
+    "[unused51]",
+    "[unused52]",
+    "[unused53]",
+    "[unused54]",
+    "[unused55]",
+    "[unused56]",
+    "[unused57]",
+    "[unused58]",
+    "[unused59]",
+    "[unused60]",
+    "[unused61]",
+    "[unused62]",
+    "[unused63]",
+    "[unused64]",
+    "[unused65]",
+    "[unused66]",
+    "[unused67]",
+    "[unused68]",
+    "[unused69]",
+    "[unused70]",
+    "[unused71]",
+    "[unused72]",
+    "[unused73]",
+    "[unused74]",
+    "[unused75]",
+    "[unused76]",
+    "[unused77]",
+    "[unused78]",
+    "[unused79]",
+    "[unused80]",
+    "[unused81]",
+    "[unused82]",
+    "[unused83]",
+    "[unused84]",
+    "[unused85]",
+    "[unused86]",
+    "[unused87]",
+    "[unused88]",
+    "[unused89]",
+    "[unused90]",
+    "[unused91]",
+    "[unused92]",
+    "[unused93]",
+    "[unused94]",
+    "[unused95]",
+    "[unused96]",
+    "[unused97]",
+    "[unused98]",
+    "[unused99]",
+    "[unused100]",
+    "[unused101]",
+    "[unused102]",
+    "[unused103]",
+    "[unused104]",
+    "[unused105]",
+    "[unused106]",
+    "[unused107]",
+    "[unused108]",
+    "[unused109]",
+    "[unused110]",
+    "[unused111]",
+    "[unused112]",
+    "[unused113]",
+    "[unused114]",
+    "[unused115]",
+    "[unused116]",
+    "[unused117]",
+    "[unused118]",
+    "[unused119]",
+    "[unused120]",
+    "[unused121]",
+    "[unused122]",
+    "[unused123]",
+    "[unused124]",
+    "[unused125]",
+    "[unused126]",
+    "[unused127]",
+    "[unused128]",
+    "[unused129]",
+    "[unused130]",
+    "[unused131]",
+    "[unused132]",
+    "[unused133]",
+    "[unused134]",
+    "[unused135]",
+    "[unused136]",
+    "[unused137]",
+    "[unused138]",
+    "[unused139]",
+    "[unused140]",
+    "[unused141]",
+    "[unused142]",
+    "[unused143]",
+    "[unused144]",
+    "[unused145]",
+    "[unused146]",
+    "[unused147]",
+    "[unused148]",
+    "[unused149]",
+    "[unused150]",
+    "[unused151]",
+    "[unused152]",
+    "[unused153]",
+    "[unused154]",
+    "[unused155]",
+    "[unused156]",
+    "[unused157]",
+    "[unused158]",
+    "[unused159]",
+    "[unused160]",
+    "[unused161]",
+    "[unused162]",
+    "[unused163]",
+    "[unused164]",
+    "[unused165]",
+    "[unused166]",
+    "[unused167]",
+    "[unused168]",
+    "[unused169]",
+    "[unused170]",
+    "[unused171]",
+    "[unused172]",
+    "[unused173]",
+    "[unused174]",
+    "[unused175]",
+    "[unused176]",
+    "[unused177]",
+    "[unused178]",
+    "[unused179]",
+    "[unused180]",
+    "[unused181]",
+    "[unused182]",
+    "[unused183]",
+    "[unused184]",
+    "[unused185]",
+    "[unused186]",
+    "[unused187]",
+    "[unused188]",
+    "[unused189]",
+    "[unused190]",
+    "[unused191]",
+    "[unused192]",
+    "[unused193]",
+    "[unused194]",
+    "[unused195]",
+    "[unused196]",
+    "[unused197]",
+    "[unused198]",
+    "[unused199]"
+  ],
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,16 @@

+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "lowercase": false,
+  "mask_token": "[MASK]",
+  "name_or_path": "tokenizer_model",
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": false,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

vocab.txt ADDED Viewed

The diff for this file is too large to render. See raw diff