cnh_latn_5mb / added_tokens.json
goldfish-models's picture
Upload cnh_latn_5mb tokenizer.
25b0c21
raw
history blame
12.1 kB
{"[XXXXX573]": 34809, "[XXXXX36]": 34272, "[XXXXX79]": 34315, "[XXXXX87]": 34323, "[XXXXX83]": 34319, "[XXXXX284]": 34520, "[XXXXX422]": 34658, "[XXXXX431]": 34667, "[XXXXX149]": 34385, "[XXXXX482]": 34718, "[XXXXX141]": 34377, "[SEP]": 34233, "[XXXXX533]": 34769, "[XXXXX383]": 34619, "[XXXXX436]": 34672, "[XXXXX152]": 34388, "[XXXXX338]": 34574, "[XXXXX389]": 34625, "[XXXXX273]": 34509, "[XXXXX528]": 34764, "[XXXXX76]": 34312, "[XXXXX74]": 34310, "[XXXXX296]": 34532, "[XXXXX115]": 34351, "[XXXXX274]": 34510, "[CLS]": 34232, "[XXXXX328]": 34564, "[XXXXX258]": 34494, "[XXXXX202]": 34438, "[XXXXX52]": 34288, "[XXXXX301]": 34537, "[XXXXX340]": 34576, "[XXXXX35]": 34271, "[XXXXX99]": 34335, "[XXXXX359]": 34595, "[XXXXX308]": 34544, "[XXXXX177]": 34413, "[XXXXX180]": 34416, "[XXXXX192]": 34428, "[XXXXX150]": 34386, "[XXXXX408]": 34644, "[XXXXX219]": 34455, "[XXXXX484]": 34720, "[XXXXX343]": 34579, "[XXXXX324]": 34560, "[XXXXX124]": 34360, "[XXXXX238]": 34474, "[XXXXX102]": 34338, "[XXXXX395]": 34631, "[XXXXX159]": 34395, "[XXXXX151]": 34387, "[XXXXX468]": 34704, "[XXXXX37]": 34273, "[XXXXX462]": 34698, "[XXXXX478]": 34714, "[XXXXX280]": 34516, "[XXXXX470]": 34706, "[XXXXX28]": 34264, "[XXXXX500]": 34736, "[XXXXX270]": 34506, "[XXXXX448]": 34684, "[XXXXX134]": 34370, "[XXXXX453]": 34689, "[XXXXX213]": 34449, "[XXXXX268]": 34504, "[XXXXX392]": 34628, "[XXXXX563]": 34799, "[XXXXX47]": 34283, "[XXXXX123]": 34359, "[XXXXX155]": 34391, "[XXXXX137]": 34373, "[XXXXX209]": 34445, "[XXXXX21]": 34257, "[XXXXX201]": 34437, "[XXXXX487]": 34723, "[XXXXX27]": 34263, "[XXXXX222]": 34458, "[XXXXX330]": 34566, "[XXXXX549]": 34785, "[XXXXX543]": 34779, "[XXXXX232]": 34468, "[XXXXX300]": 34536, "[XXXXX265]": 34501, "[XXXXX522]": 34758, "[XXXXX194]": 34430, "[XXXXX253]": 34489, "[XXXXX282]": 34518, "[XXXXX371]": 34607, "[XXXXX429]": 34665, "[XXXXX529]": 34765, "[XXXXX55]": 34291, "[XXXXX45]": 34281, "[XXXXX465]": 34701, "[XXXXX554]": 34790, "[XXXXX263]": 34499, "[XXXXX532]": 34768, "[XXXXX93]": 34329, "[XXXXX467]": 34703, "[XXXXX387]": 34623, "[XXXXX537]": 34773, "[XXXXX23]": 34259, "[XXXXX193]": 34429, "[XXXXX257]": 34493, "[XXXXX421]": 34657, "[XXXXX417]": 34653, "[XXXXX220]": 34456, "[XXXXX239]": 34475, "[XXXXX314]": 34550, "[XXXXX404]": 34640, "[XXXXX266]": 34502, "[XXXXX513]": 34749, "[XXXXX269]": 34505, "[XXXXX162]": 34398, "[XXXXX433]": 34669, "[XXXXX566]": 34802, "[XXXXX402]": 34638, "[XXXXX361]": 34597, "[XXXXX567]": 34803, "[XXXXX378]": 34614, "[XXXXX466]": 34702, "[XXXXX10]": 34246, "[XXXXX18]": 34254, "[XXXXX148]": 34384, "[XXXXX278]": 34514, "[XXXXX572]": 34808, "[XXXXX230]": 34466, "[XXXXX69]": 34305, "[XXXXX16]": 34252, "[XXXXX497]": 34733, "[XXXXX224]": 34460, "[XXXXX579]": 34815, "[XXXXX362]": 34598, "[XXXXX4]": 34240, "[XXXXX425]": 34661, "[XXXXX374]": 34610, "[XXXXX64]": 34300, "[XXXXX58]": 34294, "[XXXXX6]": 34242, "[XXXXX414]": 34650, "[XXXXX229]": 34465, "[XXXXX525]": 34761, "[XXXXX486]": 34722, "[XXXXX297]": 34533, "[XXXXX490]": 34726, "[XXXXX349]": 34585, "[XXXXX339]": 34575, "[XXXXX393]": 34629, "[XXXXX256]": 34492, "[XXXXX181]": 34417, "[XXXXX179]": 34415, "[XXXXX33]": 34269, "[XXXXX153]": 34389, "[XXXXX131]": 34367, "[XXXXX474]": 34710, "[XXXXX119]": 34355, "[XXXXX377]": 34613, "[XXXXX381]": 34617, "[XXXXX48]": 34284, "[XXXXX165]": 34401, "[XXXXX477]": 34713, "[XXXXX237]": 34473, "[XXXXX318]": 34554, "[XXXXX460]": 34696, "[XXXXX174]": 34410, "[XXXXX32]": 34268, "[XXXXX184]": 34420, "[XXXXX130]": 34366, "[XXXXX337]": 34573, "[XXXXX292]": 34528, "[XXXXX379]": 34615, "[XXXXX403]": 34639, "[XXXXX428]": 34664, "[XXXXX307]": 34543, "[XXXXX217]": 34453, "[XXXXX304]": 34540, "[XXXXX358]": 34594, "[XXXXX352]": 34588, "[XXXXX375]": 34611, "[XXXXX427]": 34663, "[XXXXX556]": 34792, "[XXXXX186]": 34422, "[XXXXX85]": 34321, "[XXXXX564]": 34800, "[XXXXX360]": 34596, "[XXXXX406]": 34642, "[XXXXX416]": 34652, "[XXXXX275]": 34511, "[XXXXX288]": 34524, "[XXXXX390]": 34626, "[XXXXX332]": 34568, "[XXXXX481]": 34717, "[XXXXX144]": 34380, "[XXXXX30]": 34266, "[XXXXX169]": 34405, "[XXXXX187]": 34423, "[XXXXX12]": 34248, "[XXXXX432]": 34668, "[XXXXX191]": 34427, "[XXXXX472]": 34708, "[XXXXX285]": 34521, "[XXXXX1]": 34237, "[XXXXX538]": 34774, "[XXXXX68]": 34304, "[XXXXX412]": 34648, "[XXXXX94]": 34330, "[XXXXX373]": 34609, "[XXXXX530]": 34766, "[XXXXX405]": 34641, "[XXXXX306]": 34542, "[XXXXX499]": 34735, "[XXXXX41]": 34277, "[XXXXX46]": 34282, "[XXXXX430]": 34666, "[XXXXX226]": 34462, "[XXXXX59]": 34295, "[XXXXX473]": 34709, "[XXXXX163]": 34399, "[XXXXX394]": 34630, "[XXXXX75]": 34311, "[XXXXX234]": 34470, "[XXXXX136]": 34372, "[XXXXX410]": 34646, "[XXXXX147]": 34383, "[XXXXX303]": 34539, "[XXXXX54]": 34290, "[XXXXX354]": 34590, "[XXXXX44]": 34280, "[XXXXX2]": 34238, "[XXXXX441]": 34677, "[XXXXX133]": 34369, "[XXXXX271]": 34507, "[XXXXX105]": 34341, "[XXXXX106]": 34342, "[XXXXX442]": 34678, "[XXXXX496]": 34732, "[MASK]": 34235, "[XXXXX527]": 34763, "[XXXXX251]": 34487, "[XXXXX200]": 34436, "[XXXXX295]": 34531, "[XXXXX277]": 34513, "[XXXXX539]": 34775, "[XXXXX43]": 34279, "[XXXXX514]": 34750, "[XXXXX305]": 34541, "[XXXXX558]": 34794, "[XXXXX508]": 34744, "[XXXXX335]": 34571, "[XXXXX126]": 34362, "[XXXXX298]": 34534, "[XXXXX446]": 34682, "[XXXXX57]": 34293, "[XXXXX411]": 34647, "[XXXXX348]": 34584, "[XXXXX140]": 34376, "[XXXXX440]": 34676, "[XXXXX512]": 34748, "[XXXXX424]": 34660, "[XXXXX259]": 34495, "[XXXXX388]": 34624, "[XXXXX423]": 34659, "[XXXXX293]": 34529, "[XXXXX128]": 34364, "[XXXXX19]": 34255, "[XXXXX122]": 34358, "[XXXXX67]": 34303, "[XXXXX396]": 34632, "[XXXXX281]": 34517, "[XXXXX507]": 34743, "[XXXXX244]": 34480, "[XXXXX96]": 34332, "[XXXXX261]": 34497, "[XXXXX116]": 34352, "[XXXXX302]": 34538, "[XXXXX517]": 34753, "[XXXXX249]": 34485, "[XXXXX541]": 34777, "[XXXXX78]": 34314, "[XXXXX205]": 34441, "[XXXXX455]": 34691, "[XXXXX243]": 34479, "[XXXXX158]": 34394, "[XXXXX276]": 34512, "[XXXXX171]": 34407, "[XXXXX550]": 34786, "[XXXXX143]": 34379, "[XXXXX111]": 34347, "[XXXXX479]": 34715, "[XXXXX493]": 34729, "[XXXXX129]": 34365, "[XXXXX250]": 34486, "[XXXXX154]": 34390, "[XXXXX176]": 34412, "[XXXXX53]": 34289, "[XXXXX0]": 34236, "[XXXXX170]": 34406, "[XXXXX3]": 34239, "[XXXXX120]": 34356, "[XXXXX461]": 34697, "[XXXXX492]": 34728, "[XXXXX400]": 34636, "[XXXXX535]": 34771, "[XXXXX443]": 34679, "[XXXXX494]": 34730, "[XXXXX551]": 34787, "[XXXXX245]": 34481, "[XXXXX384]": 34620, "[XXXXX104]": 34340, "[XXXXX459]": 34695, "[XXXXX110]": 34346, "[XXXXX101]": 34337, "[XXXXX531]": 34767, "[XXXXX450]": 34686, "[XXXXX92]": 34328, "[XXXXX40]": 34276, "[XXXXX368]": 34604, "[XXXXX310]": 34546, "[XXXXX204]": 34440, "[XXXXX498]": 34734, "[XXXXX146]": 34382, "[XXXXX38]": 34274, "[XXXXX228]": 34464, "[XXXXX189]": 34425, "[XXXXX109]": 34345, "[XXXXX435]": 34671, "[XXXXX7]": 34243, "[XXXXX112]": 34348, "[XXXXX167]": 34403, "[XXXXX142]": 34378, "[XXXXX452]": 34688, "[XXXXX216]": 34452, "[XXXXX91]": 34327, "[XXXXX262]": 34498, "[XXXXX242]": 34478, "[XXXXX418]": 34654, "[XXXXX312]": 34548, "[XXXXX173]": 34409, "[XXXXX248]": 34484, "[XXXXX559]": 34795, "[XXXXX334]": 34570, "[XXXXX401]": 34637, "[XXXXX185]": 34421, "[XXXXX457]": 34693, "[XXXXX365]": 34601, "[XXXXX520]": 34756, "[XXXXX509]": 34745, "[XXXXX132]": 34368, "[XXXXX199]": 34435, "[XXXXX70]": 34306, "[XXXXX398]": 34634, "[XXXXX206]": 34442, "[XXXXX51]": 34287, "[XXXXX208]": 34444, "[XXXXX571]": 34807, "[XXXXX575]": 34811, "[XXXXX138]": 34374, "[XXXXX196]": 34432, "[XXXXX236]": 34472, "[XXXXX56]": 34292, "[XXXXX322]": 34558, "[XXXXX121]": 34357, "[XXXXX489]": 34725, "[XXXXX576]": 34812, "[XXXXX62]": 34298, "[XXXXX504]": 34740, "[XXXXX84]": 34320, "[XXXXX399]": 34635, "[XXXXX114]": 34350, "[XXXXX315]": 34551, "[XXXXX299]": 34535, "[XXXXX320]": 34556, "[XXXXX73]": 34309, "[XXXXX413]": 34649, "[XXXXX286]": 34522, "[XXXXX345]": 34581, "[XXXXX60]": 34296, "[XXXXX233]": 34469, "[XXXXX521]": 34757, "[XXXXX382]": 34618, "[XXXXX313]": 34549, "[XXXXX495]": 34731, "[XXXXX439]": 34675, "[XXXXX325]": 34561, "[XXXXX317]": 34553, "[XXXXX437]": 34673, "[XXXXX8]": 34244, "[XXXXX503]": 34739, "[XXXXX370]": 34606, "[XXXXX519]": 34755, "[XXXXX22]": 34258, "[XXXXX577]": 34813, "[XXXXX463]": 34699, "[XXXXX254]": 34490, "[XXXXX283]": 34519, "[XXXXX156]": 34392, "[XXXXX49]": 34285, "[XXXXX107]": 34343, "[XXXXX562]": 34798, "[XXXXX26]": 34262, "[XXXXX89]": 34325, "[XXXXX445]": 34681, "[XXXXX221]": 34457, "[XXXXX279]": 34515, "[XXXXX63]": 34299, "[XXXXX255]": 34491, "[XXXXX451]": 34687, "[XXXXX188]": 34424, "[XXXXX447]": 34683, "[XXXXX164]": 34400, "[XXXXX77]": 34313, "[XXXXX272]": 34508, "[XXXXX454]": 34690, "[XXXXX183]": 34419, "[XXXXX195]": 34431, "[XXXXX95]": 34331, "[XXXXX506]": 34742, "[XXXXX438]": 34674, "[XXXXX355]": 34591, "[XXXXX108]": 34344, "[XXXXX570]": 34806, "[XXXXX356]": 34592, "[XXXXX127]": 34363, "[XXXXX480]": 34716, "[XXXXX118]": 34354, "[XXXXX13]": 34249, "[XXXXX145]": 34381, "[XXXXX501]": 34737, "[XXXXX363]": 34599, "[XXXXX100]": 34336, "[XXXXX329]": 34565, "[XXXXX88]": 34324, "[XXXXX397]": 34633, "[XXXXX578]": 34814, "[XXXXX364]": 34600, "[XXXXX246]": 34482, "[XXXXX212]": 34448, "[XXXXX309]": 34545, "[XXXXX264]": 34500, "[XXXXX565]": 34801, "[XXXXX491]": 34727, "[XXXXX29]": 34265, "[XXXXX386]": 34622, "[XXXXX407]": 34643, "[XXXXX547]": 34783, "[XXXXX161]": 34397, "[XXXXX449]": 34685, "[XXXXX235]": 34471, "[XXXXX267]": 34503, "[XXXXX20]": 34256, "[XXXXX350]": 34586, "[XXXXX215]": 34451, "[XXXXX182]": 34418, "[XXXXX14]": 34250, "[XXXXX488]": 34724, "[XXXXX331]": 34567, "[XXXXX552]": 34788, "[XXXXX342]": 34578, "[XXXXX117]": 34353, "[XXXXX294]": 34530, "[XXXXX534]": 34770, "[XXXXX321]": 34557, "[XXXXX542]": 34778, "[XXXXX81]": 34317, "[XXXXX458]": 34694, "[XXXXX569]": 34805, "[XXXXX25]": 34261, "[XXXXX505]": 34741, "[XXXXX376]": 34612, "[XXXXX241]": 34477, "[XXXXX536]": 34772, "<pad>": 34234, "[XXXXX172]": 34408, "[XXXXX419]": 34655, "[XXXXX198]": 34434, "[XXXXX125]": 34361, "[XXXXX380]": 34616, "[XXXXX548]": 34784, "[XXXXX574]": 34810, "[XXXXX434]": 34670, "[XXXXX341]": 34577, "[XXXXX157]": 34393, "[XXXXX426]": 34662, "[XXXXX326]": 34562, "[XXXXX385]": 34621, "[XXXXX72]": 34308, "[XXXXX82]": 34318, "[XXXXX240]": 34476, "[XXXXX526]": 34762, "[XXXXX560]": 34796, "[XXXXX166]": 34402, "[XXXXX9]": 34245, "[XXXXX369]": 34605, "[XXXXX15]": 34251, "[XXXXX97]": 34333, "[XXXXX545]": 34781, "[XXXXX80]": 34316, "[XXXXX485]": 34721, "[XXXXX456]": 34692, "[XXXXX546]": 34782, "[XXXXX518]": 34754, "[XXXXX190]": 34426, "[XXXXX555]": 34791, "[XXXXX366]": 34602, "[XXXXX347]": 34583, "[XXXXX225]": 34461, "[XXXXX444]": 34680, "[XXXXX197]": 34433, "[XXXXX333]": 34569, "[XXXXX290]": 34526, "[XXXXX17]": 34253, "[XXXXX464]": 34700, "[XXXXX409]": 34645, "[XXXXX516]": 34752, "[XXXXX260]": 34496, "[XXXXX252]": 34488, "[XXXXX561]": 34797, "[XXXXX210]": 34446, "[XXXXX227]": 34463, "[XXXXX511]": 34747, "[XXXXX469]": 34705, "[XXXXX311]": 34547, "[XXXXX39]": 34275, "[XXXXX544]": 34780, "[XXXXX287]": 34523, "[XXXXX66]": 34302, "[XXXXX223]": 34459, "[XXXXX61]": 34297, "[XXXXX34]": 34270, "[XXXXX510]": 34746, "[XXXXX316]": 34552, "[XXXXX218]": 34454, "[XXXXX415]": 34651, "[XXXXX476]": 34712, "[XXXXX502]": 34738, "[XXXXX336]": 34572, "[XXXXX139]": 34375, "[XXXXX42]": 34278, "[XXXXX247]": 34483, "[XXXXX113]": 34349, "[XXXXX372]": 34608, "[XXXXX568]": 34804, "[XXXXX515]": 34751, "[XXXXX65]": 34301, "[XXXXX207]": 34443, "[XXXXX319]": 34555, "[XXXXX5]": 34241, "[XXXXX168]": 34404, "[XXXXX344]": 34580, "[XXXXX353]": 34589, "[XXXXX289]": 34525, "[XXXXX11]": 34247, "[XXXXX557]": 34793, "[XXXXX31]": 34267, "[XXXXX231]": 34467, "[XXXXX90]": 34326, "[XXXXX291]": 34527, "[XXXXX50]": 34286, "[XXXXX483]": 34719, "[XXXXX323]": 34559, "[XXXXX135]": 34371, "[XXXXX327]": 34563, "[XXXXX420]": 34656, "[XXXXX160]": 34396, "[XXXXX24]": 34260, "[XXXXX98]": 34334, "[XXXXX86]": 34322, "[XXXXX175]": 34411, "[XXXXX214]": 34450, "[XXXXX346]": 34582, "[XXXXX553]": 34789, "[XXXXX211]": 34447, "[XXXXX391]": 34627, "[XXXXX71]": 34307, "[XXXXX351]": 34587, "[XXXXX523]": 34759, "[XXXXX524]": 34760, "[XXXXX540]": 34776, "[XXXXX367]": 34603, "[XXXXX475]": 34711, "[XXXXX203]": 34439, "[XXXXX178]": 34414, "[XXXXX471]": 34707, "[XXXXX103]": 34339, "[XXXXX357]": 34593}