BioTokenizer-BFD-UNI-400 / tokenizer.json

Upload 2 files

b516756 10 months ago

No virus

23.7 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "<UNK>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": {
	"type": "Lowercase"
	},
	"pre_tokenizer": {
	"type": "Whitespace"
	},
	"post_processor": null,
	"decoder": null,
	"model": {
	"type": "Unigram",
	"unk_id": 0,
	"vocab": [
	[
	"<UNK>",
	0.0
	],
	[
	"a",
	-3.632944090450515
	],
	[
	"l",
	-3.7068570202096325
	],
	[
	"g",
	-3.9042950954442333
	],
	[
	"r",
	-3.9545991750240184
	],
	[
	"v",
	-3.998234837926354
	],
	[
	"s",
	-4.14952275224373
	],
	[
	"t",
	-4.287456837510817
	],
	[
	"e",
	-4.289392028132756
	],
	[
	"p",
	-4.290234200036018
	],
	[
	"d",
	-4.3477089813135485
	],
	[
	"i",
	-4.476037236923364
	],
	[
	"m",
	-4.510006645253128
	],
	[
	"k",
	-4.708311207049434
	],
	[
	"q",
	-4.735006170352175
	],
	[
	"f",
	-4.789893467104555
	],
	[
	"h",
	-5.0101739158699665
	],
	[
	"n",
	-5.01778111762699
	],
	[
	"aa",
	-5.022472558408127
	],
	[
	"rr",
	-5.128207662147673
	],
	[
	"c",
	-5.145198760398738
	],
	[
	"al",
	-5.180538576122947
	],
	[
	"ll",
	-5.201140477172862
	],
	[
	"y",
	-5.2091061310437485
	],
	[
	"av",
	-5.23780321892715
	],
	[
	"ag",
	-5.2607155787369955
	],
	[
	"rl",
	-5.269063013646287
	],
	[
	"lv",
	-5.280324374261429
	],
	[
	"vl",
	-5.2821850373789125
	],
	[
	"la",
	-5.287911107396141
	],
	[
	"gg",
	-5.2907629902747395
	],
	[
	"lr",
	-5.296613327734477
	],
	[
	"ar",
	-5.317839521134866
	],
	[
	"w",
	-5.3283983119940075
	],
	[
	"ga",
	-5.342320152037463
	],
	[
	"lg",
	-5.354316588369922
	],
	[
	"gr",
	-5.36421754910026
	],
	[
	"ls",
	-5.365998834295857
	],
	[
	"ra",
	-5.368576042445021
	],
	[
	"el",
	-5.3742851295036225
	],
	[
	"sa",
	-5.3767074957757295
	],
	[
	"pa",
	-5.38691039440727
	],
	[
	"va",
	-5.391132319944489
	],
	[
	"sl",
	-5.391970768048152
	],
	[
	"ta",
	-5.395723638820606
	],
	[
	"gl",
	-5.397241434678124
	],
	[
	"vv",
	-5.402501062230561
	],
	[
	"as",
	-5.4054133935553565
	],
	[
	"dl",
	-5.408363687465943
	],
	[
	"da",
	-5.419504788703245
	],
	[
	"lp",
	-5.421171023058353
	],
	[
	"sg",
	-5.425624703056531
	],
	[
	"vg",
	-5.43958578125789
	],
	[
	"pg",
	-5.443116085521833
	],
	[
	"ld",
	-5.460648843958097
	],
	[
	"gv",
	-5.461726128843196
	],
	[
	"at",
	-5.4623194350596
	],
	[
	"tl",
	-5.476791998017722
	],
	[
	"ae",
	-5.485646676180689
	],
	[
	"ss",
	-5.494587867484871
	],
	[
	"ad",
	-5.503243044744844
	],
	[
	"le",
	-5.513539597107796
	],
	[
	"er",
	-5.525114213305434
	],
	[
	"dg",
	-5.52631343875059
	],
	[
	"lt",
	-5.535693817027219
	],
	[
	"ea",
	-5.535755605860464
	],
	[
	"ia",
	-5.536650327927395
	],
	[
	"tg",
	-5.545634533570794
	],
	[
	"rg",
	-5.547461100096692
	],
	[
	"vr",
	-5.5482882187858085
	],
	[
	"rv",
	-5.5960473758838525
	],
	[
	"x",
	-5.608946077998157
	],
	[
	"rs",
	-5.611870379615395
	],
	[
	"gs",
	-5.626777482170128
	],
	[
	"ap",
	-5.631965142077629
	],
	[
	"ai",
	-5.636142688236619
	],
	[
	"ve",
	-5.639894023082112
	],
	[
	"ge",
	-5.655452846064064
	],
	[
	"pl",
	-5.660462720181497
	],
	[
	"gd",
	-5.664400349464522
	],
	[
	"ee",
	-5.671898580128399
	],
	[
	"vd",
	-5.672600958807292
	],
	[
	"dv",
	-5.674250319761869
	],
	[
	"tv",
	-5.675826469215492
	],
	[
	"gt",
	-5.682517311383533
	],
	[
	"rp",
	-5.687456812768204
	],
	[
	"vt",
	-5.69613081375628
	],
	[
	"pv",
	-5.701851465097279
	],
	[
	"vs",
	-5.7061688041113765
	],
	[
	"dr",
	-5.729755803330297
	],
	[
	"ig",
	-5.740135021060222
	],
	[
	"sr",
	-5.740787547090417
	],
	[
	"re",
	-5.753341772270069
	],
	[
	"sv",
	-5.773162831092586
	],
	[
	"ev",
	-5.773357017945507
	],
	[
	"gi",
	-5.779110480802128
	],
	[
	"sp",
	-5.78792820379083
	],
	[
	"tt",
	-5.788142314600666
	],
	[
	"li",
	-5.7894561148560495
	],
	[
	"tp",
	-5.811521012006784
	],
	[
	"de",
	-5.81161181019956
	],
	[
	"pp",
	-5.817117213994678
	],
	[
	"aq",
	-5.8208411958823
	],
	[
	"vp",
	-5.82086135309334
	],
	[
	"pr",
	-5.8300324993501444
	],
	[
	"rd",
	-5.832959938409839
	],
	[
	"ps",
	-5.833765158121672
	],
	[
	"iv",
	-5.834924940666063
	],
	[
	"st",
	-5.836422116950422
	],
	[
	"ts",
	-5.845334860786274
	],
	[
	"qa",
	-5.8602742392000575
	],
	[
	"dp",
	-5.889141466606141
	],
	[
	"lf",
	-5.891988029154746
	],
	[
	"ei",
	-5.8942462729591
	],
	[
	"ql",
	-5.896280144036849
	],
	[
	"pe",
	-5.8989822786508554
	],
	[
	"dd",
	-5.899947270574684
	],
	[
	"fa",
	-5.908091119842952
	],
	[
	"il",
	-5.909313751415439
	],
	[
	"pd",
	-5.916882430976356
	],
	[
	"lk",
	-5.926334580525451
	],
	[
	"kk",
	-5.929554253156709
	],
	[
	"af",
	-5.9304097490202015
	],
	[
	"fg",
	-5.934233305800573
	],
	[
	"eg",
	-5.935506392701038
	],
	[
	"gk",
	-5.943123839548651
	],
	[
	"ak",
	-5.947812328653333
	],
	[
	"fl",
	-5.948919578195277
	],
	[
	"id",
	-5.95000787138518
	],
	[
	"ri",
	-5.953187755007269
	],
	[
	"kl",
	-5.954798705995039
	],
	[
	"vi",
	-5.9632196203527315
	],
	[
	"lq",
	-5.981225013933381
	],
	[
	"ie",
	-5.987842234328983
	],
	[
	"gp",
	-5.996294580750687
	],
	[
	"ek",
	-6.006345600102266
	],
	[
	"rt",
	-6.0121209086764775
	],
	[
	"ka",
	-6.01491898744084
	],
	[
	"gf",
	-6.015842456861838
	],
	[
	"qr",
	-6.016464987489098
	],
	[
	"is",
	-6.045176758128992
	],
	[
	"nl",
	-6.085959225557872
	],
	[
	"pt",
	-6.0880399997621435
	],
	[
	"si",
	-6.0889675409573325
	],
	[
	"ti",
	-6.089641985742588
	],
	[
	"rq",
	-6.093718164152271
	],
	[
	"tr",
	-6.102011832883294
	],
	[
	"sd",
	-6.106249171332308
	],
	[
	"gq",
	-6.1155110312256795
	],
	[
	"eq",
	-6.147800209212976
	],
	[
	"ln",
	-6.173015268781688
	],
	[
	"ng",
	-6.181732269229789
	],
	[
	"se",
	-6.198317089876394
	],
	[
	"sf",
	-6.200053977367045
	],
	[
	"na",
	-6.2011622959247745
	],
	[
	"fv",
	-6.21270323523432
	],
	[
	"et",
	-6.213211850957885
	],
	[
	"ed",
	-6.220732944550861
	],
	[
	"vf",
	-6.235612361132397
	],
	[
	"it",
	-6.236237188173279
	],
	[
	"hl",
	-6.242607870761024
	],
	[
	"rf",
	-6.242719164870595
	],
	[
	"ke",
	-6.243551778618629
	],
	[
	"fs",
	-6.247993226234106
	],
	[
	"an",
	-6.26119987064193
	],
	[
	"ma",
	-6.264278246418684
	],
	[
	"ep",
	-6.265933814263626
	],
	[
	"gn",
	-6.2690329158590625
	],
	[
	"yl",
	-6.273341632364284
	],
	[
	"fd",
	-6.276005658681607
	],
	[
	"td",
	-6.280432226339416
	],
	[
	"qv",
	-6.2888497616309
	],
	[
	"ha",
	-6.296053201668862
	],
	[
	"lh",
	-6.30246972284626
	],
	[
	"ki",
	-6.3175004329885045
	],
	[
	"ml",
	-6.328114226448536
	],
	[
	"hg",
	-6.33088519345314
	],
	[
	"gy",
	-6.333659448005497
	],
	[
	"es",
	-6.334612606722317
	],
	[
	"rk",
	-6.341736778775461
	],
	[
	"hr",
	-6.354408701195906
	],
	[
	"kv",
	-6.3558736445353805
	],
	[
	"di",
	-6.358575255803077
	],
	[
	"kt",
	-6.367157900996915
	],
	[
	"ah",
	-6.369617063929681
	],
	[
	"ks",
	-6.376100446248731
	],
	[
	"qq",
	-6.37798503837473
	],
	[
	"ir",
	-6.3857457573601675
	],
	[
	"rh",
	-6.387183443208819
	],
	[
	"kr",
	-6.389145725185193
	],
	[
	"ff",
	-6.394310366090611
	],
	[
	"np",
	-6.394909773835787
	],
	[
	"qp",
	-6.400762509720662
	],
	[
	"vk",
	-6.408006270334974
	],
	[
	"ip",
	-6.426055131001776
	],
	[
	"vq",
	-6.436536338470235
	],
	[
	"ya",
	-6.436780445351445
	],
	[
	"yg",
	-6.437618271718964
	],
	[
	"vn",
	-6.439633057236559
	],
	[
	"te",
	-6.4462825523394685
	],
	[
	"nv",
	-6.447181836880107
	],
	[
	"qg",
	-6.450639119164498
	],
	[
	"ay",
	-6.459484467855098
	],
	[
	"df",
	-6.464086065081403
	],
	[
	"gh",
	-6.466932879944524
	],
	[
	"tf",
	-6.469809893138768
	],
	[
	"sk",
	-6.479002868749177
	],
	[
	"ds",
	-6.485484141843836
	],
	[
	"sn",
	-6.4878031190687935
	],
	[
	"ii",
	-6.504258965341522
	],
	[
	"kg",
	-6.507398478812792
	],
	[
	"kp",
	-6.518085192976523
	],
	[
	"ly",
	-6.522152923737886
	],
	[
	"aaa",
	-6.526671396118413
	],
	[
	"ft",
	-6.5295453018170875
	],
	[
	"hp",
	-6.53078009472407
	],
	[
	"kd",
	-6.547001972884521
	],
	[
	"qi",
	-6.556091048164445
	],
	[
	"pi",
	-6.557328188755392
	],
	[
	"qs",
	-6.55968638147019
	],
	[
	"dt",
	-6.579466923820265
	],
	[
	"ns",
	-6.581085923267036
	],
	[
	"sq",
	-6.583571636718558
	],
	[
	"kn",
	-6.601121373063897
	],
	[
	"en",
	-6.606175960425073
	],
	[
	"fe",
	-6.60685998733843
	],
	[
	"tn",
	-6.615442326280007
	],
	[
	"wl",
	-6.618302734526351
	],
	[
	"pq",
	-6.621155961329553
	],
	[
	"ni",
	-6.624879415351742
	],
	[
	"yr",
	-6.625910436920403
	],
	[
	"qt",
	-6.6307505077769555
	],
	[
	"pf",
	-6.6308412369595455
	],
	[
	"rn",
	-6.631740741295436
	],
	[
	"in",
	-6.639584636482613
	],
	[
	"hv",
	-6.640247883065955
	],
	[
	"ik",
	-6.650348904814281
	],
	[
	"ry",
	-6.655536274374892
	],
	[
	"dq",
	-6.662650543901028
	],
	[
	"fr",
	-6.677637695424519
	],
	[
	"if",
	-6.681523029800585
	],
	[
	"mr",
	-6.701260323121653
	],
	[
	"ef",
	-6.721187968715531
	],
	[
	"am",
	-6.722392548689049
	],
	[
	"ms",
	-6.722957219864604
	],
	[
	"qe",
	-6.723551837092796
	],
	[
	"sy",
	-6.728733788049803
	],
	[
	"mt",
	-6.739373333977653
	],
	[
	"yv",
	-6.742995012080062
	],
	[
	"eh",
	-6.74876854304447
	],
	[
	"vh",
	-6.750406955107264
	],
	[
	"nn",
	-6.7566005866277745
	],
	[
	"yd",
	-6.760908295045155
	],
	[
	"tk",
	-6.762557739296788
	],
	[
	"nr",
	-6.7630435456712235
	],
	[
	"nt",
	-6.767721256103645
	],
	[
	"mv",
	-6.769331033226527
	],
	[
	"dk",
	-6.781027506796054
	],
	[
	"nd",
	-6.7895149322464246
	],
	[
	"mg",
	-6.7954010994947005
	],
	[
	"kq",
	-6.796318304309551
	],
	[
	"qk",
	-6.80278943044512
	],
	[
	"gw",
	-6.810481762215055
	],
	[
	"dy",
	-6.813654384938911
	],
	[
	"ys",
	-6.822655724067237
	],
	[
	"vy",
	-6.82632505735617
	],
	[
	"wr",
	-6.829081855085716
	],
	[
	"mp",
	-6.839761205069106
	],
	[
	"fi",
	-6.85048550044306
	],
	[
	"aw",
	-6.863259418010875
	],
	[
	"pk",
	-6.865582196294758
	],
	[
	"hd",
	-6.884016503575918
	],
	[
	"ala",
	-6.885081628055053
	],
	[
	"pn",
	-6.886470616505994
	],
	[
	"fp",
	-6.888007121641019
	],
	[
	"ty",
	-6.89272350275464
	],
	[
	"cg",
	-6.893033028167876
	],
	[
	"tq",
	-6.894648525208451
	],
	[
	"ac",
	-6.922264253254053
	],
	[
	"dh",
	-6.940052819724661
	],
	[
	"th",
	-6.946611361274389
	],
	[
	"nf",
	-6.94773061409305
	],
	[
	"yt",
	-6.947895582263566
	],
	[
	"ne",
	-6.950186435523683
	],
	[
	"laa",
	-6.9618966428821984
	],
	[
	"mk",
	-6.963876569469246
	],
	[
	"fn",
	-6.9651216440287556
	],
	[
	"mi",
	-6.973364756865921
	],
	[
	"rw",
	-6.976192205590854
	],
	[
	"sh",
	-6.976252466655415
	],
	[
	"yf",
	-6.98466102259021
	],
	[
	"ye",
	-6.985376860426236
	],
	[
	"iy",
	-6.997339619581055
	],
	[
	"ph",
	-7.002791342761229
	],
	[
	"ca",
	-7.00714306958699
	],
	[
	"nk",
	-7.01679175195992
	],
	[
	"vc",
	-7.026717686171381
	],
	[
	"dn",
	-7.0329619448909995
	],
	[
	"lc",
	-7.04162123422458
	],
	[
	"iq",
	-7.052050568293268
	],
	[
	"qd",
	-7.056415484136933
	],
	[
	"ht",
	-7.057894344010668
	],
	[
	"cr",
	-7.062175308655888
	],
	[
	"ey",
	-7.067369296531744
	],
	[
	"py",
	-7.06762170686601
	],
	[
	"gc",
	-7.068569810281758
	],
	[
	"he",
	-7.068724837052839
	],
	[
	"ws",
	-7.083834967007803
	],
	[
	"qn",
	-7.094736754675786
	],
	[
	"hs",
	-7.097107793695214
	],
	[
	"gm",
	-7.102912164950451
	],
	[
	"cl",
	-7.117371301671321
	],
	[
	"hh",
	-7.119373071454689
	],
	[
	"cs",
	-7.123202520839005
	],
	[
	"nq",
	-7.123573885267396
	],
	[
	"me",
	-7.140295705363691
	],
	[
	"fy",
	-7.145687063516517
	],
	[
	"ggg",
	-7.148129052637575
	],
	[
	"aal",
	-7.149286650265301
	],
	[
	"yp",
	-7.156139099185067
	],
	[
	"qf",
	-7.157060249855512
	],
	[
	"wa",
	-7.171499064266852
	],
	[
	"ky",
	-7.1761559132310495
	],
	[
	"ny",
	-7.181344251441002
	],
	[
	"lw",
	-7.18428827548775
	],
	[
	"sc",
	-7.188554190683416
	],
	[
	"md",
	-7.189176149884691
	],
	[
	"rc",
	-7.2023406563778956
	],
	[
	"aag",
	-7.215624965192319
	],
	[
	"lm",
	-7.21766907832032
	],
	[
	"kf",
	-7.220208888998105
	],
	[
	"sw",
	-7.235424094169154
	],
	[
	"yy",
	-7.248760750503438
	],
	[
	"rrr",
	-7.256636321517087
	],
	[
	"fk",
	-7.26446436072386
	],
	[
	"qh",
	-7.272573702000296
	],
	[
	"fq",
	-7.277283326780614
	],
	[
	"yn",
	-7.279879376992396
	],
	[
	"yq",
	-7.282357112300778
	],
	[
	"hf",
	-7.294922693842759
	],
	[
	"aga",
	-7.3089624623105145
	],
	[
	"mn",
	-7.309472360282593
	],
	[
	"ih",
	-7.33053451667163
	],
	[
	"ava",
	-7.335882965161225
	],
	[
	"cv",
	-7.339016321207151
	],
	[
	"hq",
	-7.343025081428985
	],
	[
	"wt",
	-7.3434986168208845
	],
	[
	"mq",
	-7.354400689372477
	],
	[
	"wg",
	-7.36382580580741
	],
	[
	"yi",
	-7.3644973873263915
	],
	[
	"dw",
	-7.366350376841645
	],
	[
	"em",
	-7.374811392160398
	],
	[
	"lla",
	-7.392768462701696
	],
	[
	"vaa",
	-7.41721359052082
	],
	[
	"lll",
	-7.425657863607588
	],
	[
	"wv",
	-7.428509738612526
	],
	[
	"yk",
	-7.431757248014991
	],
	[
	"vm",
	-7.433809580239302
	],
	[
	"vw",
	-7.445818765018977
	],
	[
	"aar",
	-7.4717401761921565
	],
	[
	"hi",
	-7.486342911357944
	],
	[
	"tw",
	-7.488981567298911
	],
	[
	"cp",
	-7.492932113247878
	],
	[
	"fh",
	-7.504062101936077
	],
	[
	"wi",
	-7.50616719068525
	],
	[
	"qy",
	-7.513321084614569
	],
	[
	"wp",
	-7.518799984375065
	],
	[
	"all",
	-7.53302752114244
	],
	[
	"gag",
	-7.533965743298678
	],
	[
	"ara",
	-7.537367311945358
	],
	[
	"pw",
	-7.5446201708160405
	],
	[
	"raa",
	-7.5669264012957616
	],
	[
	"kh",
	-7.577026433974584
	],
	[
	"wq",
	-7.581697650093153
	],
	[
	"lar",
	-7.584606758968199
	],
	[
	"lag",
	-7.590419078832234
	],
	[
	"tc",
	-7.59811062974657
	],
	[
	"ppp",
	-7.611586621905673
	],
	[
	"mf",
	-7.616872823329912
	],
	[
	"cd",
	-7.624203031434293
	],
	[
	"lgl",
	-7.635044047159843
	],
	[
	"rar",
	-7.648183656824516
	],
	[
	"lal",
	-7.656940108094723
	],
	[
	"arr",
	-7.661151098652345
	],
	[
	"ic",
	-7.683928182319702
	],
	[
	"wd",
	-7.688213319192755
	],
	[
	"fc",
	-7.688873646632947
	],
	[
	"hy",
	-7.701008250661593
	],
	[
	"wn",
	-7.707791318520682
	],
	[
	"ew",
	-7.708192951886026
	],
	[
	"wk",
	-7.727612230894543
	],
	[
	"vla",
	-7.729698852587845
	],
	[
	"agg",
	-7.737282862442257
	],
	[
	"rlr",
	-7.744727930333475
	],
	[
	"lae",
	-7.766894216302047
	],
	[
	"pap",
	-7.775081139986716
	],
	[
	"u",
	-17.47096861423773
	],
	[
	"b",
	-17.671612117272954
	],
	[
	"z",
	-18.91242536750401
	],
	[
	"o",
	-20.362425367503242
	]
	]
	}
	}