Initial model upload (NSP + CLS)

.gitattributes

@@ -15,3 +15,4 @@
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin filter=lfs diff=lfs merge=lfs -text

classifier_training.log

@@ -0,0 +1,57 @@
+Git commit: 01fe7efc56319afc04ab13136d485a680a1b8a82
+PyTorch CUDA version: 10.2
+Parameter pretrained: ../bert_reduced_intermediate_pe_50_epochs_VDJdb_PIRD
+Parameter data: ['PIRD']
+Parameter segment: TRB
+Parameter outdir: /home/wukevin/projects/tcr/tcr_models/temp3
+Parameter downsample: 1.0
+Parameter monitor: auprc
+Parameter bs: 128
+Parameter lr: 5e-05
+Parameter epochs: 25
+Parameter warmup: 0.1
+PIRD data 0.1655 data labelled with antigen sequence
+PIRD: Removing 95 entires with non amino acid residues
+Entries with antigen sequence: 8429/51044
+Unique antigen sequences: 73
+PIRD data TRA/TRB instances: Counter({'TRB': 46428, 'TRA': 4011, 'TRA-TRB': 605})
+No alphabet given, assuming alphabet of: ['TAAQAAVVRFQEAAN', 'ADTLQSIGATTVASN', 'LLWNGPMAV', 'RPRGEVRFL', 'ATDALMTGY', 'CINGVCWTV', 'HSKKKCDEL', 'KAFSPEVIPMF', 'KRWIILGLNK', 'KRWIIMGLNK', 'TPQDLNTML', 'EIYKRWII', 'GLNKIVRMY', 'ISPRTL-W', 'QASQEVKNW', 'FLKEKGGL', 'HPKVSSEVHI', 'IIKDYGKQM', 'LPPIVAKEI', 'RFPLTFGWCF', 'RYPLTFGWCF', 'TPGPGVRYPL', 'TQGYFPDWQNY', 'FPRPWLHGL', 'RYPLTFGWCF ', 'RYPLTFGW ', 'RYPLTFGW', 'CVETMCNEY', 'DEEDAIAAY', 'ELKRKMIYM', 'ELRRKMMYM', 'LSEFCRVLCCYVLEE', 'QIKVRVDMV', 'QIKVRVKMV', 'VLEETSVML', 'YILEETSVM', 'ARNLVPMVATVQGQN', 'CPSQEPMSIYVY', 'EDVPSGKLFMHVTLG', 'FPTKDVAL', 'IPSINVHHY', 'NLVPMVATV', 'RPHERNGFTVL', 'TPRVTGGGAM', 'YSEHPTFTSQY', 'VTEHDTLLY', 'YLAMPFATPMEAELARRSLA', 'ALTPVVVTL', 'GLCTLVAML', 'YVLDHLIVV', 'EPLPQGQLTAY', 'GPEPLPQGQLTAY', 'LPEGLPQGQLTAY', 'LPEPLGQGQLTAY', 'LPEPLPQAQLTAY', 'LPEPLPQGGLTAY', 'LPEPLPQGQGTAY', 'LPEPLPQGQLGAY', 'LPEPLPQGQLTAY', 'LPEPLPQGQLTGY', 'RAKFKQLL', 'HPVGEADYFEY', 'FLRGRAYGL', 'EENLLDFVRF', 'AVFDRKSDAK', 'IVTDFSVIK', 'NFIRMVISNPAAT', 'KRGIVEQCCTSISSL', 'KRGIVEQSSTSISSL', 'ENPVVHFFKNIVTPR', 'GILGFVFTL', 'PQPELPYPQPE', 'FWIDLFETIG']
+Generated labels of shape (6235,): [ 0  0  0 ... 72 72 72]
+Retaining 44/73 examples with sufficient examples
+Split train with 4365 examples
+Split valid with 935 examples
+Split test with 935 examples
+Loading BERT classifier with 45 labels
+Best AUROC (index=27): 0.9405
+Best AUROC (index=9): 0.9882
+Best AUROC (index=9): 0.9989
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=15): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=9): 1.0000
+Best AUROC (index=21): 1.0000
+Best AUROC (index=9): 1.0000
+epoch	25.0000
+eval_accuracy	0.7401
+eval_auprc	0.3922
+eval_auroc	0.7881
+eval_loss	1.5623
+eval_runtime	1.3134
+eval_samples_per_second	711.9190

config.json

@@ -0,0 +1,118 @@
+{
+  "_name_or_path": "/home/wukevin/projects/tcr/tcr_models/temp3/checkpoint-805",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LLWNGPMAV",
+    "1": "RPRGEVRFL",
+    "2": "ATDALMTGY",
+    "3": "HSKKKCDEL",
+    "4": "KAFSPEVIPMF",
+    "5": "KRWIILGLNK",
+    "6": "KRWIIMGLNK",
+    "7": "TPQDLNTML",
+    "8": "EIYKRWII",
+    "9": "ISPRTL-W",
+    "10": "FLKEKGGL",
+    "11": "HPKVSSEVHI",
+    "12": "IIKDYGKQM",
+    "13": "LPPIVAKEI",
+    "14": "RFPLTFGWCF",
+    "15": "RYPLTFGWCF",
+    "16": "TPGPGVRYPL",
+    "17": "TQGYFPDWQNY",
+    "18": "FPRPWLHGL",
+    "19": "RYPLTFGW",
+    "20": "ELRRKMMYM",
+    "21": "QIKVRVDMV",
+    "22": "QIKVRVKMV",
+    "23": "VLEETSVML",
+    "24": "FPTKDVAL",
+    "25": "NLVPMVATV",
+    "26": "RPHERNGFTVL",
+    "27": "TPRVTGGGAM",
+    "28": "VTEHDTLLY",
+    "29": "YLAMPFATPMEAELARRSLA",
+    "30": "GLCTLVAML",
+    "31": "YVLDHLIVV",
+    "32": "EPLPQGQLTAY",
+    "33": "RAKFKQLL",
+    "34": "HPVGEADYFEY",
+    "35": "FLRGRAYGL",
+    "36": "AVFDRKSDAK",
+    "37": "IVTDFSVIK",
+    "38": "NFIRMVISNPAAT",
+    "39": "KRGIVEQSSTSISSL",
+    "40": "ENPVVHFFKNIVTPR",
+    "41": "GILGFVFTL",
+    "42": "PQPELPYPQPE",
+    "43": "FWIDLFETIG",
+    "44": "other"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_10": 10,
+    "LABEL_11": 11,
+    "LABEL_12": 12,
+    "LABEL_13": 13,
+    "LABEL_14": 14,
+    "LABEL_15": 15,
+    "LABEL_16": 16,
+    "LABEL_17": 17,
+    "LABEL_18": 18,
+    "LABEL_19": 19,
+    "LABEL_2": 2,
+    "LABEL_20": 20,
+    "LABEL_21": 21,
+    "LABEL_22": 22,
+    "LABEL_23": 23,
+    "LABEL_24": 24,
+    "LABEL_25": 25,
+    "LABEL_26": 26,
+    "LABEL_27": 27,
+    "LABEL_28": 28,
+    "LABEL_29": 29,
+    "LABEL_3": 3,
+    "LABEL_30": 30,
+    "LABEL_31": 31,
+    "LABEL_32": 32,
+    "LABEL_33": 33,
+    "LABEL_34": 34,
+    "LABEL_35": 35,
+    "LABEL_36": 36,
+    "LABEL_37": 37,
+    "LABEL_38": 38,
+    "LABEL_39": 39,
+    "LABEL_4": 4,
+    "LABEL_40": 40,
+    "LABEL_41": 41,
+    "LABEL_42": 42,
+    "LABEL_43": 43,
+    "LABEL_44": 44,
+    "LABEL_5": 5,
+    "LABEL_6": 6,
+    "LABEL_7": 7,
+    "LABEL_8": 8,
+    "LABEL_9": 9
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 64,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 21,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.4.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 26
+}

custom_metrics.json

@@ -0,0 +1,2252 @@
+[
+    {
+        "auroc_per_label": [
+            0.6524861878453039,
+            0.599352051835853,
+            0.607391408687305,
+            0.3595596133190118,
+            0.6973522167487685,
+            0.5608810299364044,
+            0.635048231511254,
+            0.6487513572204127,
+            0.4395810955961332,
+            0.8092175777063236,
+            0.6266933497536946,
+            0.4981527093596059,
+            0.5849462365591397,
+            0.6192266380236304,
+            0.674054054054054,
+            0.6898496240601504,
+            0.7488648648648648,
+            0.30117899249732044,
+            0.5192926045016077,
+            0.33512352309344795,
+            0.44344086021505374,
+            0.5718884120171674,
+            0.4957173447537473,
+            0.6032477333417772,
+            0.2952688172043011,
+            0.5273382968822709,
+            0.48927472527472526,
+            0.9405144694533761,
+            0.7069102857711839,
+            0.5166130760986067,
+            0.458128078817734,
+            0.558555315574754,
+            0.44885550786838335,
+            0.5717344753747323,
+            0.7141327623126339,
+            0.9013933547695606,
+            0.5455519828510182,
+            0.3344062947067239,
+            0.7921286031042128,
+            0.7709003215434084,
+            0.7997858672376874,
+            0.5652524167561761
+        ],
+        "auprc_per_label": [
+            0.05116422396414794,
+            0.014307773439410278,
+            0.013751416997150906,
+            0.004792178174367253,
+            0.058524178525047786,
+            0.020614011118694132,
+            0.018768096867596736,
+            0.02910767839834261,
+            0.00488918596399621,
+            0.010187793427230047,
+            0.01675218907278232,
+            0.01061644372223641,
+            0.012175282534501192,
+            0.03417596999794957,
+            0.018110860474671996,
+            0.01480940865810795,
+            0.031201684749623447,
+            0.002283629073057393,
+            0.0033146278329941317,
+            0.00574548537174634,
+            0.0057746620753106595,
+            0.005812252637422484,
+            0.00211864406779661,
+            0.7500074151865133,
+            0.004345967583959583,
+            0.018542200747351283,
+            0.025495239295614515,
+            0.03359683794466403,
+            0.23319239127231015,
+            0.005507936166088773,
+            0.0077182183825156726,
+            0.02884659603265985,
+            0.007976016071843055,
+            0.0024937655860349127,
+            0.0037313432835820895,
+            0.015961098398169338,
+            0.004308538282284699,
+            0.003071845286899643,
+            0.5718266546399595,
+            0.030199085283831046,
+            0.005319148936170213,
+            0.04599621718631721
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8068876611418048,
+            0.794936405087593,
+            0.8196544276457884,
+            0.7040816326530612,
+            0.8034174876847291,
+            0.650767798976268,
+            0.4689174705251875,
+            0.6935784085621219,
+            0.6689044038668099,
+            0.9882100750267954,
+            0.6551724137931034,
+            0.6928879310344828,
+            0.5767741935483871,
+            0.7502685284640171,
+            0.7504864864864865,
+            0.7416756176154673,
+            0.9254054054054054,
+            0.4340836012861736,
+            0.5884244372990354,
+            0.6176154672395274,
+            0.5397849462365591,
+            0.6888412017167382,
+            0.24197002141327628,
+            0.701064986331444,
+            0.2378494623655914,
+            0.602218085931441,
+            0.6382857142857142,
+            0.5986066452304394,
+            0.6764413023996814,
+            0.447481243301179,
+            0.8462130541871921,
+            0.5855531557475402,
+            0.5296852646638054,
+            0.5524625267665952,
+            0.8640256959314775,
+            0.8381564844587353,
+            0.854769560557342,
+            0.31223175965665234,
+            0.8389269636498018,
+            0.8920150053590569,
+            0.9839400428265525,
+            0.6109022556390977
+        ],
+        "auprc_per_label": [
+            0.21210496703219345,
+            0.25614651864118004,
+            0.13261111121424057,
+            0.5031446747978188,
+            0.31553715279124683,
+            0.036314529824927784,
+            0.003851049459684123,
+            0.033643954472714795,
+            0.01093792263901621,
+            0.15000000000000002,
+            0.15443521464436338,
+            0.03525444492537657,
+            0.010688392982197158,
+            0.015540047746856633,
+            0.053871079386239536,
+            0.013705264074139824,
+            0.3863432251619167,
+            0.0027581170654982588,
+            0.004649641115120752,
+            0.007446671463862316,
+            0.008430584570530548,
+            0.011457380484355434,
+            0.0014104372355430183,
+            0.8121390586078836,
+            0.003952468410371326,
+            0.025476664506821324,
+            0.04910776337966042,
+            0.0065818986186520646,
+            0.3833488871840714,
+            0.002818389315567071,
+            0.18570815288854686,
+            0.06919011520649282,
+            0.024757317670093,
+            0.002386634844868735,
+            0.0078125,
+            0.010058485429061209,
+            0.012781497261107728,
+            0.0030021811755957984,
+            0.5846829123991921,
+            0.034730913642052566,
+            0.0625,
+            0.038122024125955396
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8389318600368324,
+            0.8483321334293257,
+            0.8840892728581713,
+            0.8163265306122449,
+            0.8486761083743842,
+            0.6871413060338143,
+            0.7583065380493034,
+            0.826198231735691,
+            0.6981740064446831,
+            0.9989281886387996,
+            0.8189655172413793,
+            0.7253694581280787,
+            0.6436559139784946,
+            0.813372717508056,
+            0.844108108108108,
+            0.949516648764769,
+            0.9231351351351351,
+            0.34994640943194,
+            0.6216505894962486,
+            0.7583243823845328,
+            0.5103225806451613,
+            0.7453505007153076,
+            0.20770877944325483,
+            0.7452371151430712,
+            0.3010752688172043,
+            0.7032728400806576,
+            0.6658021978021978,
+            0.8601286173633441,
+            0.7587374290550633,
+            0.18542336548767413,
+            0.8429802955665023,
+            0.7191024718022558,
+            0.8401287553648069,
+            0.6531049250535332,
+            0.4989293361884368,
+            0.9742765273311897,
+            0.914790996784566,
+            0.3726752503576538,
+            0.8350634952630518,
+            0.8587888531618435,
+            0.9946466809421842,
+            0.7373791621911923
+        ],
+        "auprc_per_label": [
+            0.33461722196949617,
+            0.5027700938207511,
+            0.4756079283458413,
+            0.5165600624024961,
+            0.3245631379846626,
+            0.05108546142758053,
+            0.019025157232704402,
+            0.19536225714580016,
+            0.13684949034586483,
+            0.75,
+            0.11025489301723249,
+            0.026297318295892158,
+            0.023112413812347785,
+            0.03146316646316646,
+            0.10366509116251005,
+            0.5179360403955396,
+            0.6224948105670295,
+            0.002421462913690893,
+            0.008707802369774201,
+            0.03078589677706745,
+            0.007980941386627076,
+            0.011689972967412804,
+            0.001349527665317139,
+            0.8378365042400386,
+            0.004314192189099314,
+            0.11155948142573162,
+            0.05112808180783865,
+            0.05947712418300653,
+            0.467509912097641,
+            0.0019146883048865356,
+            0.3547827409016069,
+            0.1577796574324055,
+            0.3406446540880503,
+            0.003076923076923077,
+            0.0021321961620469083,
+            0.06274131274131274,
+            0.04013605442176871,
+            0.003281083842935206,
+            0.5719650922999132,
+            0.02494855967078189,
+            0.16666666666666666,
+            0.03469355257145955
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.869171270718232,
+            0.8597312215022798,
+            0.9242860571154309,
+            0.958109559613319,
+            0.876847290640394,
+            0.7230494803784706,
+            0.890139335476956,
+            0.8610981852024197,
+            0.7599355531686358,
+            1.0,
+            0.8806958128078818,
+            0.8055726600985221,
+            0.7853763440860215,
+            0.7741675617615468,
+            0.8752432432432432,
+            0.9897959183673469,
+            0.9464864864864865,
+            0.5385852090032155,
+            0.680064308681672,
+            0.8305585392051557,
+            0.4649462365591398,
+            0.7353361945636624,
+            0.5535331905781585,
+            0.7774769639973402,
+            0.41161290322580646,
+            0.7512021095082985,
+            0.6632087912087912,
+            0.6404072883172561,
+            0.8127053669222344,
+            0.2282958199356913,
+            0.9237992610837439,
+            0.7550995920326373,
+            0.9062947067238911,
+            0.683083511777302,
+            0.7355460385438972,
+            0.9710610932475885,
+            0.905144694533762,
+            0.18776824034334763,
+            0.8754787341261843,
+            0.8963022508038585,
+            1.0,
+            0.8305585392051558
+        ],
+        "auprc_per_label": [
+            0.47822347460624137,
+            0.5452156131969919,
+            0.4958074291660544,
+            0.5643589743589744,
+            0.3448744720398437,
+            0.05388467563178169,
+            0.021306818181818184,
+            0.3866208078668693,
+            0.11341411733545226,
+            1.0,
+            0.11846245279413654,
+            0.05407341816295076,
+            0.029668402017548126,
+            0.03872993493731766,
+            0.24061690655734427,
+            0.5889423076923077,
+            0.655265616693963,
+            0.003442728442728443,
+            0.010323005106676275,
+            0.2641574474907808,
+            0.006291927680396545,
+            0.033988766923920506,
+            0.0023923444976076554,
+            0.8651475551600994,
+            0.00523489589202461,
+            0.21838595549937778,
+            0.0662648203320356,
+            0.030933834721103053,
+            0.5328878588075394,
+            0.002020817517081552,
+            0.3288327020240159,
+            0.2021405752206387,
+            0.35265265265265267,
+            0.003367003367003367,
+            0.004032258064516129,
+            0.05113636363636363,
+            0.047619047619047616,
+            0.0025618050955479705,
+            0.604585007977451,
+            0.021249076127124907,
+            1.0,
+            0.0398370846484054
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8798895027624308,
+            0.8610511159107271,
+            0.9505639548836092,
+            0.9658968850698174,
+            0.820966748768473,
+            0.7297192492632232,
+            0.9142550911039656,
+            0.802854040639057,
+            0.767454350161117,
+            1.0,
+            0.8976293103448275,
+            0.7838669950738916,
+            0.8025806451612904,
+            0.7873254564983888,
+            0.8727567567567568,
+            0.9927497314715359,
+            0.9605405405405405,
+            0.37084673097534837,
+            0.7867095391211147,
+            0.7696025778732546,
+            0.5273118279569893,
+            0.772889842632332,
+            0.8319057815845824,
+            0.7938581214443284,
+            0.4234408602150538,
+            0.7920738327904451,
+            0.7019340659340659,
+            0.579849946409432,
+            0.8521856019117794,
+            0.2722400857449089,
+            0.9119458128078818,
+            0.8837293016558675,
+            0.9184549356223175,
+            0.6648822269807281,
+            0.6820128479657387,
+            0.9780278670953912,
+            0.9619506966773849,
+            0.19563662374821175,
+            0.8842135322179668,
+            0.8877277599142551,
+            1.0,
+            0.8890977443609023
+        ],
+        "auprc_per_label": [
+            0.5569107120388834,
+            0.5673619603001047,
+            0.6395694941154976,
+            0.541617704661183,
+            0.37571772694082384,
+            0.06593920335177691,
+            0.07783882783882784,
+            0.4205319620371939,
+            0.07216289145636973,
+            1.0,
+            0.17811212783784908,
+            0.030967196753425408,
+            0.035392764996056314,
+            0.07024675576617959,
+            0.28724505590822547,
+            0.6053113553113554,
+            0.6538042371675818,
+            0.002477947593103611,
+            0.013463156629402572,
+            0.27704002279096984,
+            0.025018037660404016,
+            0.17355669902155693,
+            0.006329113924050633,
+            0.8733690912450873,
+            0.005908161932596818,
+            0.2800493025786717,
+            0.10401815825564532,
+            0.0346286701208981,
+            0.6212274363843072,
+            0.0021851666308833704,
+            0.3340201441976834,
+            0.21351630399481153,
+            0.3476719087264878,
+            0.0031847133757961785,
+            0.003355704697986577,
+            0.07291666666666666,
+            0.04083333333333333,
+            0.002547056997845693,
+            0.6108517150841762,
+            0.023262548262548262,
+            1.0,
+            0.08991601930940747
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8558011049723757,
+            0.8392128629709623,
+            0.9468442524598032,
+            0.9809344790547798,
+            0.832820197044335,
+            0.7480223359702186,
+            0.945337620578778,
+            0.8134791375833721,
+            0.8447905477980666,
+            1.0,
+            0.9067118226600985,
+            0.8180418719211822,
+            0.8270967741935483,
+            0.8778195488721804,
+            0.8889729729729731,
+            0.9948979591836734,
+            0.9598918918918918,
+            0.36173633440514474,
+            0.7138263665594855,
+            0.807733619763695,
+            0.5045161290322582,
+            0.8894849785407726,
+            0.6241970021413276,
+            0.8068696368069409,
+            0.4212903225806452,
+            0.7856367302621374,
+            0.6811428571428572,
+            0.72508038585209,
+            0.864781439808822,
+            0.18488745980707394,
+            0.9091748768472907,
+            0.8772498200143988,
+            0.9349070100143062,
+            0.6638115631691649,
+            0.778372591006424,
+            0.9715969989281886,
+            0.9823151125401929,
+            0.2367668097281831,
+            0.8735637976214473,
+            0.9268488745980706,
+            1.0,
+            0.8845327604726102
+        ],
+        "auprc_per_label": [
+            0.578310005897917,
+            0.5981528816751261,
+            0.6069518141948049,
+            0.6212273641851107,
+            0.33735692857707433,
+            0.07754235299680584,
+            0.06597222222222222,
+            0.37279861206549425,
+            0.08833506918613301,
+            1.0,
+            0.2858646389659709,
+            0.05649495346068106,
+            0.054583722377840024,
+            0.0620145785586962,
+            0.3031667489750782,
+            0.675,
+            0.701346247716274,
+            0.002474780871449273,
+            0.0192098832473527,
+            0.2970078915394517,
+            0.033399740670319006,
+            0.019731379731379735,
+            0.002840909090909091,
+            0.8807576245885824,
+            0.0053044367577905465,
+            0.2950183238176911,
+            0.05439594561145759,
+            0.029785809906291833,
+            0.6323245664516617,
+            0.0019128142480550375,
+            0.32091296245900364,
+            0.19875941646675388,
+            0.3651855390985826,
+            0.0031746031746031746,
+            0.004807692307692308,
+            0.05204216073781291,
+            0.08333333333333333,
+            0.002717366380988507,
+            0.5994515840597593,
+            0.025757575757575757,
+            1.0,
+            0.08230649717514124
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8673296500920811,
+            0.8288936885049196,
+            0.9330453563714902,
+            0.9825456498388828,
+            0.8589901477832512,
+            0.7471692259965876,
+            0.8949624866023581,
+            0.8515588645881806,
+            0.8082706766917294,
+            1.0,
+            0.9233374384236452,
+            0.7740147783251232,
+            0.8040860215053763,
+            0.7894736842105262,
+            0.8803243243243244,
+            0.9967776584317938,
+            0.9498378378378378,
+            0.44319399785637725,
+            0.7390139335476956,
+            0.8531149301825993,
+            0.5509677419354839,
+            0.9713876967095851,
+            0.5610278372591007,
+            0.8056954075762853,
+            0.5286021505376345,
+            0.825655343570653,
+            0.6976263736263736,
+            0.6763129689174705,
+            0.8677685950413223,
+            0.08520900321543406,
+            0.9436576354679803,
+            0.9053275737940965,
+            0.9513590844062948,
+            0.6702355460385439,
+            0.8329764453961457,
+            0.9640943193997856,
+            0.9812433011789925,
+            0.2925608011444922,
+            0.8658032654706713,
+            0.90005359056806,
+            1.0,
+            0.8284103114930182
+        ],
+        "auprc_per_label": [
+            0.6307155871109135,
+            0.6031665600033762,
+            0.663359782385032,
+            0.5675438596491228,
+            0.4104994719990741,
+            0.0907568384472507,
+            0.04111969111969112,
+            0.5089706454763225,
+            0.09553571428571428,
+            1.0,
+            0.4025723999907011,
+            0.09934172624708852,
+            0.07307692307692308,
+            0.10413890571370886,
+            0.29797544982126656,
+            0.6904761904761905,
+            0.6933501246196673,
+            0.0028736581337737406,
+            0.2520491803278688,
+            0.3152028845118932,
+            0.10519667718693702,
+            0.2111111111111111,
+            0.0024330900243309003,
+            0.8823253161210409,
+            0.007359691612406603,
+            0.321286388899459,
+            0.07322748394885456,
+            0.010299393789434615,
+            0.6183413335622824,
+            0.0017288698636243312,
+            0.41109813184125366,
+            0.24531401861195826,
+            0.37079341685089656,
+            0.003236245954692557,
+            0.006369426751592357,
+            0.04166666666666667,
+            0.527027027027027,
+            0.0028977716193411096,
+            0.589428967099884,
+            0.021954113924050632,
+            1.0,
+            0.12064264172274926
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8260773480662983,
+            0.8162946964242861,
+            0.9467242620590353,
+            0.9849624060150376,
+            0.8608374384236454,
+            0.7479447805180703,
+            0.9056806002143623,
+            0.7970373817279355,
+            0.8477443609022557,
+            1.0,
+            0.9239532019704434,
+            0.8636083743842364,
+            0.8750537634408603,
+            0.8842642320085929,
+            0.8694054054054055,
+            0.9994629430719656,
+            0.9755675675675676,
+            0.3815648445873527,
+            0.8687031082529476,
+            0.8504296455424275,
+            0.5662365591397849,
+            0.9535050071530757,
+            0.6027837259100642,
+            0.8133318556516049,
+            0.5455913978494624,
+            0.8419419885217931,
+            0.7002197802197803,
+            0.7068595927116827,
+            0.8818082246340736,
+            0.3397642015005359,
+            0.9501231527093595,
+            0.8671706263498921,
+            0.8626609442060086,
+            0.5685224839400428,
+            0.8950749464668094,
+            0.9715969989281887,
+            0.9930332261521972,
+            0.3834048640915594,
+            0.8617046294429885,
+            0.8914790996784566,
+            1.0,
+            0.8915145005370569
+        ],
+        "auprc_per_label": [
+            0.5954865946399535,
+            0.5970976588319198,
+            0.6346965232834797,
+            0.5739795918367347,
+            0.44892593021123717,
+            0.09151338923861635,
+            0.03398895790200138,
+            0.36745952167542095,
+            0.1365549701024214,
+            1.0,
+            0.4499694580674647,
+            0.10566482600781832,
+            0.06913919413919414,
+            0.08435287212162611,
+            0.2792237875847958,
+            0.9166666666666666,
+            0.7547809960181093,
+            0.0025298231180584125,
+            0.5040485829959515,
+            0.3226069233819011,
+            0.017921029030464573,
+            0.19676967696769676,
+            0.002688172043010753,
+            0.8889367509930737,
+            0.007795078163055398,
+            0.3384181994044785,
+            0.07868878100101934,
+            0.01446078431372549,
+            0.6510318084819748,
+            0.0024545213758562986,
+            0.538576692788283,
+            0.293549489971968,
+            0.46927083333333336,
+            0.0024752475247524753,
+            0.010101010101010102,
+            0.05208333333333333,
+            0.5666666666666667,
+            0.0034148746993775255,
+            0.6212650873340126,
+            0.01641025641025641,
+            1.0,
+            0.1755091521782444
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8611049723756907,
+            0.7932565394768418,
+            0.904127669786417,
+            0.985499462943072,
+            0.8814655172413793,
+            0.7447650069799907,
+            0.8783494105037513,
+            0.8182875756165658,
+            0.8719119226638024,
+            1.0,
+            0.927801724137931,
+            0.831896551724138,
+            0.9352688172043011,
+            0.8622448979591837,
+            0.8648648648648649,
+            0.9973147153598281,
+            0.9702702702702702,
+            0.519828510182208,
+            0.9426580921757771,
+            0.8147153598281417,
+            0.5959139784946237,
+            0.9563662374821174,
+            0.6948608137044968,
+            0.8215277118098434,
+            0.506021505376344,
+            0.8688537304172483,
+            0.6639560439560439,
+            0.6259378349410504,
+            0.8901722592850742,
+            0.2357984994640943,
+            0.978756157635468,
+            0.89464842812575,
+            0.9638769670958511,
+            0.6541755888650963,
+            0.8490364025695932,
+            0.9640943193997856,
+            0.982315112540193,
+            0.4695994277539342,
+            0.8779647920446146,
+            0.8952304394426581,
+            0.9999999999999999,
+            0.8869495166487648
+        ],
+        "auprc_per_label": [
+            0.5835550291334571,
+            0.6128707183587061,
+            0.6080132519560831,
+            0.5881818181818181,
+            0.45557450620585116,
+            0.13800539182293073,
+            0.04306983339241404,
+            0.4156859958102374,
+            0.15881062951496389,
+            1.0,
+            0.4423493229386553,
+            0.10780837711106084,
+            0.13254079254079254,
+            0.04297829208121007,
+            0.2918201074352089,
+            0.7048611111111112,
+            0.7199870129870128,
+            0.003446037646781141,
+            0.5091743119266054,
+            0.40147275405007365,
+            0.0727056987238661,
+            0.35931495405179614,
+            0.0034965034965034965,
+            0.8944805189179476,
+            0.006893387108954667,
+            0.3268017403901803,
+            0.0660758284040403,
+            0.011737823963941404,
+            0.6216616369477314,
+            0.0020444550590483094,
+            0.5514334914567833,
+            0.3772458581982392,
+            0.38611111111111107,
+            0.0030864197530864196,
+            0.007042253521126761,
+            0.04195804195804196,
+            0.5285714285714286,
+            0.0038525039008929333,
+            0.6224124379844913,
+            0.01827418865635426,
+            1.0,
+            0.2896798524192042
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8237937384898711,
+            0.8120950323974082,
+            0.918046556275498,
+            0.9914070891514499,
+            0.8295874384236454,
+            0.7404219016596867,
+            0.905144694533762,
+            0.8538855281526291,
+            0.8630504833512352,
+            1.0,
+            0.9465825123152709,
+            0.801570197044335,
+            0.7993548387096774,
+            0.9087003222341568,
+            0.8465945945945946,
+            0.9991944146079484,
+            0.951135135135135,
+            0.4523043944265809,
+            0.7904608788853161,
+            0.8230397422126746,
+            0.6436559139784946,
+            0.9710300429184548,
+            0.6616702355460385,
+            0.8087510422933621,
+            0.5563440860215054,
+            0.8778501628664495,
+            0.6378021978021978,
+            0.7031082529474812,
+            0.8874340336552822,
+            0.04823151125401931,
+            0.9298029556650247,
+            0.8717302615790737,
+            0.8265379113018598,
+            0.5877944325481799,
+            0.8522483940042827,
+            0.9689174705251875,
+            0.9876741693461951,
+            0.34942775393419173,
+            0.8708425720620843,
+            0.9268488745980707,
+            1.0,
+            0.8391514500537056
+        ],
+        "auprc_per_label": [
+            0.5757350828535388,
+            0.6150859878429475,
+            0.6556051115991076,
+            0.5916666666666667,
+            0.34744878770100857,
+            0.16135440625355435,
+            0.04444956241363427,
+            0.386840302491424,
+            0.14686450167973125,
+            1.0,
+            0.6524574368754593,
+            0.061697325232976434,
+            0.10700349561312128,
+            0.04432004776832363,
+            0.32448771350464206,
+            0.8928571428571428,
+            0.7404597559472451,
+            0.0029780869442084393,
+            0.5025445292620865,
+            0.41971556051310654,
+            0.10678310953987878,
+            0.3719887955182073,
+            0.0031545741324921135,
+            0.8832003323661889,
+            0.01189713089429532,
+            0.3821304725796221,
+            0.056821602456242175,
+            0.011780206153607687,
+            0.6565452252802171,
+            0.001674275091996611,
+            0.48738718148656035,
+            0.409910181935782,
+            0.5020576131687243,
+            0.0025906735751295338,
+            0.007194244604316547,
+            0.05347222222222223,
+            0.54,
+            0.0031521774011813017,
+            0.6097409388240062,
+            0.0225,
+            1.0,
+            0.30128737158560653
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8444935543278085,
+            0.7975761939044876,
+            0.8959683225341972,
+            0.9790547798066594,
+            0.8371305418719213,
+            0.7671785326508453,
+            0.90032154340836,
+            0.8636575151233131,
+            0.8447905477980666,
+            1.0,
+            0.9167179802955666,
+            0.8317426108374384,
+            0.8165591397849462,
+            0.8947368421052632,
+            0.8326486486486486,
+            1.0,
+            0.9150270270270271,
+            0.622722400857449,
+            0.8204715969989282,
+            0.8262620837808807,
+            0.5640860215053763,
+            0.9874821173104434,
+            0.3190578158458244,
+            0.8066479845476711,
+            0.5746236559139786,
+            0.8379866604622305,
+            0.6345054945054944,
+            0.6934619506966774,
+            0.8724484715722394,
+            0.10932475884244375,
+            0.9382697044334976,
+            0.8959683225341972,
+            0.7557224606580829,
+            0.3693790149892934,
+            0.8104925053533191,
+            0.969989281886388,
+            0.9903536977491961,
+            0.32045779685264664,
+            0.8400356110999128,
+            0.9284565916398713,
+            1.0,
+            0.8155209452201934
+        ],
+        "auprc_per_label": [
+            0.6346614607966194,
+            0.6134704956424126,
+            0.6854071256131014,
+            0.5513392857142857,
+            0.4055878681361773,
+            0.13009243918066568,
+            0.05107252298263534,
+            0.4451680874936199,
+            0.19306723821848376,
+            1.0,
+            0.5010778911363707,
+            0.12394190599445776,
+            0.15873117704713535,
+            0.046864262064806664,
+            0.3614303544839803,
+            1.0,
+            0.7789901038006474,
+            0.005606072798476436,
+            0.5029673590504451,
+            0.4765503875968992,
+            0.07208667114808538,
+            0.5277777777777778,
+            0.0015698587127158557,
+            0.8838420630904118,
+            0.01539848252710085,
+            0.3695644900638147,
+            0.0816364375240656,
+            0.010702431755063333,
+            0.6483230445614656,
+            0.001777306014785335,
+            0.483345836279315,
+            0.49487106080320126,
+            0.5014619883040936,
+            0.001694915254237288,
+            0.0056179775280898875,
+            0.05064102564102564,
+            0.55,
+            0.0030479526990802434,
+            0.586756186983493,
+            0.021749408983451537,
+            1.0,
+            0.18563945045719601
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8530018416206262,
+            0.7932565394768418,
+            0.9074874010079194,
+            0.9935553168635876,
+            0.8306650246305418,
+            0.7272374747944781,
+            0.8837084673097535,
+            0.8123158058011478,
+            0.8055853920515575,
+            0.9999999999999999,
+            0.9291871921182266,
+            0.8357450738916257,
+            0.8187096774193549,
+            0.8668098818474758,
+            0.7683243243243243,
+            1.0,
+            0.9451891891891893,
+            0.38692390139335475,
+            0.8965702036441586,
+            0.8128356605800214,
+            0.5913978494623656,
+            0.9821173104434906,
+            0.6991434689507495,
+            0.8019853709508882,
+            0.478494623655914,
+            0.891267256088103,
+            0.6592967032967032,
+            0.657556270096463,
+            0.8822065119984068,
+            0.09699892818863881,
+            0.8947044334975369,
+            0.8152147828173747,
+            0.8476394849785407,
+            0.7751605995717344,
+            0.8169164882226981,
+            0.964630225080386,
+            0.9903536977491961,
+            0.2811158798283262,
+            0.8764865954444668,
+            0.9113076098606645,
+            1.0,
+            0.8872180451127819
+        ],
+        "auprc_per_label": [
+            0.5909330041341262,
+            0.6203899166590348,
+            0.6234091650048057,
+            0.6123949579831933,
+            0.34579091146192276,
+            0.09510411705835889,
+            0.050239234449760764,
+            0.39023807716710396,
+            0.19476183028732294,
+            1.0,
+            0.6585253456221198,
+            0.08701369424017526,
+            0.12085567388909291,
+            0.06296371615785608,
+            0.3802777856110019,
+            1.0,
+            0.7825443458980044,
+            0.0025956100188526798,
+            0.2551546391752577,
+            0.5014409221902018,
+            0.10608294937407445,
+            0.44940476190476186,
+            0.0035460992907801418,
+            0.87505909534879,
+            0.00669783566455151,
+            0.32922529473549883,
+            0.06640466836944853,
+            0.010794486603685921,
+            0.6271944262448573,
+            0.0017551755175517552,
+            0.46243789574933053,
+            0.45117641133281705,
+            0.5023419203747073,
+            0.004739336492890996,
+            0.005813953488372093,
+            0.115625,
+            0.55,
+            0.002856035076609343,
+            0.6077747749494913,
+            0.019363636363636364,
+            1.0,
+            0.28364381520119225
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8188213627992633,
+            0.7910967122630189,
+            0.8861291096712264,
+            0.9656283566058002,
+            0.8308189655172413,
+            0.751900108577633,
+            0.9088960342979636,
+            0.8268962308050256,
+            0.8831901181525241,
+            1.0,
+            0.9045566502463054,
+            0.8472906403940887,
+            0.8576344086021506,
+            0.9191729323308271,
+            0.8097297297297298,
+            0.9943609022556391,
+            0.9441081081081081,
+            0.469989281886388,
+            0.770096463022508,
+            0.7851772287862514,
+            0.6103225806451613,
+            0.9935622317596566,
+            0.5535331905781585,
+            0.8002649272241749,
+            0.46150537634408606,
+            0.8387622149837135,
+            0.6238241758241758,
+            0.782422293676313,
+            0.8656775863785721,
+            0.12861736334405144,
+            0.9713669950738917,
+            0.8388528917686585,
+            0.8147353361945636,
+            0.3083511777301927,
+            0.658458244111349,
+            0.9812433011789925,
+            0.9908896034297964,
+            0.3315450643776824,
+            0.8606967681247061,
+            0.932743837084673,
+            1.0,
+            0.7720193340494093
+        ],
+        "auprc_per_label": [
+            0.6080142548736819,
+            0.6769515458780866,
+            0.5346451122424521,
+            0.5762463343108504,
+            0.31552624142896213,
+            0.09617227287762004,
+            0.027544742729306487,
+            0.3711607291725964,
+            0.12573317740194315,
+            1.0,
+            0.5640144378296928,
+            0.1465526308847871,
+            0.1673218077039733,
+            0.07697162403044756,
+            0.3451521730811155,
+            0.7291666666666666,
+            0.7604390479930436,
+            0.002943643176738402,
+            0.168997668997669,
+            0.6054135494596841,
+            0.10609407615940972,
+            0.5222222222222221,
+            0.0023923444976076554,
+            0.8734748944740289,
+            0.011529160258802258,
+            0.3912735358742126,
+            0.056853031950724435,
+            0.012785515320334262,
+            0.6514003962367889,
+            0.0018095705164882072,
+            0.5151983074396868,
+            0.4788366904639582,
+            0.6685860524632118,
+            0.0015455950540958269,
+            0.003125,
+            0.10368663594470046,
+            0.5526315789473684,
+            0.003079636467291876,
+            0.5743950544223382,
+            0.022989889263360617,
+            1.0,
+            0.30426532501269
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8150276243093921,
+            0.771418286537077,
+            0.9090472762179025,
+            0.9610633727175081,
+            0.8169642857142857,
+            0.7415076779897626,
+            0.8933547695605573,
+            0.8292228943694743,
+            0.8474758324382384,
+            1.0,
+            0.9000923645320197,
+            0.854371921182266,
+            0.9,
+            0.888829215896885,
+            0.8303783783783785,
+            0.9994629430719657,
+            0.9258378378378379,
+            0.36923901393354774,
+            0.9362272240085745,
+            0.7706766917293233,
+            0.6591397849462366,
+            0.9949928469241774,
+            0.7537473233404711,
+            0.8041306481745354,
+            0.4881720430107527,
+            0.902357685745308,
+            0.644087912087912,
+            0.6822079314040729,
+            0.8603504928806134,
+            0.19667738478027869,
+            0.947814039408867,
+            0.8323734101271898,
+            0.9148783977110158,
+            0.5546038543897217,
+            0.8747323340471093,
+            0.972668810289389,
+            0.9860664523043945,
+            0.26394849785407726,
+            0.8637875428341061,
+            0.9193461950696676,
+            1.0,
+            0.8168635875402792
+        ],
+        "auprc_per_label": [
+            0.6096887440711259,
+            0.6726708615417952,
+            0.647686901870645,
+            0.5903263403263403,
+            0.31400490611044646,
+            0.09039066808187188,
+            0.029334385687976847,
+            0.4134338711591108,
+            0.3359257714762302,
+            1.0,
+            0.6386699964271779,
+            0.11221367394249993,
+            0.25472300469483566,
+            0.06436094526645708,
+            0.34018235399076524,
+            0.8875,
+            0.7717125292596991,
+            0.0024934224967614878,
+            0.25833333333333336,
+            0.6886668611435239,
+            0.10800151113809107,
+            0.5666666666666667,
+            0.004329004329004329,
+            0.8717509774747727,
+            0.007449168223057073,
+            0.4039380305494798,
+            0.06761355320753001,
+            0.00834075834075834,
+            0.6194921391732015,
+            0.0019418246688371766,
+            0.497118815477438,
+            0.5109756715663288,
+            0.5597222222222222,
+            0.002398081534772182,
+            0.00847457627118644,
+            0.060714285714285714,
+            0.5357142857142857,
+            0.0027795383913374215,
+            0.6226424795246484,
+            0.018923611111111113,
+            1.0,
+            0.17097821902867774
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8302394106813996,
+            0.7823374130069594,
+            0.9151667866570674,
+            0.979860365198711,
+            0.7938731527093597,
+            0.7499612222739258,
+            0.8847802786709539,
+            0.8383744377229718,
+            0.8713748657357681,
+            1.0,
+            0.9042487684729064,
+            0.8571428571428571,
+            0.8451612903225806,
+            0.8888292158968851,
+            0.8207567567567567,
+            0.9994629430719656,
+            0.926054054054054,
+            0.49732047159699894,
+            0.9067524115755627,
+            0.7835660580021482,
+            0.6172043010752688,
+            0.9974964234620887,
+            0.4721627408993576,
+            0.805843175749132,
+            0.5563440860215054,
+            0.8745152784240732,
+            0.6189450549450549,
+            0.6945337620578778,
+            0.859205416708155,
+            0.15166130760986068,
+            0.9325738916256158,
+            0.839692824574034,
+            0.8830472103004292,
+            0.5246252676659529,
+            0.7644539614561028,
+            0.97481243301179,
+            0.9882100750267953,
+            0.30114449213161654,
+            0.8597560975609756,
+            0.9289924973204716,
+            1.0,
+            0.8026315789473685
+        ],
+        "auprc_per_label": [
+            0.5846359334468012,
+            0.639116143271426,
+            0.6513878046664932,
+            0.5970319634703196,
+            0.3598769222347024,
+            0.1321610264444671,
+            0.025987972508591063,
+            0.41633080220662,
+            0.3173482726423903,
+            1.0,
+            0.6610707631468876,
+            0.09704380830731821,
+            0.24945520581113803,
+            0.05488433807802657,
+            0.33854250311934697,
+            0.9166666666666666,
+            0.7445298180975498,
+            0.003253160816429913,
+            0.5056818181818182,
+            0.6887360939431397,
+            0.10623146141334353,
+            0.6095238095238096,
+            0.0020242914979757085,
+            0.877077277790747,
+            0.011366329761169296,
+            0.43267342162392375,
+            0.06534220321198556,
+            0.00884931506849315,
+            0.6583204071249459,
+            0.001856282353091593,
+            0.5168664034488133,
+            0.5768241594751472,
+            0.6696969696969697,
+            0.0022471910112359553,
+            0.004524886877828055,
+            0.0946843853820598,
+            0.5416666666666666,
+            0.0029621466600841567,
+            0.6213052689612916,
+            0.02145438121047877,
+            1.0,
+            0.1728609625668449
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8244567219152854,
+            0.7906167506599472,
+            0.8957283417326615,
+            0.9790547798066596,
+            0.8145012315270936,
+            0.741352567085466,
+            0.8660235798499464,
+            0.8387622149837135,
+            0.868421052631579,
+            1.0,
+            0.9210283251231526,
+            0.8415948275862069,
+            0.8470967741935483,
+            0.89312567132116,
+            0.8129729729729729,
+            0.998657357679914,
+            0.9324324324324325,
+            0.49464094319399776,
+            0.8440514469453375,
+            0.7784640171858217,
+            0.533763440860215,
+            0.9992846924177397,
+            0.4989293361884368,
+            0.8122156782031391,
+            0.610752688172043,
+            0.8668372886613929,
+            0.629010989010989,
+            0.6822079314040729,
+            0.864482724285572,
+            0.252411575562701,
+            0.9433497536945813,
+            0.8207343412526998,
+            0.8415593705293276,
+            0.4817987152034261,
+            0.7965738758029979,
+            0.9721329046087889,
+            0.9844587352625939,
+            0.3290414878397711,
+            0.854817577101391,
+            0.9193461950696677,
+            1.0,
+            0.8370032223415682
+        ],
+        "auprc_per_label": [
+            0.6245460338113129,
+            0.6742167156766905,
+            0.5782207357070814,
+            0.5883333333333334,
+            0.3833974879977599,
+            0.14348618969398017,
+            0.02608695652173913,
+            0.414857079928816,
+            0.31121596158521764,
+            1.0,
+            0.6899102744546095,
+            0.09190582520027811,
+            0.21819727891156462,
+            0.07140412506676527,
+            0.36465746094111706,
+            0.8611111111111112,
+            0.7156040107564987,
+            0.0031328228718114524,
+            0.5034129692832765,
+            0.688707729468599,
+            0.10500315796522938,
+            0.8666666666666667,
+            0.0021321961620469083,
+            0.8840336643094966,
+            0.014641100150533512,
+            0.4121762578338377,
+            0.08937954393374566,
+            0.007839300103603085,
+            0.6579949814332618,
+            0.002089459835938709,
+            0.4955032536872827,
+            0.5683194150650501,
+            0.6689088191330343,
+            0.002061855670103093,
+            0.005235602094240838,
+            0.09226190476190475,
+            0.532258064516129,
+            0.0031108656462315244,
+            0.5871006106185431,
+            0.01897163120567376,
+            1.0,
+            0.1709498847946718
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8202946593001841,
+            0.7843772498200143,
+            0.947204223662107,
+            0.9750268528464018,
+            0.8132697044334976,
+            0.7308825810454475,
+            0.8863879957127545,
+            0.8310842252210332,
+            0.8603651987110634,
+            1.0,
+            0.9247229064039408,
+            0.8648399014778325,
+            0.7780645161290323,
+            0.8874865735767992,
+            0.832,
+            0.998657357679914,
+            0.9437837837837838,
+            0.5042872454448017,
+            0.8118971061093248,
+            0.8039742212674544,
+            0.5675268817204301,
+            0.9985693848354793,
+            0.4646680942184154,
+            0.8046900562574545,
+            0.6144086021505376,
+            0.880409492787343,
+            0.6130989010989012,
+            0.7384780278670955,
+            0.8657771582196554,
+            0.11307609860664525,
+            0.8970135467980296,
+            0.8436525077993761,
+            0.8215307582260372,
+            0.5728051391862955,
+            0.6531049250535332,
+            0.9678456591639872,
+            0.9871382636655949,
+            0.27575107296137336,
+            0.8604616004837735,
+            0.9456055734190781,
+            1.0,
+            0.8096133190118152
+        ],
+        "auprc_per_label": [
+            0.6095467102059668,
+            0.6729959432083453,
+            0.7271014441746151,
+            0.6178955453149002,
+            0.3855619478560654,
+            0.12723721459679282,
+            0.021563814866760168,
+            0.40274845822903205,
+            0.3129894861066409,
+            1.0,
+            0.6860773155416012,
+            0.13939136443833194,
+            0.14796955262912115,
+            0.05500671111763203,
+            0.35371455209379044,
+            0.8611111111111112,
+            0.7247595146511555,
+            0.0033022226812417445,
+            0.16951566951566951,
+            0.7513623978201635,
+            0.2053868922080484,
+            0.7222222222222222,
+            0.001996007984031936,
+            0.8731327838935159,
+            0.017232617407773536,
+            0.437843505795063,
+            0.0835831822752599,
+            0.010154420374707259,
+            0.6340001839719918,
+            0.0017899265816846144,
+            0.48059938622960346,
+            0.5942656432820367,
+            0.6686586985391766,
+            0.0025,
+            0.003076923076923077,
+            0.08068181818181819,
+            0.5384615384615384,
+            0.002841917050808108,
+            0.6022044056615462,
+            0.027820121951219513,
+            1.0,
+            0.16289255469301342
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8409576427255985,
+            0.7938564914806816,
+            0.9329253659707224,
+            0.9790547798066596,
+            0.8271243842364532,
+            0.7123468279820072,
+            0.9126473740621651,
+            0.8252675663099117,
+            0.8528464017185822,
+            1.0,
+            0.9093288177339902,
+            0.8509852216748769,
+            0.7931182795698924,
+            0.8880236305048336,
+            0.8132972972972974,
+            0.9978517722878625,
+            0.9389189189189189,
+            0.4576634512325831,
+            0.8177920685959271,
+            0.7986036519871107,
+            0.5245161290322581,
+            0.9992846924177397,
+            0.5931477516059958,
+            0.804737553170155,
+            0.578494623655914,
+            0.8579959671164883,
+            0.6065494505494505,
+            0.7165058949624866,
+            0.8688638852932392,
+            0.12861736334405144,
+            0.9279556650246306,
+            0.8430525557955363,
+            0.8884120171673819,
+            0.4775160599571735,
+            0.5802997858672376,
+            0.9742765273311897,
+            0.9898177920685959,
+            0.32153075822603716,
+            0.8598232883155278,
+            0.9402465166130761,
+            1.0,
+            0.8053168635875403
+        ],
+        "auprc_per_label": [
+            0.5909574753091814,
+            0.6766685104790493,
+            0.6392883079157589,
+            0.6625,
+            0.33681298948426,
+            0.11853090611097887,
+            0.04499748617395676,
+            0.3618299630783356,
+            0.3292857142857143,
+            1.0,
+            0.663987555094633,
+            0.1354383878560225,
+            0.18716553812913558,
+            0.07268673355629877,
+            0.3033263913345615,
+            0.8333333333333333,
+            0.7404130591630591,
+            0.0030188902114358924,
+            0.5029239766081871,
+            0.7513262599469496,
+            0.20492569036240055,
+            0.8666666666666667,
+            0.0026246719160104987,
+            0.8772180590264202,
+            0.010193699131727855,
+            0.4287348355334175,
+            0.10758612366036377,
+            0.008056105954739805,
+            0.6312886117141098,
+            0.0018279715849961538,
+            0.4967187865526524,
+            0.6586616944462882,
+            0.6698412698412698,
+            0.002044989775051125,
+            0.002544529262086514,
+            0.07045454545454546,
+            0.5476190476190477,
+            0.0030747544308152466,
+            0.5828063347824851,
+            0.025603392041748205,
+            1.0,
+            0.16140172064871788
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8429097605893185,
+            0.7855771538276938,
+            0.9337652987760978,
+            0.9881847475832438,
+            0.8161945812807881,
+            0.7357685745307895,
+            0.8783494105037514,
+            0.836513106871413,
+            0.8345864661654137,
+            1.0,
+            0.9139470443349754,
+            0.854679802955665,
+            0.8094623655913978,
+            0.8719119226638024,
+            0.8130810810810811,
+            0.998657357679914,
+            0.9289729729729731,
+            0.5166130760986066,
+            0.8670953912111469,
+            0.8329752953813104,
+            0.5466666666666667,
+            0.9996423462088698,
+            0.41220556745182013,
+            0.8027980959015443,
+            0.567741935483871,
+            0.8614859624631612,
+            0.6055384615384617,
+            0.6977491961414791,
+            0.8594045603903215,
+            0.20418006430868169,
+            0.9687500000000001,
+            0.8442524598032157,
+            0.8369098712446351,
+            0.3147751605995718,
+            0.6595289079229122,
+            0.954983922829582,
+            0.9898177920685959,
+            0.2167381974248927,
+            0.8715480749848821,
+            0.9257770632368704,
+            1.0,
+            0.8012889366272825
+        ],
+        "auprc_per_label": [
+            0.6133402922461313,
+            0.673010636126359,
+            0.6533574378934276,
+            0.6717391304347826,
+            0.36056407085576175,
+            0.1053311395191679,
+            0.034106600386633525,
+            0.38287933479254777,
+            0.43021887502941875,
+            1.0,
+            0.6619303720894909,
+            0.10291434918519042,
+            0.14622330888345555,
+            0.061616821525358106,
+            0.33824498699996264,
+            0.8611111111111112,
+            0.759041620626151,
+            0.003521083860277544,
+            0.2540160642570281,
+            0.7515974440894568,
+            0.20516412441233234,
+            0.9166666666666665,
+            0.0018181818181818182,
+            0.8733189002301738,
+            0.013148643912609738,
+            0.40057132385447053,
+            0.068753421774349,
+            0.006527639727807936,
+            0.6241723933656417,
+            0.001958637582460405,
+            0.5101748433715647,
+            0.6754783040209764,
+            0.6688453159041394,
+            0.0015600624024961,
+            0.003134796238244514,
+            0.04722222222222222,
+            0.5476190476190477,
+            0.0026662152548549145,
+            0.6194186033669273,
+            0.0205947357846092,
+            1.0,
+            0.15303524550101405
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8511602209944751,
+            0.7876169906407487,
+            0.9403647708183345,
+            0.9916756176154672,
+            0.8371305418719212,
+            0.7414301225376144,
+            0.879957127545552,
+            0.831627113386071,
+            0.8689581095596133,
+            1.0,
+            0.9279556650246304,
+            0.8528325123152709,
+            0.7533333333333333,
+            0.852577873254565,
+            0.832,
+            0.9965091299677765,
+            0.9458378378378378,
+            0.4967845659163987,
+            0.8129689174705251,
+            0.8085392051557465,
+            0.5741935483870968,
+            0.9992846924177397,
+            0.35224839400428265,
+            0.8072100313479624,
+            0.5733333333333334,
+            0.8783930510314876,
+            0.6190769230769231,
+            0.7808145766345123,
+            0.8605994224833217,
+            0.15916398713826369,
+            0.9775246305418719,
+            0.8414926805855532,
+            0.8351216022889842,
+            0.4464668094218416,
+            0.5845824411134903,
+            0.9431939978563774,
+            0.9823151125401929,
+            0.2743204577968526,
+            0.8626788953839952,
+            0.9386387995712755,
+            0.9999999999999999,
+            0.8302900107411386
+        ],
+        "auprc_per_label": [
+            0.5953631604188534,
+            0.6740187340975485,
+            0.618057283726655,
+            0.65625,
+            0.361296559319471,
+            0.11204134756614328,
+            0.02288226848528356,
+            0.3900594780921697,
+            0.39851190476190473,
+            1.0,
+            0.6719312169312169,
+            0.13431903854365979,
+            0.14276737967914438,
+            0.047351476202029,
+            0.3517997745824049,
+            0.8088235294117647,
+            0.7677155172413793,
+            0.003641141141141141,
+            0.5028490028490028,
+            0.75139470013947,
+            0.2054810359629189,
+            0.8666666666666667,
+            0.0016501650165016502,
+            0.8770494520873655,
+            0.013227412207105492,
+            0.425937093546246,
+            0.10501843418674243,
+            0.008165929133106237,
+            0.615577040120926,
+            0.001873536299765808,
+            0.5259126838074206,
+            0.675342836286448,
+            0.6688218390804598,
+            0.0019305019305019305,
+            0.002570694087403599,
+            0.030202821869488534,
+            0.5285714285714286,
+            0.0028396192365165943,
+            0.6062157665911121,
+            0.02472929470295581,
+            1.0,
+            0.16400495599418952
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8467771639042356,
+            0.7868970482361412,
+            0.91792656587473,
+            0.982545649838883,
+            0.8220443349753694,
+            0.7233597021870637,
+            0.8981779206859593,
+            0.8317046688382193,
+            0.8493555316863588,
+            1.0,
+            0.9154864532019704,
+            0.8629926108374384,
+            0.7789247311827957,
+            0.8823845327604726,
+            0.8142702702702703,
+            0.997046186895811,
+            0.9379459459459459,
+            0.4533762057877813,
+            0.8563772775991426,
+            0.7929645542427497,
+            0.5133333333333333,
+            0.9996423462088698,
+            0.5546038543897216,
+            0.8056188847724898,
+            0.5584946236559141,
+            0.8630370715061269,
+            0.6165714285714287,
+            0.7224008574490889,
+            0.8699591755451558,
+            0.11521972132904607,
+            0.9539716748768473,
+            0.8291336693064555,
+            0.9091559370529328,
+            0.5267665952890792,
+            0.5974304068522485,
+            0.9646302250803859,
+            0.9898177920685959,
+            0.22532188841201717,
+            0.8675838204663039,
+            0.9397106109324759,
+            1.0,
+            0.7951127819548872
+        ],
+        "auprc_per_label": [
+            0.6011965091058803,
+            0.6738298108657019,
+            0.5682144966216951,
+            0.6649253731343283,
+            0.3389659893417105,
+            0.1192631698973282,
+            0.02675049309664694,
+            0.40110504880618353,
+            0.34002298726720903,
+            1.0,
+            0.6547295987594496,
+            0.1199950917016565,
+            0.14582974653275946,
+            0.054858942666319715,
+            0.35752694362160176,
+            0.8166666666666667,
+            0.7529691875467907,
+            0.0029738250410639537,
+            0.5037037037037037,
+            0.7512903225806452,
+            0.20482597717582857,
+            0.9166666666666665,
+            0.002398081534772182,
+            0.8764835513572053,
+            0.014818632839577794,
+            0.43189433545428946,
+            0.09836016089211244,
+            0.007466107010179958,
+            0.6472403417492121,
+            0.001813056345112351,
+            0.509682792670797,
+            0.6736837012616017,
+            0.6705577172503242,
+            0.002257336343115124,
+            0.002652519893899204,
+            0.043478260869565216,
+            0.5476190476190477,
+            0.002690631914900831,
+            0.6117206583201258,
+            0.025252525252525252,
+            1.0,
+            0.19342288485546913
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8380847145488031,
+            0.7844972402207824,
+            0.9181665466762658,
+            0.9873791621911923,
+            0.8146551724137931,
+            0.7270048084380332,
+            0.905144694533762,
+            0.8268962308050257,
+            0.8399570354457573,
+            1.0,
+            0.9202586206896551,
+            0.8640701970443351,
+            0.812258064516129,
+            0.8818474758324382,
+            0.8139459459459459,
+            0.9967776584317938,
+            0.9395675675675675,
+            0.38210075026795287,
+            0.8981779206859593,
+            0.7926960257787325,
+            0.5391397849462366,
+            1.0,
+            0.43361884368308357,
+            0.8038298343940977,
+            0.553763440860215,
+            0.8572204125950055,
+            0.5908131868131868,
+            0.7368703108252947,
+            0.8628397889076969,
+            0.17524115755627007,
+            0.9068657635467979,
+            0.8224142068634509,
+            0.8555078683834049,
+            0.4989293361884368,
+            0.6927194860813705,
+            0.969989281886388,
+            0.9914255091103966,
+            0.2503576537911302,
+            0.8648961902842169,
+            0.9343515541264737,
+            1.0,
+            0.8133727175080557
+        ],
+        "auprc_per_label": [
+            0.6017521181913742,
+            0.6736616911124377,
+            0.6204295372541015,
+            0.6704081632653062,
+            0.32082917586408227,
+            0.11321428078575083,
+            0.02905638665132336,
+            0.34452901251347207,
+            0.32367808519982433,
+            1.0,
+            0.6563575605680868,
+            0.14257058835882416,
+            0.16539342472849716,
+            0.0579514561207925,
+            0.35346108219602,
+            0.8125,
+            0.7527411647047337,
+            0.002599144089522697,
+            0.5052083333333334,
+            0.7512886597938144,
+            0.2050596135798457,
+            1.0,
+            0.0018867924528301887,
+            0.8742204227018466,
+            0.014200448144226255,
+            0.4443071135839196,
+            0.08008062368644939,
+            0.008594147012226109,
+            0.6355413028336764,
+            0.0019011665014462445,
+            0.473629013667687,
+            0.6567360315289138,
+            0.669123669123669,
+            0.0021321961620469083,
+            0.003472222222222222,
+            0.0521680216802168,
+            0.5555555555555556,
+            0.0027778506531670293,
+            0.6149976749007984,
+            0.02312889812889813,
+            1.0,
+            0.16483932223196027
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8479558011049724,
+            0.7843772498200143,
+            0.9276457883369329,
+            0.9860365198711064,
+            0.8278940886699506,
+            0.7244454785171398,
+            0.8928188638799571,
+            0.82263068093687,
+            0.84828141783029,
+            1.0,
+            0.9208743842364532,
+            0.8705357142857142,
+            0.7954838709677419,
+            0.8711063372717509,
+            0.8072432432432433,
+            0.9965091299677766,
+            0.9401081081081082,
+            0.454983922829582,
+            0.8788853161843515,
+            0.8187432867883996,
+            0.547741935483871,
+            1.0,
+            0.42077087794432544,
+            0.8033522265497187,
+            0.5655913978494624,
+            0.858461299829378,
+            0.6087472527472527,
+            0.735262593783494,
+            0.8612466394503634,
+            0.14844587352625938,
+            0.9327278325123153,
+            0.8297336213102952,
+            0.8412017167381973,
+            0.5117773019271948,
+            0.6177730192719486,
+            0.9614147909967845,
+            0.9919614147909968,
+            0.23104434907010013,
+            0.8631156352885843,
+            0.9402465166130761,
+            1.0,
+            0.8120300751879699
+        ],
+        "auprc_per_label": [
+            0.6149264393481625,
+            0.6738306597217948,
+            0.6100892682431905,
+            0.6685185185185185,
+            0.33740476867075325,
+            0.12754155362464137,
+            0.019876912840984695,
+            0.38252728248287104,
+            0.3342034672341935,
+            1.0,
+            0.6721584081430692,
+            0.14626202750344428,
+            0.14640528634361236,
+            0.0533725882594417,
+            0.36465230266579735,
+            0.8088235294117647,
+            0.7460668591043843,
+            0.0030379456608964806,
+            0.5043859649122807,
+            0.7514727540500736,
+            0.2051711690941363,
+            1.0,
+            0.0018450184501845018,
+            0.8732603208471792,
+            0.014049475975768267,
+            0.43570261044993774,
+            0.08226001320409711,
+            0.0076546391752577315,
+            0.638323863756341,
+            0.0018592417204035764,
+            0.49090785952044713,
+            0.6730278671314351,
+            0.668903803131991,
+            0.002188183807439825,
+            0.002793296089385475,
+            0.04044117647058823,
+            0.5588235294117647,
+            0.0027201594082939752,
+            0.609894766936412,
+            0.02542892156862745,
+            1.0,
+            0.1698005489684343
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8454880294659299,
+            0.7858171346292296,
+            0.9205663546916246,
+            0.9774436090225564,
+            0.8298953201970443,
+            0.7232045912827672,
+            0.8965702036441586,
+            0.8288351171087327,
+            0.8458646616541354,
+            1.0,
+            0.9167179802955664,
+            0.8659174876847291,
+            0.7946236559139785,
+            0.8735230934479055,
+            0.8088648648648649,
+            0.9962406015037594,
+            0.9457297297297298,
+            0.4453376205787781,
+            0.8992497320471597,
+            0.800483351235231,
+            0.5389247311827957,
+            1.0,
+            0.43040685224839403,
+            0.80649493894008,
+            0.5621505376344086,
+            0.8585388552815263,
+            0.6008351648351649,
+            0.737406216505895,
+            0.8592552026286966,
+            0.1538049303322615,
+            0.9521243842364532,
+            0.8290136789056876,
+            0.8537195994277539,
+            0.47430406852248397,
+            0.6563169164882228,
+            0.9603429796355841,
+            0.9919614147909968,
+            0.24713876967095846,
+            0.8660048377343278,
+            0.942390139335477,
+            1.0,
+            0.8157894736842104
+        ],
+        "auprc_per_label": [
+            0.6074677123780654,
+            0.6739676394211147,
+            0.6282476239248467,
+            0.6616279069767441,
+            0.33732209414553205,
+            0.11495870272988419,
+            0.024435678281832126,
+            0.38827991695766906,
+            0.32528592883548985,
+            1.0,
+            0.671503009185936,
+            0.14791731664248264,
+            0.14741029048794382,
+            0.054165233344799904,
+            0.34720024240495695,
+            0.7463235294117647,
+            0.7682814992025518,
+            0.0029966535512015577,
+            0.5052631578947369,
+            0.751338688085676,
+            0.20508010509276334,
+            1.0,
+            0.001876172607879925,
+            0.8762417129537968,
+            0.014245019481236128,
+            0.43787158649939395,
+            0.10214019880401648,
+            0.008036945133719327,
+            0.6453092558008503,
+            0.0018648426233242484,
+            0.5005768264752709,
+            0.6568450199936421,
+            0.669093851132686,
+            0.0020325203252032522,
+            0.003105590062111801,
+            0.03826530612244898,
+            0.5588235294117647,
+            0.002754986679387497,
+            0.6106182725181926,
+            0.02639344262295082,
+            1.0,
+            0.17292476754015215
+        ]
+    },
+    {
+        "auroc_per_label": [
+            0.8432780847145488,
+            0.7859371250299976,
+            0.9215262778977683,
+            0.9750268528464017,
+            0.8326662561576355,
+            0.7211105940747634,
+            0.8981779206859593,
+            0.829998448890957,
+            0.8404940923737917,
+            0.9999999999999999,
+            0.9188731527093597,
+            0.8642241379310345,
+            0.7926881720430108,
+            0.8805048335123523,
+            0.8107027027027027,
+            0.9962406015037594,
+            0.9447567567567567,
+            0.4571275455519829,
+            0.8767416934619507,
+            0.800483351235231,
+            0.5393548387096775,
+            1.0,
+            0.42505353319057815,
+            0.8058352595970151,
+            0.5572043010752687,
+            0.8602450752287886,
+            0.6026373626373628,
+            0.7491961414790997,
+            0.8604500647216968,
+            0.135048231511254,
+            0.9496613300492611,
+            0.8321334293256539,
+            0.8505007153075822,
+            0.4914346895074947,
+            0.6466809421841542,
+            0.9635584137191855,
+            0.9914255091103965,
+            0.2378397711015737,
+            0.8657024793388429,
+            0.9434619506966774,
+            1.0,
+            0.8171321160042965
+        ],
+        "auprc_per_label": [
+            0.6066162147496059,
+            0.6739386142736772,
+            0.6317348927875245,
+            0.6605263157894736,
+            0.33325192028614753,
+            0.11429272374588151,
+            0.024542614904060685,
+            0.39381682934968976,
+            0.32010427712972783,
+            1.0,
+            0.6622675769985169,
+            0.14801059751949497,
+            0.14765298776097915,
+            0.05189256307100677,
+            0.3480631370380245,
+            0.7463235294117647,
+            0.7678012241963256,
+            0.0030561893631624835,
+            0.5043103448275862,
+            0.751338688085676,
+            0.2050741945649132,
+            1.0,
+            0.0018587360594795538,
+            0.8757548645041626,
+            0.014570951005455957,
+            0.43766105633660846,
+            0.08168294096770064,
+            0.008021846731524151,
+            0.6448212946063439,
+            0.0018369909796774738,
+            0.49917136064206663,
+            0.6729762484142603,
+            0.6690419635787807,
+            0.0021008403361344537,
+            0.0030211480362537764,
+            0.04380952380952381,
+            0.5555555555555556,
+            0.0027187591186705116,
+            0.6128637773430864,
+            0.02687074829931973,
+            1.0,
+            0.18124943520694017
+        ]
+    }
+]

params.json

@@ -0,0 +1,14 @@
+{
+    "pretrained": "../bert_reduced_intermediate_pe_50_epochs_VDJdb_PIRD",
+    "data": [
+        "PIRD"
+    ],
+    "segment": "TRB",
+    "outdir": "/home/wukevin/projects/tcr/tcr_models/temp3",
+    "downsample": 1.0,
+    "monitor": "auprc",
+    "bs": 128,
+    "lr": 5e-05,
+    "epochs": 25,
+    "warmup": 0.1
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eeaa2af01f05cada4ab2f1ce3c0062574768551a04d10e1cca14aab63c780d4d
+size 229773257

trained_labels.txt

@@ -0,0 +1,45 @@
+LLWNGPMAV
+RPRGEVRFL
+ATDALMTGY
+HSKKKCDEL
+KAFSPEVIPMF
+KRWIILGLNK
+KRWIIMGLNK
+TPQDLNTML
+EIYKRWII
+ISPRTL-W
+FLKEKGGL
+HPKVSSEVHI
+IIKDYGKQM
+LPPIVAKEI
+RFPLTFGWCF
+RYPLTFGWCF
+TPGPGVRYPL
+TQGYFPDWQNY
+FPRPWLHGL
+RYPLTFGW
+ELRRKMMYM
+QIKVRVDMV
+QIKVRVKMV
+VLEETSVML
+FPTKDVAL
+NLVPMVATV
+RPHERNGFTVL
+TPRVTGGGAM
+VTEHDTLLY
+YLAMPFATPMEAELARRSLA
+GLCTLVAML
+YVLDHLIVV
+EPLPQGQLTAY
+RAKFKQLL
+HPVGEADYFEY
+FLRGRAYGL
+AVFDRKSDAK
+IVTDFSVIK
+NFIRMVISNPAAT
+KRGIVEQSSTSISSL
+ENPVVHFFKNIVTPR
+GILGFVFTL
+PQPELPYPQPE
+FWIDLFETIG
+other

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f9573751a3e2b1e00be090074d5169b6b93f9e5a73fbde50f18c64af4f54d54
+size 2351