Added "Active-OR" report and modified report name

notebooks/protac_degradation_predictor.py

@@ -767,7 +767,7 @@ def objective(
     # Generate the hyperparameters
     hidden_dim = trial.suggest_categorical('hidden_dim', hidden_dim_options)
     batch_size = trial.suggest_categorical('batch_size', batch_size_options)
-    learning_rate = trial.suggest_loguniform('learning_rate', *learning_rate_options)
+    learning_rate = trial.suggest_float('learning_rate', *learning_rate_options, log=True)
     max_epochs = trial.suggest_categorical('max_epochs', max_epochs_options)
     smote_k_neighbors = trial.suggest_categorical('smote_k_neighbors', smote_k_neighbors_options)
 
@@ -939,5 +939,5 @@ for group_type in ['random', 'uniprot', 'tanimoto']:
         report.append(stats)
 report = pd.DataFrame(report)
 report.to_csv(
-    f'../reports/cv_report_hparam_search_{n_splits}-splits.csv', index=False,
+    f'../reports/cv_report_hparam_search_{n_splits}-splits_{active_col.replace(' ', '').lower()}.csv', index=False,
 )

reports/cv_report_hparam_search_5-splits.csv

@@ -1,16 +0,0 @@
-fold,group_type,train_len,val_len,train_perc,val_perc,train_active_perc,train_inactive_perc,val_active_perc,val_inactive_perc,test_active_perc,test_inactive_perc,num_leaking_uniprot,num_leaking_smiles,val_loss,val_acc,val_f1_score,val_hp_metric,val_opt_score,val_precision,val_recall,val_roc_auc,test_loss,test_acc,test_f1_score,test_hp_metric,test_opt_score,test_precision,test_recall,test_roc_auc,hidden_dim,batch_size,learning_rate,max_epochs,smote_k_neighbors,train_unique_groups,val_unique_groups
-0,random,1297,325,0.7996300863131935,0.2003699136868064,0.5065535851966075,0.49344641480339246,0.5076923076923077,0.49230769230769234,0.5333333333333333,0.4666666666666667,60,154,0.46033793687820435,0.8030769228935242,0.8106508851051331,0.8030769228935242,1.6137278079986572,0.7919074892997742,0.8303030133247375,0.8807765245437622,1.3106863498687744,0.6000000238418579,0.5,0.6000000238418579,1.100000023841858,0.75,0.375,0.5892857313156128,768,16,1.7650765084287508e-05,50,13,,
-1,random,1297,325,0.7996300863131935,0.2003699136868064,0.5065535851966075,0.49344641480339246,0.5076923076923077,0.49230769230769234,0.5333333333333333,0.4666666666666667,61,169,0.44147610664367676,0.8092307448387146,0.822857141494751,0.8092307448387146,1.6320879459381104,0.7783783674240112,0.8727272748947144,0.8891098499298096,1.133339524269104,0.5333333611488342,0.4615384638309479,0.5333333611488342,0.9948718547821045,0.6000000238418579,0.375,0.5357142686843872,512,16,1.804776106620677e-05,50,3,,
-2,random,1298,324,0.8002466091245376,0.1997533908754624,0.5069337442218799,0.4930662557781202,0.5061728395061729,0.49382716049382713,0.5333333333333333,0.4666666666666667,59,155,0.45973488688468933,0.8179012537002563,0.8206686973571777,0.8179012537002563,1.638569951057434,0.8181818127632141,0.8231707215309143,0.8919017314910889,1.4019314050674438,0.46666666865348816,0.20000000298023224,0.46666666865348816,0.6666666865348816,0.5,0.125,0.5535714030265808,768,8,1.2804458790386793e-05,10,12,,
-3,random,1298,324,0.8002466091245376,0.1997533908754624,0.5069337442218799,0.4930662557781202,0.5061728395061729,0.49382716049382713,0.5333333333333333,0.4666666666666667,62,150,0.4237369894981384,0.8209876418113708,0.8304093480110168,0.8209876418113708,1.6513969898223877,0.7977527976036072,0.8658536672592163,0.8956174254417419,1.1620509624481201,0.5333333611488342,0.4615384638309479,0.5333333611488342,0.9948718547821045,0.6000000238418579,0.375,0.5714285969734192,256,8,2.558961887141644e-05,20,11,,
-4,random,1298,324,0.8002466091245376,0.1997533908754624,0.5069337442218799,0.4930662557781202,0.5061728395061729,0.49382716049382713,0.5333333333333333,0.4666666666666667,56,149,0.4890975058078766,0.7839506268501282,0.7941176295280457,0.7839506268501282,1.5780682563781738,0.7670454382896423,0.8231707215309143,0.8685213327407837,1.1361300945281982,0.5333333611488342,0.4615384638309479,0.5333333611488342,0.9948718547821045,0.6000000238418579,0.375,0.625,256,8,2.236552489550707e-05,20,14,,
-0,uniprot,1037,585,0.6393341553637485,0.36066584463625156,0.5255544840887174,0.47444551591128253,0.4735042735042735,0.5264957264957265,0.5333333333333333,0.4666666666666667,0,104,0.7222811579704285,0.5880342125892639,0.6016529202461243,0.5880342125892639,1.1896871328353882,0.5548780560493469,0.6570397019386292,0.6614234447479248,0.9520992636680603,0.46666666865348816,0.3333333432674408,0.46666666865348816,0.800000011920929,0.5,0.25,0.4821428656578064,768,16,1.065787533203932e-05,50,5,66.0,21.0
-1,uniprot,1259,363,0.7762022194821209,0.22379778051787916,0.5250198570293884,0.4749801429706116,0.44352617079889806,0.5564738292011019,0.5333333333333333,0.4666666666666667,0,17,0.719026505947113,0.5234159827232361,0.483582079410553,0.5234159827232361,1.006998062133789,0.4655172526836395,0.5031055808067322,0.5270586013793945,0.6793550252914429,0.6666666865348816,0.7058823704719543,0.6666666865348816,1.372549057006836,0.6666666865348816,0.75,0.6964285373687744,256,16,1.0155833981843356e-05,10,7,73.0,14.0
-2,uniprot,1471,151,0.906905055487053,0.09309494451294698,0.4983004758667573,0.5016995241332427,0.5894039735099338,0.4105960264900662,0.5333333333333333,0.4666666666666667,0,4,0.7619567513465881,0.6092715263366699,0.6242038011550903,0.6092715263366699,1.2334753274917603,0.720588207244873,0.550561785697937,0.6234142780303955,1.0701696872711182,0.5333333611488342,0.4615384638309479,0.5333333611488342,0.9948718547821045,0.6000000238418579,0.375,0.6071428656578064,512,32,2.0463837545929977e-05,50,4,74.0,13.0
-3,uniprot,1304,318,0.8039457459926017,0.1960542540073983,0.5015337423312883,0.49846625766871167,0.5283018867924528,0.4716981132075472,0.5333333333333333,0.4666666666666667,0,68,0.6827182173728943,0.5125786066055298,0.5259938836097717,0.5125786066055298,1.0385725498199463,0.5408805012702942,0.511904776096344,0.5704166889190674,0.78708815574646,0.4000000059604645,0.4000000059604645,0.4000000059604645,0.800000011920929,0.4285714328289032,0.375,0.4821428656578064,512,32,1.026894095562527e-05,20,7,66.0,21.0
-4,uniprot,1417,205,0.8736128236744759,0.12638717632552404,0.49047282992237123,0.5095271700776288,0.6195121951219512,0.3804878048780488,0.5333333333333333,0.4666666666666667,0,59,0.6809023022651672,0.6926829218864441,0.7509881258010864,0.6926829218864441,1.4436709880828857,0.7539682388305664,0.748031497001648,0.7631233334541321,1.3846888542175293,0.46666666865348816,0.4285714328289032,0.46666666865348816,0.8952381014823914,0.5,0.375,0.4999999701976776,256,16,5.082100906073162e-05,10,14,69.0,18.0
-0,tanimoto,1305,317,0.8045622688039458,0.19543773119605426,0.49885057471264366,0.5011494252873563,0.5394321766561514,0.4605678233438486,0.5333333333333333,0.4666666666666667,52,0,0.576474130153656,0.7287065982818604,0.742514967918396,0.7287065982818604,1.4712215662002563,0.7607361674308777,0.7251461744308472,0.7918368577957153,0.8999716639518738,0.6666666865348816,0.6666666865348816,0.6666666865348816,1.3333333730697632,0.7142857313156128,0.625,0.6071428656578064,512,16,1.3391215361544627e-05,20,11,54.0,14.0
-1,tanimoto,1034,588,0.6374845869297164,0.3625154130702836,0.4990328820116054,0.5009671179883946,0.5204081632653061,0.47959183673469385,0.5333333333333333,0.4666666666666667,54,0,0.5704006552696228,0.7227891087532043,0.7223168611526489,0.7227891087532043,1.445106029510498,0.754448413848877,0.6928104758262634,0.782778263092041,0.8248798251152039,0.6000000238418579,0.5,0.6000000238418579,1.100000023841858,0.75,0.375,0.5714285969734192,512,32,1.903929298739061e-05,20,13,51.0,17.0
-2,tanimoto,1412,210,0.8705302096177558,0.12946979038224415,0.5063739376770539,0.4936260623229462,0.5095238095238095,0.49047619047619045,0.5333333333333333,0.4666666666666667,46,0,0.4993876516819,0.7523809671401978,0.7373737096786499,0.7523809671401978,1.4897546768188477,0.8021978139877319,0.6822429895401001,0.8527356386184692,1.0459524393081665,0.5333333611488342,0.4615384638309479,0.5333333611488342,0.9948718547821045,0.6000000238418579,0.375,0.625,512,8,1.044195215191643e-05,20,14,58.0,10.0
-3,tanimoto,1277,345,0.7872996300863132,0.2127003699136868,0.5231010180109632,0.4768989819890368,0.4463768115942029,0.553623188405797,0.5333333333333333,0.4666666666666667,46,0,0.5563611388206482,0.7072463631629944,0.6480836272239685,0.7072463631629944,1.355329990386963,0.6992481350898743,0.6038960814476013,0.8062146902084351,1.265333652496338,0.46666666865348816,0.20000000298023224,0.46666666865348816,0.6666666865348816,0.5,0.125,0.5178571343421936,256,16,2.1987041897131907e-05,10,5,57.0,11.0
-4,tanimoto,1460,162,0.9001233045622689,0.0998766954377312,0.5054794520547945,0.4945205479452055,0.5185185185185185,0.48148148148148145,0.5333333333333333,0.4666666666666667,39,0,0.5791259407997131,0.7654321193695068,0.773809552192688,0.7654321193695068,1.5392416715621948,0.773809552192688,0.773809552192688,0.783348560333252,1.0499552488327026,0.3333333432674408,0.0,0.3333333432674408,0.3333333432674408,0.0,0.0,0.5357142686843872,768,16,1.0302046481625505e-05,10,12,52.0,16.0

reports/cv_report_hparam_search_5-splits_active-or.csv

@@ -0,0 +1,16 @@
+fold,group_type,train_len,val_len,train_perc,val_perc,train_active_perc,train_inactive_perc,val_active_perc,val_inactive_perc,test_active_perc,test_inactive_perc,num_leaking_uniprot,num_leaking_smiles,val_loss,val_acc,val_f1_score,val_hp_metric,val_opt_score,val_precision,val_recall,val_roc_auc,test_loss,test_acc,test_f1_score,test_hp_metric,test_opt_score,test_precision,test_recall,test_roc_auc,hidden_dim,batch_size,learning_rate,max_epochs,smote_k_neighbors,train_unique_groups,val_unique_groups
+0,random,1297,325,0.7996300863131935,0.2003699136868064,0.5065535851966075,0.49344641480339246,0.5076923076923077,0.49230769230769234,0.5333333333333333,0.4666666666666667,60,154,0.5019046068191528,0.7692307829856873,0.7678018808364868,0.7692307829856873,1.5370326042175293,0.7848101258277893,0.7515151500701904,0.8548105955123901,0.663665235042572,0.6666666865348816,0.6153846383094788,0.6666666865348816,1.2820513248443604,0.800000011920929,0.5,0.7321428060531616,768,32,2.2629818034524454e-05,50,3,,
+1,random,1297,325,0.7996300863131935,0.2003699136868064,0.5065535851966075,0.49344641480339246,0.5076923076923077,0.49230769230769234,0.5333333333333333,0.4666666666666667,61,169,0.4757218062877655,0.7938461303710938,0.8011869192123413,0.7938461303710938,1.595033049583435,0.7848837375640869,0.8181818127632141,0.89179927110672,1.0935707092285156,0.5333333611488342,0.5333333611488342,0.5333333611488342,1.0666667222976685,0.5714285969734192,0.5,0.5714285373687744,768,32,3.7417697937891526e-05,50,13,,
+2,random,1298,324,0.8002466091245376,0.1997533908754624,0.5069337442218799,0.4930662557781202,0.5061728395061729,0.49382716049382713,0.5333333333333333,0.4666666666666667,59,155,0.4547022581100464,0.7932098507881165,0.7873015999794006,0.7932098507881165,1.580511450767517,0.8211920261383057,0.7560975551605225,0.8943597674369812,0.9725640416145325,0.6000000238418579,0.5,0.6000000238418579,1.100000023841858,0.75,0.375,0.5535714030265808,256,8,2.81158230421478e-05,50,12,,
+3,random,1298,324,0.8002466091245376,0.1997533908754624,0.5069337442218799,0.4930662557781202,0.5061728395061729,0.49382716049382713,0.5333333333333333,0.4666666666666667,62,150,0.43522852659225464,0.8055555820465088,0.8096676468849182,0.8055555820465088,1.6152231693267822,0.802395224571228,0.8170731663703918,0.884070098400116,0.6184438467025757,0.7333333492279053,0.7142857313156128,0.7333333492279053,1.447619080543518,0.8333333134651184,0.625,0.75,512,16,2.1996927883652865e-05,10,15,,
+4,random,1298,324,0.8002466091245376,0.1997533908754624,0.5069337442218799,0.4930662557781202,0.5061728395061729,0.49382716049382713,0.5333333333333333,0.4666666666666667,56,149,0.5347408056259155,0.7561728358268738,0.7641791105270386,0.7561728358268738,1.5203518867492676,0.7485380172729492,0.7804877758026123,0.8434069752693176,0.5826601386070251,0.6666666865348816,0.6666666865348816,0.6666666865348816,1.3333333730697632,0.7142857313156128,0.625,0.75,512,16,1.9330317064010958e-05,50,9,,
+0,uniprot,1037,585,0.6393341553637485,0.36066584463625156,0.5255544840887174,0.47444551591128253,0.4735042735042735,0.5264957264957265,0.5333333333333333,0.4666666666666667,0,104,0.6796713471412659,0.5470085740089417,0.510166347026825,0.5470085740089417,1.0571749210357666,0.5227272510528564,0.49819493293762207,0.5657614469528198,0.6970714330673218,0.46666666865348816,0.4285714328289032,0.46666666865348816,0.8952381014823914,0.5,0.375,0.5535714030265808,256,16,1.0244924728107385e-05,10,10,66.0,21.0
+1,uniprot,1259,363,0.7762022194821209,0.22379778051787916,0.5250198570293884,0.4749801429706116,0.44352617079889806,0.5564738292011019,0.5333333333333333,0.4666666666666667,0,17,0.7094647288322449,0.5619834661483765,0.5691056847572327,0.5619834661483765,1.131089210510254,0.504807710647583,0.6521739363670349,0.5538097023963928,0.6740671396255493,0.6000000238418579,0.699999988079071,0.6000000238418579,1.2999999523162842,0.5833333134651184,0.875,0.7857142686843872,256,32,1.3703737801715468e-05,20,4,73.0,14.0
+2,uniprot,1471,151,0.906905055487053,0.09309494451294698,0.4983004758667573,0.5016995241332427,0.5894039735099338,0.4105960264900662,0.5333333333333333,0.4666666666666667,0,4,0.6605747938156128,0.6225165724754333,0.6850828528404236,0.6225165724754333,1.307599425315857,0.6739130616188049,0.6966292262077332,0.6585719585418701,0.6942979097366333,0.46666666865348816,0.5,0.46666666865348816,0.9666666984558105,0.5,0.5,0.6071428060531616,768,32,1.3666257429597246e-05,10,11,74.0,13.0
+3,uniprot,1304,318,0.8039457459926017,0.1960542540073983,0.5015337423312883,0.49846625766871167,0.5283018867924528,0.4716981132075472,0.5333333333333333,0.4666666666666667,0,68,0.6705124378204346,0.5157232880592346,0.5443786978721619,0.5157232880592346,1.0601019859313965,0.5411764979362488,0.5476190447807312,0.5866666436195374,0.7147824764251709,0.5333333611488342,0.6315789222717285,0.5333333611488342,1.164912223815918,0.5454545617103577,0.75,0.4642857015132904,256,32,1.5353188804765107e-05,10,3,66.0,21.0
+4,uniprot,1417,205,0.8736128236744759,0.12638717632552404,0.49047282992237123,0.5095271700776288,0.6195121951219512,0.3804878048780488,0.5333333333333333,0.4666666666666667,0,59,0.7100696563720703,0.6634146571159363,0.7356321811676025,0.6634146571159363,1.3990468978881836,0.7164179086685181,0.7559055089950562,0.7065414786338806,0.7504292726516724,0.6666666865348816,0.6153846383094788,0.6666666865348816,1.2820513248443604,0.800000011920929,0.5,0.6428570747375488,256,8,1.879856972588871e-05,20,11,69.0,18.0
+0,tanimoto,1316,306,0.8113440197287299,0.18865598027127004,0.5091185410334347,0.4908814589665654,0.49673202614379086,0.5032679738562091,0.5333333333333333,0.4666666666666667,54,0,0.5755400657653809,0.7189542651176453,0.7261146306991577,0.7189542651176453,1.4450688362121582,0.7037037014961243,0.75,0.7605946063995361,0.683353841304779,0.6000000238418579,0.625,0.6000000238418579,1.225000023841858,0.625,0.625,0.5178571343421936,512,32,1.2276134335095347e-05,50,5,56.0,14.0
+1,tanimoto,1454,168,0.8964241676942046,0.10357583230579531,0.5123796423658872,0.48762035763411277,0.4583333333333333,0.5416666666666666,0.5333333333333333,0.4666666666666667,43,0,0.5296748876571655,0.8095238208770752,0.7681159377098083,0.8095238208770752,1.5776398181915283,0.868852436542511,0.6883116960525513,0.8766947388648987,1.0112444162368774,0.46666666865348816,0.20000000298023224,0.46666666865348816,0.6666666865348816,0.5,0.125,0.6071428656578064,512,32,2.937124744712966e-05,10,10,60.0,10.0
+2,tanimoto,1249,373,0.7700369913686806,0.22996300863131935,0.5100080064051241,0.4899919935948759,0.4959785522788204,0.5040214477211796,0.5333333333333333,0.4666666666666667,56,0,0.593582034111023,0.7131367325782776,0.670769214630127,0.7131367325782776,1.3839058876037598,0.7785714268684387,0.5891891717910767,0.7807791829109192,0.8015115261077881,0.46666666865348816,0.20000000298023224,0.46666666865348816,0.6666666865348816,0.5,0.125,0.5714285373687744,512,16,1.216722143525285e-05,10,11,52.0,18.0
+3,tanimoto,1235,387,0.7614056720098644,0.23859432799013564,0.4939271255060729,0.5060728744939271,0.5478036175710594,0.45219638242894056,0.5333333333333333,0.4666666666666667,53,0,0.6089445352554321,0.6511628031730652,0.6683046817779541,0.6511628031730652,1.319467544555664,0.6974359154701233,0.6415094137191772,0.7357951402664185,0.612866997718811,0.6666666865348816,0.6153846383094788,0.6666666865348816,1.2820513248443604,0.800000011920929,0.5,0.7678571343421936,256,8,1.0536939044536473e-05,50,15,62.0,8.0
+4,tanimoto,1234,388,0.7607891491985204,0.23921085080147966,0.5072933549432739,0.49270664505672607,0.5051546391752577,0.4948453608247423,0.5333333333333333,0.4666666666666667,55,0,0.599588930606842,0.7036082744598389,0.7058823704719543,0.7036082744598389,1.4094905853271484,0.7076923251152039,0.704081654548645,0.7936065196990967,0.7881692051887512,0.5333333611488342,0.4615384638309479,0.5333333611488342,0.9948718547821045,0.6000000238418579,0.375,0.625,256,8,1.8370174889009875e-05,50,8,50.0,20.0