Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87bc001e53d8b757685c61a97c1467021cdf90723becdce2757020b3652b982a
+oid sha256:ae560f0a8af4322d27fb0580c44757e660e906316573b46defc0fb402c0f9311
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:90d1bf4eb6d1bb9271d5126b8f5a1c6dd17fbeae0f60a8ab31d61a5d4ebec11f
+oid sha256:d3b74523819017cbcb7086281f5577045559fa4f75317bbc4dd084f44e2577c9
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1d172b6355928a488e306b3c62c213f58a259204be7f1f4099838d768732abc
+oid sha256:237a0ac2efe433e18d94872643e503375cb570a9f13d83482b5b8a5c3ee52245
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8949affef14be53a53951982b2c566ab947f5b43a210616fc35cd404aaa9e59
+oid sha256:466d7528300efc91fe3a7fe887d60e328b080520c102b054e4be51ce66b6e70d
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.4888201502430402,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=10-learning_rate=4.4e-04-warmup_ratio=0.020-num_cycles=5.500/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.535483870967742,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=11-learning_rate=1.0e-04-warmup_ratio=0.080-num_cycles=3.000/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.515156626701355,
-      "learning_rate": 0.000312265868621247,
-      "loss": 1.1977,
+      "grad_norm": 0.806952953338623,
+      "learning_rate": 7.432336402923124e-05,
+      "loss": 1.855,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.19393421709537506,
-      "learning_rate": 0.0003136951242119503,
-      "loss": 0.06,
+      "grad_norm": 0.47002729773521423,
+      "learning_rate": 9.453407081430295e-05,
+      "loss": 0.1665,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.09234824031591415,
-      "learning_rate": 2.98272159149201e-06,
-      "loss": 0.0129,
+      "grad_norm": 0.22726435959339142,
+      "learning_rate": 4.922460243282525e-05,
+      "loss": 0.0456,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.11658934503793716,
-      "learning_rate": 0.0002447133294201148,
-      "loss": 0.0087,
+      "grad_norm": 0.15578638017177582,
+      "learning_rate": 6.484064135881745e-06,
+      "loss": 0.0238,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.1623298078775406,
-      "learning_rate": 0.0004185994767142762,
-      "loss": 0.0393,
+      "grad_norm": 0.17915856838226318,
+      "learning_rate": 6.569215571421697e-06,
+      "loss": 0.0192,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.06424859166145325,
-      "learning_rate": 7.844446743689375e-05,
-      "loss": 0.0136,
+      "grad_norm": 0.23383675515651703,
+      "learning_rate": 4.939985370478998e-05,
+      "loss": 0.0193,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.05626371130347252,
-      "learning_rate": 9.709552744129087e-05,
-      "loss": 0.0045,
+      "grad_norm": 0.2026490569114685,
+      "learning_rate": 9.463435501865708e-05,
+      "loss": 0.021,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.36843961477279663,
-      "learning_rate": 0.00042669351739376836,
-      "loss": 0.0176,
+      "grad_norm": 0.20101529359817505,
+      "learning_rate": 9.9666928389484e-05,
+      "loss": 0.0157,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.14914071559906006,
-      "learning_rate": 0.00022148081700229912,
-      "loss": 0.0497,
+      "grad_norm": 0.12849584221839905,
+      "learning_rate": 5.975745811745197e-05,
+      "loss": 0.0088,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.13498260080814362,
-      "learning_rate": 8.038929879251136e-06,
-      "loss": 0.0083,
+      "grad_norm": 0.08845894783735275,
+      "learning_rate": 1.2496157532491428e-05,
+      "loss": 0.005,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.4888201502430402,
-      "eval_e3_equal": 0.7847105612019443,
+      "eval_all_ligands_equal": 0.535483870967742,
+      "eval_e3_equal": 0.8047724259832081,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.99584622182943,
-      "eval_e3_heavy_atoms_difference": 0.45903667697746353,
-      "eval_e3_heavy_atoms_difference_norm": 0.007645187116914639,
+      "eval_e3_has_attachment_point(s)": 0.9923994697304463,
+      "eval_e3_heavy_atoms_difference": 0.4260715863897481,
+      "eval_e3_heavy_atoms_difference_norm": 0.0061653545557522485,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.99584622182943,
-      "eval_has_all_attachment_points": 0.9968183826778613,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 2.8648696420680513,
-      "eval_heavy_atoms_difference_norm": 0.035877785702251856,
-      "eval_linker_equal": 0.7178082191780822,
+      "eval_e3_valid": 0.9923994697304463,
+      "eval_has_all_attachment_points": 0.9844454264250995,
+      "eval_has_three_substructures": 0.9995581087052585,
+      "eval_heavy_atoms_difference": 6.345735749005745,
+      "eval_heavy_atoms_difference_norm": 0.08440506037431739,
+      "eval_linker_equal": 0.7666813963764914,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9992929739284137,
-      "eval_linker_heavy_atoms_difference": 0.3897481219619974,
-      "eval_linker_heavy_atoms_difference_norm": -0.006463945515568503,
+      "eval_linker_has_attachment_point(s)": 0.9980556783031375,
+      "eval_linker_heavy_atoms_difference": 0.3330092797171896,
+      "eval_linker_heavy_atoms_difference_norm": 0.001455031907522027,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9992929739284137,
-      "eval_loss": 0.28984424471855164,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.7379584622182943,
+      "eval_linker_valid": 0.9980556783031375,
+      "eval_loss": 0.289915531873703,
+      "eval_num_fragments": 2.9999116217410515,
+      "eval_poi_equal": 0.764471939902784,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9726027397260274,
-      "eval_poi_heavy_atoms_difference": 0.5621741051701281,
-      "eval_poi_heavy_atoms_difference_norm": 0.0017014581758733986,
+      "eval_poi_has_attachment_point(s)": 0.9332744144940345,
+      "eval_poi_heavy_atoms_difference": 1.9040212107821477,
+      "eval_poi_heavy_atoms_difference_norm": 0.06226196358827351,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9726027397260274,
-      "eval_reassembly": 0.4968625718073354,
-      "eval_reassembly_nostereo": 0.5237295625276182,
-      "eval_runtime": 1576.4712,
-      "eval_samples_per_second": 7.177,
-      "eval_steps_per_second": 0.112,
+      "eval_poi_valid": 0.9332744144940345,
+      "eval_reassembly": 0.5429960229783474,
+      "eval_reassembly_nostereo": 0.5680954485196642,
+      "eval_runtime": 1506.6911,
+      "eval_samples_per_second": 7.51,
+      "eval_steps_per_second": 0.117,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9680070702607159,
+      "eval_valid": 0.9248784798939461,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.11920630186796188,
-      "learning_rate": 0.00033406568194236334,
-      "loss": 0.0075,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.5241788625717163,
-      "learning_rate": 0.00036296734876936724,
-      "loss": 0.0502,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.3233102560043335,
-      "learning_rate": 2.0581448478610224e-05,
-      "loss": 0.0285,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.22846156358718872,
-      "learning_rate": 0.00018547973538572384,
-      "loss": 0.0155,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 1.3288661241531372,
-      "learning_rate": 0.0004345286134851596,
-      "loss": 0.0361,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.04896155545735749,
-      "eval_e3_equal": 0.40309323906319044,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9882456915598763,
-      "eval_e3_heavy_atoms_difference": 3.2123729562527616,
-      "eval_e3_heavy_atoms_difference_norm": 0.08806829276528134,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9882456915598763,
-      "eval_has_all_attachment_points": 0.9847105612019443,
-      "eval_has_three_substructures": 0.9967300044189129,
-      "eval_heavy_atoms_difference": 16.28095448519664,
-      "eval_heavy_atoms_difference_norm": 0.19772022011725113,
-      "eval_linker_equal": 0.19611135660627485,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9905435262925321,
-      "eval_linker_heavy_atoms_difference": 3.3720724701723377,
-      "eval_linker_heavy_atoms_difference_norm": 0.12169105760225543,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9905435262925321,
-      "eval_loss": 0.3659327030181885,
-      "eval_num_fragments": 2.997083517454706,
-      "eval_poi_equal": 0.37021652673442335,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.8987185152452497,
-      "eval_poi_heavy_atoms_difference": 5.3116217410517015,
-      "eval_poi_heavy_atoms_difference_norm": 0.13395514166933128,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.8987185152452497,
-      "eval_reassembly": 0.04993371630578878,
-      "eval_reassembly_nostereo": 0.058241272646928854,
-      "eval_runtime": 1444.1216,
-      "eval_samples_per_second": 7.835,
-      "eval_steps_per_second": 0.123,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8870525850640743,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.36206263303756714,
-      "learning_rate": 0.00012866172649195304,
-      "loss": 0.0394,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.40258172154426575,
-      "learning_rate": 5.274202368198822e-05,
-      "loss": 0.0172,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.4322950541973114,
-      "learning_rate": 0.00040158157654602796,
-      "loss": 0.0193,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.4600200355052948,
-      "learning_rate": 0.0002800483785552586,
-      "loss": 0.0508,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.38120532035827637,
-      "learning_rate": 0.0,
-      "loss": 0.0198,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.31842686699072026,
-      "eval_e3_equal": 0.6669023420238621,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9989394608926204,
-      "eval_e3_heavy_atoms_difference": 0.1460008837825895,
-      "eval_e3_heavy_atoms_difference_norm": -0.010953956022147847,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9989394608926204,
-      "eval_has_all_attachment_points": 0.9998232434821034,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 0.7110914714980114,
-      "eval_heavy_atoms_difference_norm": 0.003622114487280091,
-      "eval_linker_equal": 0.5198409191338931,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9997348652231551,
-      "eval_linker_heavy_atoms_difference": 0.3863013698630137,
-      "eval_linker_heavy_atoms_difference_norm": -0.044767838129064215,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9997348652231551,
-      "eval_loss": 0.3589182198047638,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.6580645161290323,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.994078656650464,
-      "eval_poi_heavy_atoms_difference": -0.21882456915598764,
-      "eval_poi_heavy_atoms_difference_norm": -0.03121567410490773,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.994078656650464,
-      "eval_reassembly": 0.3226690234202386,
-      "eval_reassembly_nostereo": 0.34202386212991603,
-      "eval_runtime": 1587.8107,
-      "eval_samples_per_second": 7.126,
-      "eval_steps_per_second": 0.111,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9927529827662395,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=10-learning_rate=4.4e-04-warmup_ratio=0.020-num_cycles=5.500",
+  "trial_name": "trial-number=11-learning_rate=1.0e-04-warmup_ratio=0.080-num_cycles=3.000",
   "trial_params": {
-    "learning_rate": 0.00043654768433250334,
-    "num_cycles": 5.5,
-    "warmup_ratio": 0.02
+    "learning_rate": 0.00010390406291286528,
+    "num_cycles": 3.0,
+    "warmup_ratio": 0.08
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84b6473490296441fd6c5782f475d881910820d5064757237fe5ee5e2079acd0
+oid sha256:e5ce96d452633241b4fc2730028e61040d2a68346c50445afdee5f71841371b2
 size 7416