Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d78355b9d349c5518858b86ce7ecf948eb9b97666fe4508dea5d442a9f9c4257
+oid sha256:ca4fe7c2b27306b825b74ed0fca6c007623edcc95ed6a1066831daad87297111
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5df527b2d9ccfff6691f10ef70b8552eab6e09a6c3d3d3adc901e2cc5b3f3710
+oid sha256:c2453bf4e543ffc35925652d75fb01c1ae88dc37c9f16530e003f8bfc84e7993
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:467678a03880abc67f8f466fe2af00759b5be3112b8eaea586305a7011e36322
+oid sha256:737c37e64d8976b6a98de2e50f10afefd5a40eb5d5622d8f238c62b443f8adc0
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e71287ba09c7f357fed0abd66b3eed52c8161ab12ba588db68125e7661394bc
+oid sha256:9a0422bab0c4fbf63011708fabc615bc4cfb890b30611cd7933acaf26bce321d
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5098541758727353,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=2-learning_rate=4.1e-04-warmup_ratio=0.050-num_cycles=5.000/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.4693769332744145,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=3-learning_rate=5.4e-05-warmup_ratio=0.060-num_cycles=7.500/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.40457019209861755,
-      "learning_rate": 0.0002910271019893122,
-      "loss": 1.2229,
+      "grad_norm": 0.9083260297775269,
+      "learning_rate": 3.874709791916775e-05,
+      "loss": 2.3058,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.18502722680568695,
-      "learning_rate": 0.0003104691238246562,
-      "loss": 0.0606,
+      "grad_norm": 0.5939458608627319,
+      "learning_rate": 2.832337249910246e-05,
+      "loss": 0.3015,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.09909135848283768,
-      "learning_rate": 1.90362287142549e-05,
-      "loss": 0.0141,
+      "grad_norm": 0.4679681658744812,
+      "learning_rate": 1.0605163273926215e-05,
+      "loss": 0.1326,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.11582358181476593,
-      "learning_rate": 0.00013982074470021187,
-      "loss": 0.0077,
+      "grad_norm": 0.33971089124679565,
+      "learning_rate": 5.289075774840619e-05,
+      "loss": 0.1021,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.285866916179657,
-      "learning_rate": 0.0004028025121205152,
-      "loss": 0.0283,
+      "grad_norm": 0.29710039496421814,
+      "learning_rate": 1.209189635946932e-06,
+      "loss": 0.0513,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.06700430065393448,
-      "learning_rate": 0.00022007191153947144,
-      "loss": 0.0225,
+      "grad_norm": 0.30043792724609375,
+      "learning_rate": 4.374419101779672e-05,
+      "loss": 0.0404,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.05584639683365822,
-      "learning_rate": 1.3284381938488023e-07,
-      "loss": 0.005,
+      "grad_norm": 0.18167661130428314,
+      "learning_rate": 2.5616651285315406e-05,
+      "loss": 0.0329,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.0648360401391983,
-      "learning_rate": 0.00023467070171868308,
-      "loss": 0.0042,
+      "grad_norm": 0.2189311981201172,
+      "learning_rate": 1.2832850915256788e-05,
+      "loss": 0.02,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.3304208517074585,
-      "learning_rate": 0.00039936372660964736,
-      "loss": 0.0329,
+      "grad_norm": 0.2206682711839676,
+      "learning_rate": 5.194136325168918e-05,
+      "loss": 0.0207,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.0806419625878334,
-      "learning_rate": 0.00012603196998424004,
+      "grad_norm": 0.15758731961250305,
+      "learning_rate": 5.396630538749317e-07,
       "loss": 0.0148,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.5098541758727353,
-      "eval_e3_equal": 0.8019443216968626,
+      "eval_all_ligands_equal": 0.4693769332744145,
+      "eval_e3_equal": 0.7895713654441008,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9963764913831198,
-      "eval_e3_heavy_atoms_difference": 0.3095890410958904,
-      "eval_e3_heavy_atoms_difference_norm": 0.00024917772776295255,
+      "eval_e3_has_attachment_point(s)": 0.9946973044631021,
+      "eval_e3_heavy_atoms_difference": 0.1929297392841361,
+      "eval_e3_heavy_atoms_difference_norm": -0.0033631920311861557,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9963764913831198,
-      "eval_has_all_attachment_points": 0.9939902783915157,
-      "eval_has_three_substructures": 0.9995581087052585,
-      "eval_heavy_atoms_difference": 4.5589924878479895,
-      "eval_heavy_atoms_difference_norm": 0.058620664316153995,
-      "eval_linker_equal": 0.7363676535572249,
+      "eval_e3_valid": 0.9946973044631021,
+      "eval_has_all_attachment_points": 0.9870083959346001,
+      "eval_has_three_substructures": 0.9992045956694653,
+      "eval_heavy_atoms_difference": 6.721696862571807,
+      "eval_heavy_atoms_difference_norm": 0.0889528639189911,
+      "eval_linker_equal": 0.671586389748122,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9984975695978789,
-      "eval_linker_heavy_atoms_difference": 0.6367653557224923,
-      "eval_linker_heavy_atoms_difference_norm": 0.017842498112545725,
+      "eval_linker_has_attachment_point(s)": 0.9954927087936367,
+      "eval_linker_heavy_atoms_difference": 0.38320813079982324,
+      "eval_linker_heavy_atoms_difference_norm": -0.0011190134002451928,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9984975695978789,
-      "eval_loss": 0.2861309349536896,
-      "eval_num_fragments": 2.9995581087052585,
-      "eval_poi_equal": 0.7490941228457799,
+      "eval_linker_valid": 0.9954927087936367,
+      "eval_loss": 0.28346458077430725,
+      "eval_num_fragments": 3.0004418912947415,
+      "eval_poi_equal": 0.7376933274414494,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9545735749005745,
-      "eval_poi_heavy_atoms_difference": 1.2116659301811754,
-      "eval_poi_heavy_atoms_difference_norm": 0.02950574177471354,
+      "eval_poi_has_attachment_point(s)": 0.9247017233760495,
+      "eval_poi_heavy_atoms_difference": 2.202121078214759,
+      "eval_poi_heavy_atoms_difference_norm": 0.07112158874926038,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9545735749005745,
-      "eval_reassembly": 0.5175430844012373,
-      "eval_reassembly_nostereo": 0.5445868316394167,
-      "eval_runtime": 1571.8887,
-      "eval_samples_per_second": 7.198,
-      "eval_steps_per_second": 0.113,
+      "eval_poi_valid": 0.9247017233760495,
+      "eval_reassembly": 0.4767123287671233,
+      "eval_reassembly_nostereo": 0.5031374281926646,
+      "eval_runtime": 1511.6507,
+      "eval_samples_per_second": 7.485,
+      "eval_steps_per_second": 0.117,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9505081749889527,
+      "eval_valid": 0.9180733539549271,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.067192941904068,
-      "learning_rate": 2.5722998347357026e-05,
-      "loss": 0.004,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.15581953525543213,
-      "learning_rate": 0.00032268281141035576,
-      "loss": 0.005,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.336742103099823,
-      "learning_rate": 0.0003530420755759163,
-      "loss": 0.0502,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.08952895551919937,
-      "learning_rate": 4.893105364892502e-05,
-      "loss": 0.0127,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.1288328915834427,
-      "learning_rate": 9.020598403374023e-05,
-      "loss": 0.0056,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5009279717189571,
-      "eval_e3_equal": 0.7919575784357048,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9945205479452055,
-      "eval_e3_heavy_atoms_difference": 0.43605832965090585,
-      "eval_e3_heavy_atoms_difference_norm": 0.006540535530209271,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9945205479452055,
-      "eval_has_all_attachment_points": 0.9980556783031375,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 3.0039770216526733,
-      "eval_heavy_atoms_difference_norm": 0.03823200102104782,
-      "eval_linker_equal": 0.7307998232434821,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9989394608926204,
-      "eval_linker_heavy_atoms_difference": 0.4122845779938135,
-      "eval_linker_heavy_atoms_difference_norm": -0.004311368730086423,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9989394608926204,
-      "eval_loss": 0.3087325096130371,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.7456473707467963,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9756959787892179,
-      "eval_poi_heavy_atoms_difference": 0.6555899248784799,
-      "eval_poi_heavy_atoms_difference_norm": 0.008748681011243051,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9756959787892179,
-      "eval_reassembly": 0.5077330976579761,
-      "eval_reassembly_nostereo": 0.5368095448519664,
-      "eval_runtime": 1535.6614,
-      "eval_samples_per_second": 7.368,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.969244365885992,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.2932059168815613,
-      "learning_rate": 0.00038459458164837333,
-      "loss": 0.0096,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.6754857301712036,
-      "learning_rate": 0.0002739756007213787,
-      "loss": 0.0496,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.2761557996273041,
-      "learning_rate": 5.643610823486575e-06,
-      "loss": 0.0147,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.22613073885440826,
-      "learning_rate": 0.00017946893608695752,
-      "loss": 0.0093,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 1.2744674682617188,
-      "learning_rate": 0.00040685588858105847,
-      "loss": 0.029,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.06637207247017234,
-      "eval_e3_equal": 0.40397702165267346,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.971011931064958,
-      "eval_e3_heavy_atoms_difference": 2.9835616438356163,
-      "eval_e3_heavy_atoms_difference_norm": 0.08521778499278253,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.971011931064958,
-      "eval_has_all_attachment_points": 0.9942554131683606,
-      "eval_has_three_substructures": 0.9984975695978789,
-      "eval_heavy_atoms_difference": 13.69712770658418,
-      "eval_heavy_atoms_difference_norm": 0.16811466909371175,
-      "eval_linker_equal": 0.23862129916040653,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9972602739726028,
-      "eval_linker_heavy_atoms_difference": 3.032788334069819,
-      "eval_linker_heavy_atoms_difference_norm": 0.11622827127109912,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9972602739726028,
-      "eval_loss": 0.35788974165916443,
-      "eval_num_fragments": 2.998497569597879,
-      "eval_poi_equal": 0.4328767123287671,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9274414494034467,
-      "eval_poi_heavy_atoms_difference": 3.7181617322138756,
-      "eval_poi_heavy_atoms_difference_norm": 0.08639693676345507,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9274414494034467,
-      "eval_reassembly": 0.06787450287229342,
-      "eval_reassembly_nostereo": 0.0758285461776403,
-      "eval_runtime": 1609.6576,
-      "eval_samples_per_second": 7.029,
-      "eval_steps_per_second": 0.11,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9008395934600089,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=2-learning_rate=4.1e-04-warmup_ratio=0.050-num_cycles=5.000",
+  "trial_name": "trial-number=3-learning_rate=5.4e-05-warmup_ratio=0.060-num_cycles=7.500",
   "trial_params": {
-    "learning_rate": 0.00040685588858105847,
-    "num_cycles": 5.0,
-    "warmup_ratio": 0.05
+    "learning_rate": 5.4168442890996515e-05,
+    "num_cycles": 7.5,
+    "warmup_ratio": 0.060000000000000005
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab597388b314148cc4e5cc608f3d5772b22047e13fa26a07fccdb041e424a037
+oid sha256:250cc162b7f9447737d6a41a883326079236d83d8d2a9d749bc6baaafc398b96
 size 7416