Training in progress, step 5120, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a79993ca3bb2e40d715e49b6365049f27102e49dd8b3a9ce020c6ea5a9f9fe9
+oid sha256:869b484e58461f1536cc21d599303cb514c3d752e29ef44231f6d6c363605817
 size 4725595416

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1bf2691edb5f20acb6de9eb1f6120c2449bed48ca00eecc968c5be167084b7bb
+oid sha256:14a089b06ff32225e1d8e28e6b6c1cab9f01e70af7cbc2eac64ef56b2989354d
 size 9179193343

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c349c1691bbeda5a6b16abd459bd4b17c698c1ae8b87b93b48229ee14acd38e
+oid sha256:dc1239486b7f83e4a2231cde24a50b503b22ee79d6ee232760274da141c18674
 size 14503

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28472ecbb49d175fddb5467d2d36c375ce76e352a7c4d1642d73ecb32735946a
+oid sha256:0ce8999a5d1b14a256d594f72ba3d10015736fefa5fd7057a03491428983da78
 size 623

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 19020.044921875,
-  "best_model_checkpoint": "mgh6/TCS_Pairing_VAE/checkpoint-2560",
-  "epoch": 0.18914450899082907,
+  "best_metric": 4288.33056640625,
+  "best_model_checkpoint": "mgh6/TCS_Pairing_VAE/checkpoint-5120",
+  "epoch": 0.37828901798165815,
   "eval_steps": 512,
-  "global_step": 2560,
+  "global_step": 5120,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -107,6 +107,106 @@
       "eval_samples_per_second": 69.222,
       "eval_steps_per_second": 69.222,
       "step": 2560
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 9.58386286389833e-05,
+      "loss": 15475.9717,
+      "step": 2816
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 9.546032215161815e-05,
+      "loss": 11322.8867,
+      "step": 3072
+    },
+    {
+      "epoch": 0.23,
+      "eval_loss": 8387.529296875,
+      "eval_runtime": 56.5145,
+      "eval_samples_per_second": 60.126,
+      "eval_steps_per_second": 60.126,
+      "step": 3072
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 9.5082015664253e-05,
+      "loss": 8684.2373,
+      "step": 3328
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 9.470370917688785e-05,
+      "loss": 6917.9409,
+      "step": 3584
+    },
+    {
+      "epoch": 0.26,
+      "eval_loss": 5089.6796875,
+      "eval_runtime": 52.3172,
+      "eval_samples_per_second": 64.95,
+      "eval_steps_per_second": 64.95,
+      "step": 3584
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 9.43254026895227e-05,
+      "loss": 6025.4263,
+      "step": 3840
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 9.394709620215754e-05,
+      "loss": 5538.1548,
+      "step": 4096
+    },
+    {
+      "epoch": 0.3,
+      "eval_loss": 4470.64990234375,
+      "eval_runtime": 49.8023,
+      "eval_samples_per_second": 68.23,
+      "eval_steps_per_second": 68.23,
+      "step": 4096
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 9.356878971479238e-05,
+      "loss": 5257.3623,
+      "step": 4352
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 9.319048322742722e-05,
+      "loss": 5375.1353,
+      "step": 4608
+    },
+    {
+      "epoch": 0.34,
+      "eval_loss": 4827.8271484375,
+      "eval_runtime": 49.3721,
+      "eval_samples_per_second": 68.824,
+      "eval_steps_per_second": 68.824,
+      "step": 4608
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 9.281217674006207e-05,
+      "loss": 5494.5615,
+      "step": 4864
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 9.243387025269692e-05,
+      "loss": 5258.2065,
+      "step": 5120
+    },
+    {
+      "epoch": 0.38,
+      "eval_loss": 4288.33056640625,
+      "eval_runtime": 51.0091,
+      "eval_samples_per_second": 66.616,
+      "eval_steps_per_second": 66.616,
+      "step": 5120
     }
   ],
   "logging_steps": 256,