Training in progress, step 10240, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ca37dad208975487ae890dd2447d9fd111a4d500fddec1a394f9efeef88557e
+oid sha256:3326816602b76848a6a8b29dd3a51168076de7c214927edd394c75013a454acf
 size 4725595416

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0b8169b03af949ee4c5ddf91eed20622e053b6dde7d912b4c3311f20f6495a7
+oid sha256:69eff366bb4160a16c74c582bd7f1c525455e190d832ad429ff154102b6ca5ef
 size 9179193343

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb732be4c200b0b68d66033f545d69b981d9d12cd9b1fae529a2b5f11bc8689a
+oid sha256:7b60dc5f90831a3cb044b4ac92382bb3956ae1e5aaac38365110e8ca62c577d1
 size 14503

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7802dd5061761c471fafcb314e7fa5bea1fb541e8e1ce0821e89ee84f8e88b84
+oid sha256:85e731ed421ccadf919d8023a4de52dddc9be17926d4166cc53e8083d7604c53
 size 623

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 3752.7509765625,
   "best_model_checkpoint": "mgh6/TCS_Pairing_VAE/checkpoint-7680",
-  "epoch": 0.5674335269724873,
+  "epoch": 0.7565780359633163,
   "eval_steps": 512,
-  "global_step": 7680,
+  "global_step": 10240,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -307,6 +307,106 @@
       "eval_samples_per_second": 66.508,
       "eval_steps_per_second": 66.508,
       "step": 7680
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 8.827249889168022e-05,
+      "loss": 4972.3188,
+      "step": 7936
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 8.789419240431506e-05,
+      "loss": 5409.0205,
+      "step": 8192
+    },
+    {
+      "epoch": 0.61,
+      "eval_loss": 4419.3115234375,
+      "eval_runtime": 56.0194,
+      "eval_samples_per_second": 60.658,
+      "eval_steps_per_second": 60.658,
+      "step": 8192
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 8.751588591694991e-05,
+      "loss": 4755.2881,
+      "step": 8448
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 8.713757942958476e-05,
+      "loss": 4503.3687,
+      "step": 8704
+    },
+    {
+      "epoch": 0.64,
+      "eval_loss": 4440.9599609375,
+      "eval_runtime": 50.1462,
+      "eval_samples_per_second": 67.762,
+      "eval_steps_per_second": 67.762,
+      "step": 8704
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 8.67592729422196e-05,
+      "loss": 4803.3394,
+      "step": 8960
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 8.638096645485444e-05,
+      "loss": 5031.4937,
+      "step": 9216
+    },
+    {
+      "epoch": 0.68,
+      "eval_loss": 5361.60546875,
+      "eval_runtime": 49.6714,
+      "eval_samples_per_second": 68.41,
+      "eval_steps_per_second": 68.41,
+      "step": 9216
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 8.600265996748929e-05,
+      "loss": 4789.9038,
+      "step": 9472
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 8.562435348012414e-05,
+      "loss": 5079.5186,
+      "step": 9728
+    },
+    {
+      "epoch": 0.72,
+      "eval_loss": 4070.673828125,
+      "eval_runtime": 49.4243,
+      "eval_samples_per_second": 68.752,
+      "eval_steps_per_second": 68.752,
+      "step": 9728
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 8.524604699275897e-05,
+      "loss": 5474.73,
+      "step": 9984
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.486774050539382e-05,
+      "loss": 4787.0361,
+      "step": 10240
+    },
+    {
+      "epoch": 0.76,
+      "eval_loss": 4277.46337890625,
+      "eval_runtime": 49.554,
+      "eval_samples_per_second": 68.572,
+      "eval_steps_per_second": 68.572,
+      "step": 10240
     }
   ],
   "logging_steps": 256,