Training in progress, step 41, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75c994c2b8e40583ea1266a83afba7b309c13c0736f382473516c812300756a8
+oid sha256:47ba6cef829437aca06076e054db31b50ccd670782c5e2ca2c3c9032ed9c87ab
 size 671466706

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f65cc0509dca9e259e216abe2448677d8dbd226be11e4d43f72621718f1a1b7d
+oid sha256:84cb21656a90a91063da172e61acea46a94ed08e7c2b1c1dd90941e49d21fd81
 size 14960

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8417cdbc48451efd84d464b892e54e01366a0db2134dac4bf5992a0c76f43de7
+oid sha256:2e382f60f14fc4aa02782a5c0c35afab3dbe82bd45b2513cee0064bc5289c90e
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5011b75b693487d81a98dc1115265d676ebe9c4f861c56e0635308a3cc72d950
+oid sha256:2290648a1cf4ccf99f19e1d621c55908f8dd38658134cc41c6cd75bdb4b09c9c
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:841079347968c8add61b5e057ca132c08232cb63d23c815664f9b12338aea3fc
+oid sha256:bd487667f16d2c2471b752e2d79fe67f2433fd52a785b53cb1dd0c1af289ec2b
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc6b77c0cbcc35c1c43ee60e2766b6027d2e111edae589e45f75af1b1040b32d
+oid sha256:84fc982e8229d2121603832a24f4c07bbdbf63985ed1d38e11762164a04dad3b
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 1.8691588785046729,
+  "epoch": 3.0654205607476634,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 41,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,118 @@
       "eval_samples_per_second": 31.396,
       "eval_steps_per_second": 4.186,
       "step": 25
+    },
+    {
+      "epoch": 1.9439252336448598,
+      "grad_norm": NaN,
+      "learning_rate": 3.226975564787322e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 2.0186915887850465,
+      "grad_norm": NaN,
+      "learning_rate": 2.8565371929847284e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 2.0934579439252334,
+      "grad_norm": NaN,
+      "learning_rate": 2.500000000000001e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 2.1682242990654204,
+      "grad_norm": NaN,
+      "learning_rate": 2.1596762663442218e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 2.2429906542056073,
+      "grad_norm": NaN,
+      "learning_rate": 1.837773122023114e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 2.317757009345794,
+      "grad_norm": NaN,
+      "learning_rate": 1.536378232452003e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 2.392523364485981,
+      "grad_norm": NaN,
+      "learning_rate": 1.257446259144494e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 2.467289719626168,
+      "grad_norm": NaN,
+      "learning_rate": 1.0027861829824952e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 2.542056074766355,
+      "grad_norm": NaN,
+      "learning_rate": 7.740495722810271e-06,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 2.616822429906542,
+      "grad_norm": NaN,
+      "learning_rate": 5.727198717339511e-06,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 2.691588785046729,
+      "grad_norm": NaN,
+      "learning_rate": 4.001027817058789e-06,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 2.7663551401869158,
+      "grad_norm": NaN,
+      "learning_rate": 2.573177902642726e-06,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 2.8411214953271027,
+      "grad_norm": NaN,
+      "learning_rate": 1.4529091286973995e-06,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 2.9158878504672896,
+      "grad_norm": NaN,
+      "learning_rate": 6.474868681043578e-07,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 2.9906542056074765,
+      "grad_norm": NaN,
+      "learning_rate": 1.6213459328950352e-07,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 3.0654205607476634,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 41
     }
   ],
   "logging_steps": 1,
@@ -221,12 +333,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2.90486891839488e+17,
+  "total_flos": 4.763985026167603e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null