Training in progress, step 50, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8f778e8b69dae331d2a23702c4b9de9b128b3fd135601c63d192c486ef4fd13
+oid sha256:45fcf42f5a42558a39fc0d394f03662f9e288bf90866c5014d4e104a1fef4d9f
 size 180543866

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d27939d800185d643932238e68be76d14cda83650e61932ec7f9004be0466d12
+oid sha256:c5382a2b876c47a4a3dba990c5e3d357308154d058f2a8082f6d52ed6904a1c6
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:884bf6290d3fc3e884f845ddbf8f96312ff2d276653f3656adbb7503273b6a34
+oid sha256:e12af52381ca9f526212a318c379215e9937bfa32db33f37122fe6d062e2928b
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88c9b2253e2990dd5268c4e2b91fcca36d9a42ca11b7349975cfb3ccd2cec5a2
+oid sha256:87c0b82ad798d2b7e63cdbc6bf2a0d9bc5abdf2d7fc7b25499904e7af5e2eb13
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:717703755061944b2c06aef64226f57b0d7d7d1b4efeaa2afd5610d4df3b7fdb
+oid sha256:85bd3f6643377e28992532dd4076a8ffac43fdc8f9941ad2bb30f1eea0fd7015
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5eda964d5b06109c7ff637060199d72e397eeb7027b50e10972a323edbe81312
+oid sha256:fb0ce41087f0939d63a9a6eb297660d82f62bda022868309a8c97328e5da41a5
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 0.06260760682422914,
+  "epoch": 0.12521521364845828,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,189 @@
       "eval_samples_per_second": 34.596,
       "eval_steps_per_second": 8.995,
       "step": 25
+    },
+    {
+      "epoch": 0.06511191109719831,
+      "grad_norm": NaN,
+      "learning_rate": 7.917848237560709e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 0.06761621537016747,
+      "grad_norm": NaN,
+      "learning_rate": 7.75e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 0.07012051964313665,
+      "grad_norm": NaN,
+      "learning_rate": 7.577868759557654e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 0.07262482391610581,
+      "grad_norm": NaN,
+      "learning_rate": 7.401782177833148e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 0.07512912818907497,
+      "grad_norm": NaN,
+      "learning_rate": 7.222075445642904e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.07763343246204414,
+      "grad_norm": NaN,
+      "learning_rate": 7.03909064496551e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 0.0801377367350133,
+      "grad_norm": NaN,
+      "learning_rate": 6.853176097769229e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 0.08264204100798248,
+      "grad_norm": NaN,
+      "learning_rate": 6.664685702961344e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 0.08514634528095164,
+      "grad_norm": NaN,
+      "learning_rate": 6.473978262721463e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 0.0876506495539208,
+      "grad_norm": NaN,
+      "learning_rate": 6.281416799501188e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 0.09015495382688997,
+      "grad_norm": NaN,
+      "learning_rate": 6.087367864990233e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 0.09265925809985913,
+      "grad_norm": NaN,
+      "learning_rate": 5.8922008423644624e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 0.0951635623728283,
+      "grad_norm": NaN,
+      "learning_rate": 5.696287243144013e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 0.09766786664579746,
+      "grad_norm": NaN,
+      "learning_rate": 5.500000000000001e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 0.10017217091876664,
+      "grad_norm": NaN,
+      "learning_rate": 5.303712756855988e-05,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.1026764751917358,
+      "grad_norm": NaN,
+      "learning_rate": 5.107799157635538e-05,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 0.10518077946470496,
+      "grad_norm": NaN,
+      "learning_rate": 4.912632135009769e-05,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 0.10768508373767413,
+      "grad_norm": NaN,
+      "learning_rate": 4.718583200498814e-05,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 0.11018938801064329,
+      "grad_norm": NaN,
+      "learning_rate": 4.526021737278538e-05,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 0.11269369228361246,
+      "grad_norm": NaN,
+      "learning_rate": 4.3353142970386564e-05,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 0.11519799655658162,
+      "grad_norm": NaN,
+      "learning_rate": 4.146823902230772e-05,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 0.11770230082955078,
+      "grad_norm": NaN,
+      "learning_rate": 3.960909355034491e-05,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 0.12020660510251996,
+      "grad_norm": NaN,
+      "learning_rate": 3.777924554357096e-05,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 0.12271090937548912,
+      "grad_norm": NaN,
+      "learning_rate": 3.598217822166854e-05,
+      "loss": 0.0,
+      "step": 49
+    },
+    {
+      "epoch": 0.12521521364845828,
+      "grad_norm": NaN,
+      "learning_rate": 3.422131240442349e-05,
+      "loss": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.12521521364845828,
+      "eval_loss": NaN,
+      "eval_runtime": 1.5015,
+      "eval_samples_per_second": 33.3,
+      "eval_steps_per_second": 8.658,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -212,7 +395,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -221,12 +404,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3.91521434075136e+16,
+  "total_flos": 7.83042868150272e+16,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null