Training in progress, step 50, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75562e3246a15d26b7654f8abe42110e9bb645beea72e7da02ed584dd6d51aa0
+oid sha256:5cf92819c598c0f176d87f4df9c4e0c28e10ceafdf2b928e48456d66e8c369ee
 size 671466706

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c9fbb24223984c1ca76d160ce5e7ec045ee3dc7f5391c94d27526e68213f15e
+oid sha256:14da97aee98c48f4c32359218ab21974b1801f09c31159f1b77234dcd3bd018b
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f6fc895c6ae4f53690a7fed0af8327affea39a211153f482383d1a92212f368
+oid sha256:740134b6a7dd120cf35e1144a39fda13e92e96e37c14dd39fe5c40898c7c5456
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0c737edb66b5b15506cabf5a63323613aa73e4015e74b4b2daa221ab4205c1a
+oid sha256:e0a1a8ad0ed2d2d3daab9ab0baa265ec551a4cff0bd3803b40768afb38ae68a3
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3698166bce2f719baf84c9ab843476d5312b9576b3786853cee31f2fea3199be
+oid sha256:684325cbbe2e70a59964e02b3c6ae3e56848c2eff7994d96557f99b545b68088
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d271cdb95f63cd655315f063ca2e25c78dc5ae4275523c5d4f80f367586b3351
+oid sha256:5607f6de446164d9d9adb8b91c44cec55b14aa391e24ba5637c08b834eedda2a
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 0.09436187780136825,
+  "epoch": 0.1887237556027365,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,189 @@
       "eval_samples_per_second": 28.796,
       "eval_steps_per_second": 7.487,
       "step": 25
+    },
+    {
+      "epoch": 0.09813635291342297,
+      "grad_norm": NaN,
+      "learning_rate": 5.500000000000001e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 0.10191082802547771,
+      "grad_norm": NaN,
+      "learning_rate": 5.205685918464356e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 0.10568530313753244,
+      "grad_norm": NaN,
+      "learning_rate": 4.912632135009769e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 0.10945977824958716,
+      "grad_norm": NaN,
+      "learning_rate": 4.6220935509274235e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 0.11323425336164189,
+      "grad_norm": NaN,
+      "learning_rate": 4.3353142970386564e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.11700872847369663,
+      "grad_norm": NaN,
+      "learning_rate": 4.053522406135775e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 0.12078320358575136,
+      "grad_norm": NaN,
+      "learning_rate": 3.777924554357096e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 0.12455767869780608,
+      "grad_norm": NaN,
+      "learning_rate": 3.509700894014496e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 0.12833215380986082,
+      "grad_norm": NaN,
+      "learning_rate": 3.250000000000001e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 0.13210662892191555,
+      "grad_norm": NaN,
+      "learning_rate": 2.9999339514117912e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 0.13588110403397027,
+      "grad_norm": NaN,
+      "learning_rate": 2.760573569460757e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 0.139655579146025,
+      "grad_norm": NaN,
+      "learning_rate": 2.53294383204969e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 0.14343005425807973,
+      "grad_norm": NaN,
+      "learning_rate": 2.3180194846605367e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 0.14720452937013448,
+      "grad_norm": NaN,
+      "learning_rate": 2.1167208663446025e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 0.1509790044821892,
+      "grad_norm": NaN,
+      "learning_rate": 1.9299099686894423e-05,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.15475347959424393,
+      "grad_norm": NaN,
+      "learning_rate": 1.758386744638546e-05,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 0.15852795470629866,
+      "grad_norm": NaN,
+      "learning_rate": 1.602885682970026e-05,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 0.16230242981835338,
+      "grad_norm": NaN,
+      "learning_rate": 1.464072663102903e-05,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 0.1660769049304081,
+      "grad_norm": NaN,
+      "learning_rate": 1.3425421036992098e-05,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 0.16985138004246284,
+      "grad_norm": NaN,
+      "learning_rate": 1.2388144172720251e-05,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 0.1736258551545176,
+      "grad_norm": NaN,
+      "learning_rate": 1.1533337816991932e-05,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 0.17740033026657231,
+      "grad_norm": NaN,
+      "learning_rate": 1.0864662381854632e-05,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 0.18117480537862704,
+      "grad_norm": NaN,
+      "learning_rate": 1.0384981238178534e-05,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 0.18494928049068177,
+      "grad_norm": NaN,
+      "learning_rate": 1.0096348454262845e-05,
+      "loss": 0.0,
+      "step": 49
+    },
+    {
+      "epoch": 0.1887237556027365,
+      "grad_norm": NaN,
+      "learning_rate": 1e-05,
+      "loss": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.1887237556027365,
+      "eval_loss": NaN,
+      "eval_runtime": 1.7525,
+      "eval_samples_per_second": 28.53,
+      "eval_steps_per_second": 7.418,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -212,7 +395,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -221,12 +404,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2.829010669142016e+17,
+  "total_flos": 5.658021338284032e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null