Training in progress, step 50, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:422d9785c7e5ae607d8558fe078388cd4f5de28b810906bebb036de4da400453
+oid sha256:23a30d0b928b4bf551f0c47963042b439e40fde1544b7358ad36c00d8bc609f5
 size 75471860

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ee3245932971d68d44c15040c5b344cd7a9591c04218bc54d69711582b88920
+oid sha256:8956016ef6f672709872466580d9271cba47505d3848e8a2007c5338b539de7e
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6676fe28230ae15b45fb334c871c6fdf1a7984a935952b9f8650896c37a8c106
+oid sha256:b1df0528620c07325b8faa7567e59b0c1e86a1f1ee6af1245a69c6c0463fe4e2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 0.08960573476702509,
+  "epoch": 0.17921146953405018,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,189 @@
       "eval_samples_per_second": 17.009,
       "eval_steps_per_second": 8.504,
       "step": 25
+    },
+    {
+      "epoch": 0.0931899641577061,
+      "grad_norm": NaN,
+      "learning_rate": 6.545084971874738e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 0.0967741935483871,
+      "grad_norm": NaN,
+      "learning_rate": 6.167226819279528e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 0.1003584229390681,
+      "grad_norm": NaN,
+      "learning_rate": 5.782172325201155e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 0.1039426523297491,
+      "grad_norm": NaN,
+      "learning_rate": 5.392295478639225e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 0.10752688172043011,
+      "grad_norm": NaN,
+      "learning_rate": 5e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.1111111111111111,
+      "grad_norm": NaN,
+      "learning_rate": 4.607704521360776e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 0.11469534050179211,
+      "grad_norm": NaN,
+      "learning_rate": 4.2178276747988446e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 0.11827956989247312,
+      "grad_norm": NaN,
+      "learning_rate": 3.832773180720475e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 0.12186379928315412,
+      "grad_norm": NaN,
+      "learning_rate": 3.4549150281252636e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 0.12544802867383512,
+      "grad_norm": NaN,
+      "learning_rate": 3.086582838174551e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 0.12903225806451613,
+      "grad_norm": NaN,
+      "learning_rate": 2.7300475013022663e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 0.13261648745519714,
+      "grad_norm": NaN,
+      "learning_rate": 2.3875071764202563e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 0.13620071684587814,
+      "grad_norm": NaN,
+      "learning_rate": 2.061073738537635e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 0.13978494623655913,
+      "grad_norm": NaN,
+      "learning_rate": 1.7527597583490822e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 0.14336917562724014,
+      "grad_norm": NaN,
+      "learning_rate": 1.4644660940672627e-05,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.14695340501792115,
+      "grad_norm": NaN,
+      "learning_rate": 1.1979701719998453e-05,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 0.15053763440860216,
+      "grad_norm": NaN,
+      "learning_rate": 9.549150281252633e-06,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 0.15412186379928317,
+      "grad_norm": NaN,
+      "learning_rate": 7.367991782295391e-06,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 0.15770609318996415,
+      "grad_norm": NaN,
+      "learning_rate": 5.449673790581611e-06,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 0.16129032258064516,
+      "grad_norm": NaN,
+      "learning_rate": 3.8060233744356633e-06,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 0.16487455197132617,
+      "grad_norm": NaN,
+      "learning_rate": 2.4471741852423237e-06,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 0.16845878136200718,
+      "grad_norm": NaN,
+      "learning_rate": 1.3815039801161721e-06,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 0.17204301075268819,
+      "grad_norm": NaN,
+      "learning_rate": 6.15582970243117e-07,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 0.17562724014336917,
+      "grad_norm": NaN,
+      "learning_rate": 1.5413331334360182e-07,
+      "loss": 0.0,
+      "step": 49
+    },
+    {
+      "epoch": 0.17921146953405018,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.17921146953405018,
+      "eval_loss": NaN,
+      "eval_runtime": 27.5615,
+      "eval_samples_per_second": 17.053,
+      "eval_steps_per_second": 8.526,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -212,7 +395,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -221,12 +404,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 1.340975697887232e+16,
+  "total_flos": 2.691884549092147e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null