Training checkpoint at step 33500

trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 33000,
-  "best_metric": 2.480618953704834,
+  "best_global_step": 33400,
+  "best_metric": 2.48046875,
   "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-33000",
-  "epoch": 0.66,
+  "epoch": 0.67,
   "eval_steps": 100,
-  "global_step": 33000,
+  "global_step": 33500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -11888,6 +11888,186 @@
       "eval_samples_per_second": 2.471,
       "eval_steps_per_second": 1.235,
       "step": 33000
+    },
+    {
+      "epoch": 0.6605,
+      "grad_norm": 1.4210561637451704,
+      "learning_rate": 3.7724444444444445e-06,
+      "loss": 2.4701,
+      "step": 33025
+    },
+    {
+      "epoch": 0.661,
+      "grad_norm": 2.1617711578734915,
+      "learning_rate": 3.7668888888888893e-06,
+      "loss": 2.4788,
+      "step": 33050
+    },
+    {
+      "epoch": 0.6615,
+      "grad_norm": 1.5320070011690634,
+      "learning_rate": 3.7613333333333333e-06,
+      "loss": 2.47,
+      "step": 33075
+    },
+    {
+      "epoch": 0.662,
+      "grad_norm": 1.3400391346818974,
+      "learning_rate": 3.755777777777778e-06,
+      "loss": 2.4792,
+      "step": 33100
+    },
+    {
+      "epoch": 0.662,
+      "eval_loss": 2.481220006942749,
+      "eval_runtime": 41.9988,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 33100
+    },
+    {
+      "epoch": 0.6625,
+      "grad_norm": 1.769214182110644,
+      "learning_rate": 3.7502222222222225e-06,
+      "loss": 2.4669,
+      "step": 33125
+    },
+    {
+      "epoch": 0.663,
+      "grad_norm": 1.4499877924560598,
+      "learning_rate": 3.744666666666667e-06,
+      "loss": 2.47,
+      "step": 33150
+    },
+    {
+      "epoch": 0.6635,
+      "grad_norm": 1.2272721031062317,
+      "learning_rate": 3.7391111111111118e-06,
+      "loss": 2.4808,
+      "step": 33175
+    },
+    {
+      "epoch": 0.664,
+      "grad_norm": 1.9524804604619508,
+      "learning_rate": 3.7335555555555557e-06,
+      "loss": 2.4855,
+      "step": 33200
+    },
+    {
+      "epoch": 0.664,
+      "eval_loss": 2.481595516204834,
+      "eval_runtime": 42.0663,
+      "eval_samples_per_second": 2.472,
+      "eval_steps_per_second": 1.236,
+      "step": 33200
+    },
+    {
+      "epoch": 0.6645,
+      "grad_norm": 1.5855882612813827,
+      "learning_rate": 3.7280000000000006e-06,
+      "loss": 2.4839,
+      "step": 33225
+    },
+    {
+      "epoch": 0.665,
+      "grad_norm": 1.7981131055660284,
+      "learning_rate": 3.7224444444444445e-06,
+      "loss": 2.4831,
+      "step": 33250
+    },
+    {
+      "epoch": 0.6655,
+      "grad_norm": 1.8893217376664102,
+      "learning_rate": 3.7168888888888894e-06,
+      "loss": 2.4689,
+      "step": 33275
+    },
+    {
+      "epoch": 0.666,
+      "grad_norm": 1.5504407193892469,
+      "learning_rate": 3.7113333333333333e-06,
+      "loss": 2.4748,
+      "step": 33300
+    },
+    {
+      "epoch": 0.666,
+      "eval_loss": 2.480543851852417,
+      "eval_runtime": 42.0898,
+      "eval_samples_per_second": 2.471,
+      "eval_steps_per_second": 1.235,
+      "step": 33300
+    },
+    {
+      "epoch": 0.6665,
+      "grad_norm": 1.7468260304964456,
+      "learning_rate": 3.705777777777778e-06,
+      "loss": 2.478,
+      "step": 33325
+    },
+    {
+      "epoch": 0.667,
+      "grad_norm": 1.527950475107732,
+      "learning_rate": 3.700222222222222e-06,
+      "loss": 2.476,
+      "step": 33350
+    },
+    {
+      "epoch": 0.6675,
+      "grad_norm": 1.6583388548480227,
+      "learning_rate": 3.694666666666667e-06,
+      "loss": 2.4675,
+      "step": 33375
+    },
+    {
+      "epoch": 0.668,
+      "grad_norm": 1.5937276979972617,
+      "learning_rate": 3.689111111111112e-06,
+      "loss": 2.471,
+      "step": 33400
+    },
+    {
+      "epoch": 0.668,
+      "eval_loss": 2.48046875,
+      "eval_runtime": 41.9774,
+      "eval_samples_per_second": 2.478,
+      "eval_steps_per_second": 1.239,
+      "step": 33400
+    },
+    {
+      "epoch": 0.6685,
+      "grad_norm": 1.4618192489732302,
+      "learning_rate": 3.6835555555555558e-06,
+      "loss": 2.4687,
+      "step": 33425
+    },
+    {
+      "epoch": 0.669,
+      "grad_norm": 1.686028538338107,
+      "learning_rate": 3.6780000000000006e-06,
+      "loss": 2.4859,
+      "step": 33450
+    },
+    {
+      "epoch": 0.6695,
+      "grad_norm": 1.352022736772511,
+      "learning_rate": 3.6724444444444446e-06,
+      "loss": 2.4611,
+      "step": 33475
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 1.9063367987545683,
+      "learning_rate": 3.6668888888888894e-06,
+      "loss": 2.4724,
+      "step": 33500
+    },
+    {
+      "epoch": 0.67,
+      "eval_loss": 2.480919361114502,
+      "eval_runtime": 42.1121,
+      "eval_samples_per_second": 2.47,
+      "eval_steps_per_second": 1.235,
+      "step": 33500
     }
   ],
   "logging_steps": 25,
@@ -11907,7 +12087,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.405923896183593e+19,
+  "total_flos": 7.518134864593918e+19,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null