Training in progress, step 1500, checkpoint

checkpoint-1500/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce2f5735e5e32165761898e54dd44169e6e20af1de01e224d2a34974d1b2e588
+oid sha256:1223e27fb67e3beeade3dd730e8edcae50bc10dc9be0da2d365ec4c1af7c6518
 size 577789320

checkpoint-1500/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ccc5af26222587c8c0b4cb952d9e105aba08fbe6d93dcb5a24f3a026d102a209
+oid sha256:df7cfa4b5ff609d48e6b29506e8abd1c59cdfbbcbf8fc0fb6386d5918a288755
 size 1155772233

checkpoint-1500/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67b25cb5cffe50f8e1fdd41327222f764da97f9e106e5c5fadae6733b1c7f2dc
+oid sha256:745d3819bd3295e0ef6dcb6ce3f22ed0a39f64918bbe93c40a7a57145abd034d
 size 14244

checkpoint-1500/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da86c4b4b06a89e55348d6269fc3197017e00fb59e7396175e717c218a809958
+oid sha256:9a10c2a667bac8f3132a44d97aa0b0d8e6bcfe8fb39437143ae7c1fbb5bbbba5
 size 1064

checkpoint-1500/trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 0.4741882085800171,
+  "best_metric": 0.4463141858577728,
   "best_model_checkpoint": "mikhail-panzo/fil_b32_le3_s4000/checkpoint-500",
-  "epoch": 33.20879120879121,
+  "epoch": 33.333333333333336,
   "eval_steps": 500,
   "global_step": 1500,
   "is_hyper_param_search": false,
@@ -9,237 +9,237 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.098901098901099,
-      "grad_norm": 2.8254339694976807,
+      "epoch": 1.1111111111111112,
+      "grad_norm": 3.548590898513794,
       "learning_rate": 2.5e-05,
-      "loss": 0.736,
+      "loss": 0.7354,
       "step": 50
     },
     {
-      "epoch": 2.197802197802198,
-      "grad_norm": 4.995375633239746,
+      "epoch": 2.2222222222222223,
+      "grad_norm": 5.517350196838379,
       "learning_rate": 4.9500000000000004e-05,
-      "loss": 0.5913,
+      "loss": 0.5892,
       "step": 100
     },
     {
-      "epoch": 3.2967032967032965,
-      "grad_norm": 2.352879285812378,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 4.143002033233643,
       "learning_rate": 7.45e-05,
-      "loss": 0.5174,
+      "loss": 0.532,
       "step": 150
     },
     {
-      "epoch": 4.395604395604396,
-      "grad_norm": 4.0169596672058105,
+      "epoch": 4.444444444444445,
+      "grad_norm": 6.414477825164795,
       "learning_rate": 9.95e-05,
-      "loss": 0.5275,
+      "loss": 0.5284,
       "step": 200
     },
     {
-      "epoch": 5.4945054945054945,
-      "grad_norm": 5.557097434997559,
+      "epoch": 5.555555555555555,
+      "grad_norm": 2.265143394470215,
       "learning_rate": 0.0001245,
-      "loss": 0.5231,
+      "loss": 0.517,
       "step": 250
     },
     {
-      "epoch": 6.593406593406593,
-      "grad_norm": 8.755484580993652,
+      "epoch": 6.666666666666667,
+      "grad_norm": 4.954311847686768,
       "learning_rate": 0.0001495,
-      "loss": 0.5127,
+      "loss": 0.5295,
       "step": 300
     },
     {
-      "epoch": 7.6923076923076925,
-      "grad_norm": 2.5811164379119873,
+      "epoch": 7.777777777777778,
+      "grad_norm": 2.5993571281433105,
       "learning_rate": 0.00017449999999999999,
-      "loss": 0.5179,
+      "loss": 0.517,
       "step": 350
     },
     {
-      "epoch": 8.791208791208792,
-      "grad_norm": 7.064955234527588,
+      "epoch": 8.88888888888889,
+      "grad_norm": 3.27908992767334,
       "learning_rate": 0.00019950000000000002,
-      "loss": 0.4961,
+      "loss": 0.4939,
       "step": 400
     },
     {
-      "epoch": 9.89010989010989,
-      "grad_norm": 8.559194564819336,
+      "epoch": 10.0,
+      "grad_norm": 4.22811222076416,
       "learning_rate": 0.0002245,
-      "loss": 0.4976,
+      "loss": 0.5043,
       "step": 450
     },
     {
-      "epoch": 10.989010989010989,
-      "grad_norm": 7.322611331939697,
+      "epoch": 11.11111111111111,
+      "grad_norm": 2.7051336765289307,
       "learning_rate": 0.0002495,
-      "loss": 0.502,
+      "loss": 0.5028,
       "step": 500
     },
     {
-      "epoch": 10.989010989010989,
-      "eval_loss": 0.4741882085800171,
-      "eval_runtime": 7.5067,
-      "eval_samples_per_second": 21.448,
-      "eval_steps_per_second": 2.798,
+      "epoch": 11.11111111111111,
+      "eval_loss": 0.4463141858577728,
+      "eval_runtime": 16.8864,
+      "eval_samples_per_second": 9.416,
+      "eval_steps_per_second": 1.184,
       "step": 500
     },
     {
-      "epoch": 12.087912087912088,
-      "grad_norm": 3.533902883529663,
+      "epoch": 12.222222222222221,
+      "grad_norm": 3.0811643600463867,
       "learning_rate": 0.0002745,
-      "loss": 0.5146,
+      "loss": 0.545,
       "step": 550
     },
     {
-      "epoch": 13.186813186813186,
-      "grad_norm": 9.026822090148926,
+      "epoch": 13.333333333333334,
+      "grad_norm": 7.707492828369141,
       "learning_rate": 0.0002995,
-      "loss": 0.4956,
+      "loss": 0.4979,
       "step": 600
     },
     {
-      "epoch": 14.285714285714286,
-      "grad_norm": 7.3526291847229,
+      "epoch": 14.444444444444445,
+      "grad_norm": 5.85659122467041,
       "learning_rate": 0.00032450000000000003,
-      "loss": 0.5406,
+      "loss": 0.4916,
       "step": 650
     },
     {
-      "epoch": 15.384615384615385,
-      "grad_norm": 2.889016628265381,
+      "epoch": 15.555555555555555,
+      "grad_norm": 8.300439834594727,
       "learning_rate": 0.0003495,
-      "loss": 0.5153,
+      "loss": 0.5112,
       "step": 700
     },
     {
-      "epoch": 16.483516483516482,
-      "grad_norm": 7.851646423339844,
+      "epoch": 16.666666666666668,
+      "grad_norm": 10.664773941040039,
       "learning_rate": 0.0003745,
-      "loss": 0.5326,
+      "loss": 0.5312,
       "step": 750
     },
     {
-      "epoch": 17.582417582417584,
-      "grad_norm": 2.468174934387207,
+      "epoch": 17.77777777777778,
+      "grad_norm": 6.4097161293029785,
       "learning_rate": 0.0003995,
-      "loss": 0.5575,
+      "loss": 0.6087,
       "step": 800
     },
     {
-      "epoch": 18.681318681318682,
-      "grad_norm": 10.550352096557617,
+      "epoch": 18.88888888888889,
+      "grad_norm": 5.435739994049072,
       "learning_rate": 0.0004245,
-      "loss": 0.5893,
+      "loss": 0.6127,
       "step": 850
     },
     {
-      "epoch": 19.78021978021978,
-      "grad_norm": 6.15378475189209,
+      "epoch": 20.0,
+      "grad_norm": 7.4649834632873535,
       "learning_rate": 0.00044950000000000003,
-      "loss": 0.5769,
+      "loss": 0.6904,
       "step": 900
     },
     {
-      "epoch": 20.87912087912088,
-      "grad_norm": 8.328354835510254,
+      "epoch": 21.11111111111111,
+      "grad_norm": 10.81910228729248,
       "learning_rate": 0.0004745,
-      "loss": 0.541,
+      "loss": 0.6948,
       "step": 950
     },
     {
-      "epoch": 21.978021978021978,
-      "grad_norm": 6.08469820022583,
+      "epoch": 22.22222222222222,
+      "grad_norm": 4.4550886154174805,
       "learning_rate": 0.0004995,
-      "loss": 0.5446,
+      "loss": 0.6348,
       "step": 1000
     },
     {
-      "epoch": 21.978021978021978,
-      "eval_loss": 0.4899168014526367,
-      "eval_runtime": 6.7313,
-      "eval_samples_per_second": 23.918,
-      "eval_steps_per_second": 3.12,
+      "epoch": 22.22222222222222,
+      "eval_loss": 0.5990382432937622,
+      "eval_runtime": 9.0191,
+      "eval_samples_per_second": 17.629,
+      "eval_steps_per_second": 2.218,
       "step": 1000
     },
     {
-      "epoch": 23.318681318681318,
-      "grad_norm": 2.388040065765381,
+      "epoch": 23.333333333333332,
+      "grad_norm": 3.101961374282837,
       "learning_rate": 0.0005245,
-      "loss": 0.4972,
+      "loss": 0.6609,
       "step": 1050
     },
     {
-      "epoch": 24.417582417582416,
-      "grad_norm": 5.6526970863342285,
+      "epoch": 24.444444444444443,
+      "grad_norm": 5.606886386871338,
       "learning_rate": 0.0005495,
-      "loss": 0.513,
+      "loss": 0.6128,
       "step": 1100
     },
     {
-      "epoch": 25.516483516483518,
-      "grad_norm": 1.4301636219024658,
+      "epoch": 25.555555555555557,
+      "grad_norm": 3.7152516841888428,
       "learning_rate": 0.0005745,
-      "loss": 0.4917,
+      "loss": 0.6197,
       "step": 1150
     },
     {
-      "epoch": 26.615384615384617,
-      "grad_norm": 5.919724464416504,
-      "learning_rate": 0.000599,
-      "loss": 0.5389,
+      "epoch": 26.666666666666668,
+      "grad_norm": 2.9999840259552,
+      "learning_rate": 0.0005995000000000001,
+      "loss": 0.6346,
       "step": 1200
     },
     {
-      "epoch": 27.714285714285715,
-      "grad_norm": 7.149015426635742,
-      "learning_rate": 0.000624,
-      "loss": 0.5462,
+      "epoch": 27.77777777777778,
+      "grad_norm": 3.5235953330993652,
+      "learning_rate": 0.0006245000000000001,
+      "loss": 0.6246,
       "step": 1250
     },
     {
-      "epoch": 28.813186813186814,
-      "grad_norm": 7.119908809661865,
-      "learning_rate": 0.0006490000000000001,
-      "loss": 0.5536,
+      "epoch": 28.88888888888889,
+      "grad_norm": 3.9665956497192383,
+      "learning_rate": 0.0006495,
+      "loss": 0.6143,
       "step": 1300
     },
     {
-      "epoch": 29.912087912087912,
-      "grad_norm": 8.452560424804688,
+      "epoch": 30.0,
+      "grad_norm": 2.920466661453247,
       "learning_rate": 0.000674,
-      "loss": 0.5798,
+      "loss": 0.6245,
       "step": 1350
     },
     {
-      "epoch": 31.01098901098901,
-      "grad_norm": 5.2523579597473145,
+      "epoch": 31.11111111111111,
+      "grad_norm": 16.839616775512695,
       "learning_rate": 0.000699,
-      "loss": 0.6361,
+      "loss": 0.8179,
       "step": 1400
     },
     {
-      "epoch": 32.10989010989011,
-      "grad_norm": 6.702617168426514,
-      "learning_rate": 0.000724,
-      "loss": 0.6097,
+      "epoch": 32.22222222222222,
+      "grad_norm": 0.23306678235530853,
+      "learning_rate": 0.0007235000000000001,
+      "loss": 1.8511,
       "step": 1450
     },
     {
-      "epoch": 33.20879120879121,
-      "grad_norm": NaN,
-      "learning_rate": 0.0007485,
-      "loss": 0.732,
+      "epoch": 33.333333333333336,
+      "grad_norm": 0.6476005911827087,
+      "learning_rate": 0.000748,
+      "loss": 1.7172,
       "step": 1500
     },
     {
-      "epoch": 33.20879120879121,
-      "eval_loss": 0.8121626973152161,
-      "eval_runtime": 7.8589,
-      "eval_samples_per_second": 20.486,
-      "eval_steps_per_second": 2.672,
+      "epoch": 33.333333333333336,
+      "eval_loss": 1.566503882408142,
+      "eval_runtime": 8.82,
+      "eval_samples_per_second": 18.027,
+      "eval_steps_per_second": 2.268,
       "step": 1500
     }
   ],
@@ -260,7 +260,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.1033285484668832e+16,
+  "total_flos": 1.0329358944735072e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

checkpoint-1500/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:14843871eecbc1c54c5866a837dba6eb3435378311d34cfc4d98271e475bc5d9
+oid sha256:86205dd0f5e4c109b754d1a2e96d6be52f2878290f435671b3cc191b44de8f2c
 size 5304