Training in progress, step 1000, checkpoint

checkpoint-1000/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:619909d8f05905cdd482bfdb5a3d15c08701ab82c621307ee9c63dcb0016dd44
+oid sha256:f8262041f13b16e05a0d04e7543c6599df4e2085df8b5748d4bdf3ff610d45ab
 size 577789320

checkpoint-1000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0153b6221f15cead1b8798c5ac71d5ea2a36de6fb796e353f29dd029a02712a0
+oid sha256:e7502d80491d2506d2d361e2b088abc740a9b1147bed70ff1f5630e7f4119a1a
 size 1155772233

checkpoint-1000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad25c861e7b4be417740fc37745394735f9a504d2d18d9671b07aa1e7479a2a3
+oid sha256:5478438a1de30c854dabe597eb36bfc57fbab4a70587d21537bf0e86acc6df36
 size 14244

checkpoint-1000/trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 0.4741882085800171,
+  "best_metric": 0.4463141858577728,
   "best_model_checkpoint": "mikhail-panzo/fil_b32_le3_s4000/checkpoint-500",
-  "epoch": 21.978021978021978,
+  "epoch": 22.22222222222222,
   "eval_steps": 500,
   "global_step": 1000,
   "is_hyper_param_search": false,
@@ -9,159 +9,159 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.098901098901099,
-      "grad_norm": 2.8254339694976807,
+      "epoch": 1.1111111111111112,
+      "grad_norm": 3.548590898513794,
       "learning_rate": 2.5e-05,
-      "loss": 0.736,
+      "loss": 0.7354,
       "step": 50
     },
     {
-      "epoch": 2.197802197802198,
-      "grad_norm": 4.995375633239746,
+      "epoch": 2.2222222222222223,
+      "grad_norm": 5.517350196838379,
       "learning_rate": 4.9500000000000004e-05,
-      "loss": 0.5913,
+      "loss": 0.5892,
       "step": 100
     },
     {
-      "epoch": 3.2967032967032965,
-      "grad_norm": 2.352879285812378,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 4.143002033233643,
       "learning_rate": 7.45e-05,
-      "loss": 0.5174,
+      "loss": 0.532,
       "step": 150
     },
     {
-      "epoch": 4.395604395604396,
-      "grad_norm": 4.0169596672058105,
+      "epoch": 4.444444444444445,
+      "grad_norm": 6.414477825164795,
       "learning_rate": 9.95e-05,
-      "loss": 0.5275,
+      "loss": 0.5284,
       "step": 200
     },
     {
-      "epoch": 5.4945054945054945,
-      "grad_norm": 5.557097434997559,
+      "epoch": 5.555555555555555,
+      "grad_norm": 2.265143394470215,
       "learning_rate": 0.0001245,
-      "loss": 0.5231,
+      "loss": 0.517,
       "step": 250
     },
     {
-      "epoch": 6.593406593406593,
-      "grad_norm": 8.755484580993652,
+      "epoch": 6.666666666666667,
+      "grad_norm": 4.954311847686768,
       "learning_rate": 0.0001495,
-      "loss": 0.5127,
+      "loss": 0.5295,
       "step": 300
     },
     {
-      "epoch": 7.6923076923076925,
-      "grad_norm": 2.5811164379119873,
+      "epoch": 7.777777777777778,
+      "grad_norm": 2.5993571281433105,
       "learning_rate": 0.00017449999999999999,
-      "loss": 0.5179,
+      "loss": 0.517,
       "step": 350
     },
     {
-      "epoch": 8.791208791208792,
-      "grad_norm": 7.064955234527588,
+      "epoch": 8.88888888888889,
+      "grad_norm": 3.27908992767334,
       "learning_rate": 0.00019950000000000002,
-      "loss": 0.4961,
+      "loss": 0.4939,
       "step": 400
     },
     {
-      "epoch": 9.89010989010989,
-      "grad_norm": 8.559194564819336,
+      "epoch": 10.0,
+      "grad_norm": 4.22811222076416,
       "learning_rate": 0.0002245,
-      "loss": 0.4976,
+      "loss": 0.5043,
       "step": 450
     },
     {
-      "epoch": 10.989010989010989,
-      "grad_norm": 7.322611331939697,
+      "epoch": 11.11111111111111,
+      "grad_norm": 2.7051336765289307,
       "learning_rate": 0.0002495,
-      "loss": 0.502,
+      "loss": 0.5028,
       "step": 500
     },
     {
-      "epoch": 10.989010989010989,
-      "eval_loss": 0.4741882085800171,
-      "eval_runtime": 7.5067,
-      "eval_samples_per_second": 21.448,
-      "eval_steps_per_second": 2.798,
+      "epoch": 11.11111111111111,
+      "eval_loss": 0.4463141858577728,
+      "eval_runtime": 16.8864,
+      "eval_samples_per_second": 9.416,
+      "eval_steps_per_second": 1.184,
       "step": 500
     },
     {
-      "epoch": 12.087912087912088,
-      "grad_norm": 3.533902883529663,
+      "epoch": 12.222222222222221,
+      "grad_norm": 3.0811643600463867,
       "learning_rate": 0.0002745,
-      "loss": 0.5146,
+      "loss": 0.545,
       "step": 550
     },
     {
-      "epoch": 13.186813186813186,
-      "grad_norm": 9.026822090148926,
+      "epoch": 13.333333333333334,
+      "grad_norm": 7.707492828369141,
       "learning_rate": 0.0002995,
-      "loss": 0.4956,
+      "loss": 0.4979,
       "step": 600
     },
     {
-      "epoch": 14.285714285714286,
-      "grad_norm": 7.3526291847229,
+      "epoch": 14.444444444444445,
+      "grad_norm": 5.85659122467041,
       "learning_rate": 0.00032450000000000003,
-      "loss": 0.5406,
+      "loss": 0.4916,
       "step": 650
     },
     {
-      "epoch": 15.384615384615385,
-      "grad_norm": 2.889016628265381,
+      "epoch": 15.555555555555555,
+      "grad_norm": 8.300439834594727,
       "learning_rate": 0.0003495,
-      "loss": 0.5153,
+      "loss": 0.5112,
       "step": 700
     },
     {
-      "epoch": 16.483516483516482,
-      "grad_norm": 7.851646423339844,
+      "epoch": 16.666666666666668,
+      "grad_norm": 10.664773941040039,
       "learning_rate": 0.0003745,
-      "loss": 0.5326,
+      "loss": 0.5312,
       "step": 750
     },
     {
-      "epoch": 17.582417582417584,
-      "grad_norm": 2.468174934387207,
+      "epoch": 17.77777777777778,
+      "grad_norm": 6.4097161293029785,
       "learning_rate": 0.0003995,
-      "loss": 0.5575,
+      "loss": 0.6087,
       "step": 800
     },
     {
-      "epoch": 18.681318681318682,
-      "grad_norm": 10.550352096557617,
+      "epoch": 18.88888888888889,
+      "grad_norm": 5.435739994049072,
       "learning_rate": 0.0004245,
-      "loss": 0.5893,
+      "loss": 0.6127,
       "step": 850
     },
     {
-      "epoch": 19.78021978021978,
-      "grad_norm": 6.15378475189209,
+      "epoch": 20.0,
+      "grad_norm": 7.4649834632873535,
       "learning_rate": 0.00044950000000000003,
-      "loss": 0.5769,
+      "loss": 0.6904,
       "step": 900
     },
     {
-      "epoch": 20.87912087912088,
-      "grad_norm": 8.328354835510254,
+      "epoch": 21.11111111111111,
+      "grad_norm": 10.81910228729248,
       "learning_rate": 0.0004745,
-      "loss": 0.541,
+      "loss": 0.6948,
       "step": 950
     },
     {
-      "epoch": 21.978021978021978,
-      "grad_norm": 6.08469820022583,
+      "epoch": 22.22222222222222,
+      "grad_norm": 4.4550886154174805,
       "learning_rate": 0.0004995,
-      "loss": 0.5446,
+      "loss": 0.6348,
       "step": 1000
     },
     {
-      "epoch": 21.978021978021978,
-      "eval_loss": 0.4899168014526367,
-      "eval_runtime": 6.7313,
-      "eval_samples_per_second": 23.918,
-      "eval_steps_per_second": 3.12,
+      "epoch": 22.22222222222222,
+      "eval_loss": 0.5990382432937622,
+      "eval_runtime": 9.0191,
+      "eval_samples_per_second": 17.629,
+      "eval_steps_per_second": 2.218,
       "step": 1000
     }
   ],
@@ -182,7 +182,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7353193286254464.0,
+  "total_flos": 6884217255754080.0,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

checkpoint-1000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7108ae4bf8bd54061271f16e95ebc27c1c713cee5a07308d46dd91440b38577c
+oid sha256:86205dd0f5e4c109b754d1a2e96d6be52f2878290f435671b3cc191b44de8f2c
 size 5304