Training in progress, step 900

checkpoint-500/latest

@@ -1 +0,0 @@
-global_step500

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:784c0410b2a330cc433f981f1f5121bebabb1f3c1d781da96b8ef18c2e0db0c5
+oid sha256:c836b489002b4f2bc5a29c06d9ac1ff0ff9ee30d6de5c19ef31aa606835ed938
 size 10107626487

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab7a186e1928203b48ef1c8148819f63ee35e5e289f3d284193685d064f813fb
+oid sha256:4638b20ef605dce6f96e75096cbfbe87a7877b1abd3a9b5afa3af8cc52517378
 size 10107626487

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84d6ee17959a269d6d3e3b64d1b432370d3b086eb26b333e9a61a532db3253be
+oid sha256:dfa69d6467f9bab3fe28d1cafc0998fc3f9c75a2e03996a04ce6fba11aeb676b
 size 10107626487

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c55b861ec085a2534503a9e5de7134d033ee8cb3625c35db2f3528370ae95c8d
+oid sha256:d12fa247d8406d10eafbdbb9e572469bb46469fb29b3ac3c995838172c4d47bf
 size 10107626487

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c905c2c37c0f2f94f4a5ae2d10d79032961adea61c10e1cbb2614f6e20d67fda
+oid sha256:27053d51c07b64de70a3af83bd9016c36eb79b84a0dd2fb87265c9b049096f2d
 size 10107626487

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4824286c51d845b36a99e1044983ef94a086ad5d007203581ec617fd89edb112
+oid sha256:22b618f502d939e19b4a622a3fef3afa25bf90479b8a9d95f720c791005f095f
 size 10107626487

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70ae89087b135570ef769d71356ac53cb1162eeecd4e647c2ea4069b8e19094b
+oid sha256:02425bbc1465b95b6560156504de32ec52855dd852eaf031c47a5568fe26e41d
 size 10107626487

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83612a4a188eb3093e946a69b95f9a0c3ddbdfe58f70767b69309388acb7cc83
+oid sha256:c6b125eda1defc963593bac1d3086653fad3dd4e8dc70af33e84bc47f2edb5c7
 size 10107626487

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8f31bdab7ff163280101e1923f64561cf4a6d974c46612977fe58c68ac04e22
+oid sha256:ecaf7b76d8592c78c8d76cedb0651a24329208a93db287965af7bdfc9587e4cb
 size 168086

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3129eceffca8659953e6eac5b948129a0984740d232358cc99e9dab7d4426c27
+oid sha256:fe3a2cfd541e47e579feddb0c9c70ece50f40de01bea95d993706f57708b0f10
 size 168086

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:251ecd563c2937490b8ba62da01d1c0bdd9597f24d5d788954b9c7073385b0ba
+oid sha256:e6c7b547ced115282b43a097bec3858ae4be3f9d8eacbfe01abbe9c124852b28
 size 168086

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:541652ce8ea40c93f93845b200b752fd459ea34dc30b036e763b7c0bbfe00041
+oid sha256:7be2ade1bb19c43fa3475e66a5cac9425535095126eacd7834f3a050e3615189
 size 168086

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35ea0ff60f07ad502cd98ed64c92b5bf21728b26f8ec4ad810862d72b85b770f
+oid sha256:46d8ecc23ea617dfdf47c0cf31104948b3d98ae33c56c21b8064ff14b2b7b4e4
 size 168086

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85e23b7fcb735641a5db63cc2d01383e208488fea01050c3b293389297ce7410
+oid sha256:b541819e7f1bd67ada94b000d3d6ae85f17e1992c76dab6be7f76e14f3fd99ff
 size 168086

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a80c0240d46abab60a4dda1cd710e6cd7d2379e125e99d7f698dc8732ee786ed
+oid sha256:752a19e2ff2dce2c503cd3c5d3a9a87205f3073b5dff8606dc8ceae6c5635e86
 size 168086

{checkpoint-500/global_step500 → checkpoint-900/global_step900}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdeed66d696bbb7c4c3335a09232fd9fad0c780a4b1c78f93d28188e80446add
+oid sha256:6d0a0f4b76828975e030fcb18822398f75abb2aa403baaf7b4b0837b13cd85d3
 size 168086

checkpoint-900/latest

@@ -0,0 +1 @@
+global_step900

{checkpoint-500 → checkpoint-900}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be1ee9759da30984c76dadddd1d8c7c94c3758ab4a57cf347cff3fa275944df4
+oid sha256:99a737b5b9f70216ba3b2df80ced780c4425ff708bbf429fde9862ab588ca6c4
 size 9976576392

{checkpoint-500 → checkpoint-900}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c8789b1f2769d0cd20c1574875ba9b08d058a97d8713b8a2a36e33e481dd3d1
+oid sha256:fb8020484c2edfec4b31c0e0297c66b7bcc1add6fb17a98cf8c8de7f9aeef9bd
 size 3500296504

{checkpoint-500 → checkpoint-900}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.16666666666666666,
-  "global_step": 500,
+  "epoch": 0.3,
+  "global_step": 900,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -306,11 +306,251 @@
       "learning_rate": 0.0003,
       "loss": 2.1092,
       "step": 500
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.0715,
+      "step": 510
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.0473,
+      "step": 520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 2.0566,
+      "step": 530
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 1.9818,
+      "step": 540
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 1.9592,
+      "step": 550
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.8939,
+      "step": 560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.8441,
+      "step": 570
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.808,
+      "step": 580
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.765,
+      "step": 590
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.706,
+      "step": 600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.6328,
+      "step": 610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.6197,
+      "step": 620
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.6106,
+      "step": 630
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.5649,
+      "step": 640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.5001,
+      "step": 650
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.4926,
+      "step": 660
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.4682,
+      "step": 670
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.442,
+      "step": 680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.4117,
+      "step": 690
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.379,
+      "step": 700
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.3609,
+      "step": 710
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.3224,
+      "step": 720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.2853,
+      "step": 730
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2762,
+      "step": 740
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2265,
+      "step": 750
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2232,
+      "step": 760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.2075,
+      "step": 770
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.168,
+      "step": 780
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.1472,
+      "step": 790
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.118,
+      "step": 800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.1228,
+      "step": 810
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.1339,
+      "step": 820
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0853,
+      "step": 830
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0676,
+      "step": 840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0905,
+      "step": 850
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.076,
+      "step": 860
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.0202,
+      "step": 870
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.0123,
+      "step": 880
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9863,
+      "step": 890
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9347,
+      "step": 900
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 209379655680000.0,
+  "total_flos": 376883380224000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b518fc726384856248944d1be48f0582a8e8298752371b3b3f5437577456a8b7
-size 16711
+oid sha256:bf0edb971b8a90918c57c5b33c853a049ca331754786d7d176c54919449c9aaa
+size 18281