Training in progress, step 1400

checkpoint-1000/latest

@@ -1 +0,0 @@
-global_step1000

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5703ed353a444bcb887496e437dcc7b16f8bdaae831e4e6d4f7ae38a220a5df3
+oid sha256:d48928da7236975b7d563675900115bd40723d9bb946a5e12aeb3802cda65a29
 size 10107626487

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65d6937e778c81679ecadc56c59bb90e942cd63ffc72959111931f27b22bea57
+oid sha256:60d0660e0465b031017aa8ac057dadaad3fefbbe9a853e985e3fbcc3b9e3d14b
 size 10107626487

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7078c4a9cb10247701dadff610cea1a671df2436f25c85ce7ec7f830d5864cff
+oid sha256:fc3b54faf82a144d566ec1e13391ced58d82ba950098490d54bc7e6c12b2f665
 size 10107626487

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:632c044c424e36e9ac4d04c3db4d08e0453493e42ad88a6bbfdceadec83fd685
+oid sha256:8730071eb5352476076fecea2c16332b76414b1587e9170294cd8be64d658ae4
 size 10107626487

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c31e28791a65e07de46a9f174419774d315db15b86b1c34705af7a4bc7d99a66
+oid sha256:e1ecb36573935c513134e905740b8404414b0756f654afb39ce67f9eed0f089c
 size 10107626487

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07b03db6da0fdf58a5abbc86c3e32353bc4ea225df4814dd9b0c87094cba0a38
+oid sha256:2977e65540cd59b6e490d355dee2caa45edce69df4ccad7d08cd12e64717c58d
 size 10107626487

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1621abeb507c8e04e54646297d7dfc104a12de8750d5ef2e450703166e2e6a23
+oid sha256:a91e4e9224ed4db284c54c1c60b66d8d8db36f96ef264a6d0724d17d6d4159e2
 size 10107626487

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb1c8f3f16701d2f9fe90ecb59b247f72b3a0bde6d3db2267a5c44807cf77842
+oid sha256:88a2ea44cd053990b0f804df2960b863b7ee45e2d96229c63aeb1b4f124b33f1
 size 10107626487

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4658b6b3c7f7a48dd296534d1ec3dc6d546541ef01a0a3c11c403c23380b9050
+oid sha256:5b427970d78d97a2990a387d0111436e57780839b349873130d3acb3276e6590
 size 168086

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc1c982bfd73913563caf1b20156081f202141ef2b4942de7b04a60a2735599a
+oid sha256:25cd95fea28ed0b382448d3984a3b9c57253251be9b58ca7a14bdb60e0044800
 size 168086

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c42c53335aafb877cfbba5749485ee3cd61082a493faf56f57f8709eb5e6252
+oid sha256:a12da95cea77c0e4bf6c95cf547c52358a1a3dcc47d277ebc6a798c9caa01814
 size 168086

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ccd7e1a57d35ed384ee88e36e9a684ec710295b14dacde4ea9e97a2e0dd5e2f4
+oid sha256:883c364ce237d2eaae2ccaffd1020d9bf975a0709445f76ec935f476a9e0355c
 size 168086

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa0a5cc62566f10c81232feb782efb1f9f3ef3a38928981584b06095d904ad93
+oid sha256:1f7dee3b066702da3217434a6fdf3be471ea2359e4a7f550407604a5b0eac806
 size 168086

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8585a4986217ec7db11028c99ffd2fb8f753ad9973841c9b2be8b872fa0f5af7
+oid sha256:144d70ba74905a12764aa691b542b267d9b7798360cf8174acfb33c6916e7df6
 size 168086

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:689aff8732618feb10643e53dc24544b9e68c3fe79059dd39c4324f792c95f8e
+oid sha256:1c9ff5c63b0581bc3bf5296b9188a1bf794ee1128efd17617a1b6d6c5bd71ef6
 size 168086

{checkpoint-1000/global_step1000 → checkpoint-1400/global_step1400}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:200527511d1203250da233f27a50a276bf203ca991380ba5c7c096f1955a2752
+oid sha256:ddbafdcfd970d545721c1221e491df7ab9c50322e6312b7ec1b036b6481b3dcb
 size 168086

checkpoint-1400/latest

@@ -0,0 +1 @@
+global_step1400

{checkpoint-1000 → checkpoint-1400}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11913c7f1753ca039f36c4c2b6bbefb982404ad4203fe8ba96a2f892793f3922
+oid sha256:d81d64070bf6ada487dbdc2aaec06e52c932926918d08c4298cb20772e593337
 size 9976576392

{checkpoint-1000 → checkpoint-1400}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9877b1573d48279b0f52a3d16935fdb3bf67e40baa64bc8774bbb8005dd3519
+oid sha256:df84e357f53f857462e754af1e9a7f50bdbf8395868f7a7403b6b72c5d94b5a7
 size 3500296504

{checkpoint-1000 → checkpoint-1400}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.3333333333333333,
-  "global_step": 1000,
+  "epoch": 0.4666666666666667,
+  "global_step": 1400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -606,11 +606,251 @@
       "learning_rate": 0.0003,
       "loss": 0.7357,
       "step": 1000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6983,
+      "step": 1010
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6989,
+      "step": 1020
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6692,
+      "step": 1030
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6553,
+      "step": 1040
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6272,
+      "step": 1050
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6092,
+      "step": 1060
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.604,
+      "step": 1070
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.6494,
+      "step": 1080
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.6155,
+      "step": 1090
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003,
+      "loss": 0.5713,
+      "step": 1100
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003,
+      "loss": 0.5358,
+      "step": 1110
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003,
+      "loss": 0.5005,
+      "step": 1120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003,
+      "loss": 0.483,
+      "step": 1130
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003,
+      "loss": 0.4736,
+      "step": 1140
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003,
+      "loss": 0.4585,
+      "step": 1150
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003,
+      "loss": 0.4442,
+      "step": 1160
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003,
+      "loss": 0.4392,
+      "step": 1170
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003,
+      "loss": 0.4084,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003,
+      "loss": 0.4104,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003,
+      "loss": 0.3855,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003,
+      "loss": 0.3662,
+      "step": 1210
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003,
+      "loss": 0.3727,
+      "step": 1220
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003,
+      "loss": 0.3563,
+      "step": 1230
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003,
+      "loss": 0.3297,
+      "step": 1240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0003,
+      "loss": 0.314,
+      "step": 1250
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0003,
+      "loss": 0.2996,
+      "step": 1260
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0003,
+      "loss": 0.3,
+      "step": 1270
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0003,
+      "loss": 0.2773,
+      "step": 1280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0003,
+      "loss": 0.2665,
+      "step": 1290
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0003,
+      "loss": 0.2646,
+      "step": 1300
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0003,
+      "loss": 0.2406,
+      "step": 1310
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0003,
+      "loss": 0.2206,
+      "step": 1320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0003,
+      "loss": 0.2325,
+      "step": 1330
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0003,
+      "loss": 0.2152,
+      "step": 1340
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0003,
+      "loss": 0.2011,
+      "step": 1350
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0003,
+      "loss": 0.1953,
+      "step": 1360
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0003,
+      "loss": 0.1862,
+      "step": 1370
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0003,
+      "loss": 0.1814,
+      "step": 1380
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0003,
+      "loss": 0.1677,
+      "step": 1390
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0003,
+      "loss": 0.1657,
+      "step": 1400
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 418759311360000.0,
+  "total_flos": 586263035904000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88f41ffe5f5f16201f54cd7acfed3173d526460b94c4275a088ef0eba3eb4bac
-size 24561
+oid sha256:4b5454467270806d8ae2ed14569e1e08a427c329f3a4cebd109224d30bd23d58
+size 26131