Training in progress, step 2200

checkpoint-1800/latest

@@ -1 +0,0 @@
-global_step1800

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65ac051edb6a7231e7ff80bf7b6ff872456d98c899aa44c7f0de57ec100e241a
+oid sha256:4e3de1f6ed0c91905320e4f3d6eb8149a7321097c6e422a5e66e96dd3ead20bb
 size 10107626487

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f63f4158c79b9097827e52efe21c448978712c950ba30906df8d4e81779f0705
+oid sha256:0d6049942a6346ee62050a04a487e2ff6996de4173c7e6709642eecd59ca8846
 size 10107626487

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a06a42dbae3be95f0f13a0c6a7e3d8fff63f06a1b88e67198a327c7c3f2b4fdf
+oid sha256:2c785db79058e106e14d7f10c31104d1b15877f6f2dad07cffa86889822ae089
 size 10107626487

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2d439a4859cf52b20fb93dc23b4dba8c8083d3d58b8bd2b35129a82e4f9847f
+oid sha256:84ef7caf982c837d86319b4809266bfe0034dfaac1fcaab8707e65f30a77434b
 size 10107626487

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56e9a0ca5c754b3520440fc3506fe5a4e46f2a2eade8194719f6c07e5ee7b32e
+oid sha256:5c7d71042fe43a020fe99829afa84900dd6b64aacf374a041aa004d70782d3ed
 size 10107626487

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea9a156039d51fadfd2736deea73e96bf8dfb0dbf11a6c3b10eb075a358c9b8f
+oid sha256:96a6f35eb2a5d74873f0d52cf6d8e3ee340a8cd41bb6876c29e3f05076efdef6
 size 10107626487

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:103ef36796553ded7aa4fea4a7e84fb84384939f9f20cda0a94ecf5087a56b7d
+oid sha256:cb589add5394037a80e2aa7ad1684a3712834d5ec6f3e01faf0c7d59e847b8f8
 size 10107626487

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:478d4c670c95787a5e189a35cf100f0c722538933effcc1efdf43d7a3a79ec87
+oid sha256:bb1df2b451f5442f92b20ec0c6aa921c45b654d615d6258d8b20a775c914db58
 size 10107626487

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a3559441b3540dc1b5a49a8fb7253c34731a3906e17ecc6f79b18ee56a4ba0e
+oid sha256:bc5beb79e0c17f4883e73fc1d86c62e293d253a4b710f34bdf075ea8d3c0948f
 size 168086

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c7f165ac77d9d755737a9b47099cbc30144ddc12bbdf2a30877c884fe0aa613
+oid sha256:5292ecd35a4d6b299ce503f942577436b9490e59d5037c6e7d934eedbb1ff589
 size 168086

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d7943fb9f5e3008adcd3ed98b51af946f70d290a7337c4ffbe0eb7a9b97be6c
+oid sha256:524b59f333eff994650fc2adca7b3a3643aacb6d7c3c6dc093db0c146d03f489
 size 168086

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:734fbce19436e77d545003a57bdb0fcc5fbd70f67630be95f23d1f26b813fc8b
+oid sha256:20c0eb77991260ad958007ca687b20a0afd9d26b1fed86c679ca1358dd3d2d8b
 size 168086

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb7a3c3a029b300ee81735e4b5d95db67561d27acbcc726de134c1749f3d5442
+oid sha256:d3b45f67a5c87020b1de5c66b0f84170840ae2bc56c3c0100ed1555919ac3cac
 size 168086

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afac81b2fa8bcebadbac9d41f1d41c618419c5aa49f521a8b868d88affb1b81b
+oid sha256:8913e88466a2a0af695b1a2df7aabd5441bd4f010bbb7ae1395d71c3971dc259
 size 168086

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c36b334041e232c5d22af305a776884cf9a86554d427f772411f35d878d40e0
+oid sha256:c46c8008a47e8e9afa844bf69d37341e7113bf2c75914e1ef052c526b2e95fd9
 size 168086

{checkpoint-1800/global_step1800 → checkpoint-2200/global_step2200}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4741731558644f602064546705ffa83f9879c68a3314de93ea623d611af243e5
+oid sha256:8879c52948ce81ce49a3864313623dd93244d519b021a17b8f9b0dab3f3bec72
 size 168086

checkpoint-2200/latest

@@ -0,0 +1 @@
+global_step2200

{checkpoint-1800 → checkpoint-2200}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc8b8d03a8fe3e6b5c85db79b65e4586478a90485d98555e56952666cdae15e5
+oid sha256:85d5e4fe559d56cec1b1fa412e29b6a90bbf8a69fb1db74a0c739a225f6b302f
 size 9976576392

{checkpoint-1800 → checkpoint-2200}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad1f107946442c1ba1bd23d3ab4b3b2906ea95617d47c6b2f8fe6b989084b50d
+oid sha256:0c31f01aa8839468adecb9155e8925979cd25a6c002a11bac6b7027931452e12
 size 3500296504

{checkpoint-1800 → checkpoint-2200}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6,
-  "global_step": 1800,
+  "epoch": 0.7333333333333333,
+  "global_step": 2200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1086,11 +1086,251 @@
       "learning_rate": 0.0003,
       "loss": 0.0525,
       "step": 1800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0519,
+      "step": 1810
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0478,
+      "step": 1820
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0473,
+      "step": 1830
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0473,
+      "step": 1840
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0492,
+      "step": 1850
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0451,
+      "step": 1860
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0453,
+      "step": 1870
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0439,
+      "step": 1880
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0435,
+      "step": 1890
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0435,
+      "step": 1900
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.0412,
+      "step": 1910
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.0441,
+      "step": 1920
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.04,
+      "step": 1930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.041,
+      "step": 1940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.1391,
+      "step": 1950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.1502,
+      "step": 1960
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0767,
+      "step": 1970
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0563,
+      "step": 1980
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0487,
+      "step": 1990
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.042,
+      "step": 2000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.0388,
+      "step": 2010
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.0424,
+      "step": 2020
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.0413,
+      "step": 2030
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.036,
+      "step": 2040
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.0333,
+      "step": 2050
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0334,
+      "step": 2060
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0298,
+      "step": 2070
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0302,
+      "step": 2080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003,
+      "loss": 0.0302,
+      "step": 2090
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003,
+      "loss": 0.0293,
+      "step": 2100
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003,
+      "loss": 0.0286,
+      "step": 2110
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0003,
+      "loss": 0.0279,
+      "step": 2120
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0003,
+      "loss": 0.0258,
+      "step": 2130
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0003,
+      "loss": 0.0259,
+      "step": 2140
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0003,
+      "loss": 0.0273,
+      "step": 2150
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0003,
+      "loss": 0.0253,
+      "step": 2160
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0003,
+      "loss": 0.0247,
+      "step": 2170
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0003,
+      "loss": 0.0237,
+      "step": 2180
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0003,
+      "loss": 0.0231,
+      "step": 2190
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0003,
+      "loss": 0.0213,
+      "step": 2200
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 753766760448000.0,
+  "total_flos": 921270484992000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3476f29a411313ed060c01b35f1409f573b4dc1eb8ce083996033c57780434b
-size 37121
+oid sha256:5a541254ae717180ce583645ab2027b88252e5c9429179141c869f80155d67f8
+size 38691