diff --git a/README.md b/README.md
index fd306713a3df1a6c0cbc0d2c377bb446762c84b3..3ecf66b011f3978a2dfcf4a0eba80029dcbb7cec 100644
--- a/README.md
+++ b/README.md
@@ -33,14 +33,18 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.001
+- learning_rate: 0.0001
 - train_batch_size: 2
 - eval_batch_size: 8
 - seed: 42
+- distributed_type: multi-GPU
+- num_devices: 32
+- total_train_batch_size: 64
+- total_eval_batch_size: 256
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- lr_scheduler_warmup_steps: 1
-- num_epochs: 1
+- lr_scheduler_warmup_steps: 150
+- num_epochs: 3
 
 ### Training results
 
@@ -48,8 +52,8 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- PEFT 0.9.0
+- PEFT 0.8.2
 - Transformers 4.41.0.dev0
-- Pytorch 2.1.2+cu118
+- Pytorch 2.0.1+cu118
 - Datasets 2.14.7
 - Tokenizers 0.19.1
\ No newline at end of file
diff --git a/adapter_config.json b/adapter_config.json
index 74aca9ff02202666665736ea809e7572aec06b03..c302c617b48c77cec64d5c0849869c16dd365f6a 100644
--- a/adapter_config.json
+++ b/adapter_config.json
@@ -21,7 +21,18 @@
   "r": 8,
   "rank_pattern": {},
   "revision": null,
-  "target_modules": ".*(vision_model|modality_projection).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj|out_proj|fc1|fc2).*$",
+  "target_modules": [
+    "fc1",
+    "down_proj",
+    "v_proj",
+    "fc2",
+    "q_proj",
+    "gate_proj",
+    "up_proj",
+    "k_proj",
+    "out_proj",
+    "o_proj"
+  ],
   "task_type": null,
   "use_dora": true,
   "use_rslora": false
diff --git a/adapter_model.safetensors b/adapter_model.safetensors
index 55fc9e2e3307df06c64719e613691420f3894a16..0b7377ced059703efbf7179779a269234116eb75 100644
--- a/adapter_model.safetensors
+++ b/adapter_model.safetensors
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52d837bec138c69794cc2273939e86b8800ad8ca8894ea91f4b467fd888f06ba
-size 10170512
+oid sha256:e44ce263e6fd885f50d82ca515b9325375b43ee36ededb75acf161ce88bc2e41
+size 48
diff --git a/global_step1851/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..57996c78624688e4c7b4dc352f1493ae1ed4cd7d
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:025b1cd5afbca8d38f5c47b3cfa0228d14de98d80bf3d7dd5e7eb3792c25f2bd
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63779928a669d42e87191df39f51b9ac33fd16de
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d952918d209940accbb6b6760be3aeaff64ea5f48b9013ce3b02dc088eae41f
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ceead56a0cc4cd89d0286198566f8660a7cae55f
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b48e6c001144fecc9cf24d6c0234aaa8f57cbb6546fb301452e3183a44c3641
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..174a6798a6f41b68f08a14dc2993ae8f0357ff2a
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f41afe7b754464fa6f66f38795a12471f2e8797aec0226deb2f58be8b5bf83d2
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63992181be1e224a5fa8f47c1a2faf7621952dd8
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f19db86359349ad86ed7bc9fa2ac6785d527c213c5060d30573aa725a60f3c08
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1826da805a4b4092fa0d599f7dfdf97d8a3016e
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5270cb7c4b931ba77e4ae3179412d47b28d319b4c9e3007368bc63919eed4f02
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..23228b7d9ca51e2f88f54382889ad803834bf7d1
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2986af12f9b1e5b1635139fbf4a19c5fe7eb2d34650d78879a887af731ae0edc
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..59ae67d7267a367973a9e14ac2074ed8205bdc77
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffb257e34930ba938b49a5890a8454cb3267493523c68badfb3e8caa3afa0984
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e3966e6dc1cdc6682902ad819225cb2a8675f06d
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30b14cc55558232a05c86540694a3306d830228bfb73ada754159eab17530ccb
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..07626b93488dd69029b03f98980fe86463fb33ed
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bc88792755a9da0da4e14d653531861fdd73db813b681f5782ec1d1a4b4fbbb
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c3c23574c362f6eb1e4f50ff3206bf77ff55937
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ccfee04a1de5af9afc9bf807e491ae6fa445afc90d08f2478c5c23668ada51
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6ee8ad23e23dfe4a16f2d897aa2bf843316a570a
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ac4aae4488ba4c5323e8d76bb0d80f94103fc7d1d4fb43f3518976865b3ee76
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d0573d834d0a635492d4bc41830a3556b92113b
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d21f3d8ca5a232578f84f269ec2411b9eb2b49f87d60a6c5e3b753c832df4afb
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ca2a64a48bf7689ee2609cd2852fc96dff8e323
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f07f1c42880fa800c5c5014d57bad92bb104337262d4be0048f9945442bf9df3
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1ab4f351b6c53a2d523545d58f82b426dc0b74a1
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99cac8d7f6437d6a2ae851c0404ba1f45b672ba92302b71c6c4a5534f7e3c9eb
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..318a86ab0d95ee808ebed3e28c54c13f6504ce80
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e12cd939152eac3931f1d1718ffc59378cbdf8173831e63d4c30d338b5560ca
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51d5113f736cd54d963fd0920066730128ad5618
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b214204ac9205421e0da3948f11bd9fe3c7fd39f91ec1be34d37701802444dc4
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cdd420a92f761b67ec3f36ffcb2efd7bd86d23ea
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3c3f9a89d26fa299bcf0f43ebc82958ce6b1bbae886b10a167d41c8edbe3e3d
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..804bff36454475ee9c772b96685364e41dd4b1c0
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d557f242ecefbd95d1e660b8f5657abbbdb80ff8187f641c64780e1ae387841
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bc77226dae57961def703f2eaf01fcf3b6f0ccc
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebb17752213250f062cab0d3a0668fc2c267071bfa8ff63947dfa3bb19e5953c
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb1e692b6f6021ef8c1bf7e2500663348590e4fa
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6555304629e1a3d37cefe0b2220e0d2f2d780e1a96ea1f280f0f2b13396f4e33
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3153bfe3dda27a4e753391e625eef63a156fdba5
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc525ebc87c1f7aa58d60a510d2aa0ab9518d5c8bf6233caa3d14555e78417e5
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e01dd10accf4ef552d7ea70390756f0e1af136c3
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:920f2e69684990176b3380c873136767031d3c5b72ca3f2daee75ad1875fecc0
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d613e7efdaad3bbd4c6e5057214652226f4cc540
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4baed8cbc04198e4e73835a48a65a53f4edc654b3a24b1c73b3f5d8624e4174a
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cffc18cda7231b009cab022317ac8ecfe1d52a27
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a23982b6bc7a206a97855ce54f5d6ad0c568f6a055d2663d88687228bda7d6bd
+size 11065661
diff --git a/global_step1851/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b15dc8348bcb9b56296ac52a2e0aa90be8af1fa
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90b5f73da5763d8bd618db0dbad365aad25216bc27f8afae03ee4660096112dd
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54acd1dcebcb5ada8bdfd8fe103895b9143b4a82
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16bcf9ac0b62036551b8acbc9db9b3c1eb0358b58319bc6b976223b0c6243457
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d0daf2ad6a1f39f8123ca30cf2db66593b0a6ba
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80c3f3b0535b060821b883c728f18955902716d2718d5e741ffd223bf002d8a6
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d9f3cbbd63a6d6bee02c28df904b6a40cbbcbb14
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bf8b9e5f6e87e3f78eb6202c28bb3c559e66a83f3f51b1b57242c738e7e3282
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d57c7421fc90a5d2394942758e47c03337e821f9
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6838931b32868aae01141d6f194e8302bf8d9a84ca685302d59280ab58466e4f
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d4addfe69697ebfa79c7913a84ad5246214d4d9b
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a56344a10b279fadb8d0d141b60209e77c02449e61f154b329f29716691c32d2
+size 11065655
diff --git a/global_step1851/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/global_step1851/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df4eb4a33add11da82fbe84a0cb43253efbd4030
--- /dev/null
+++ b/global_step1851/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76827fa3b7dad13afd552e09d03fe62c1aa41a48f7d55e9b170a50cafa6b4ec4
+size 11065655
diff --git a/global_step1851/zero_pp_rank_0_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_0_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e2076d79a5552308d1de5a367f5acdf0194be70d
--- /dev/null
+++ b/global_step1851/zero_pp_rank_0_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c93b3877dd23615af54355e8c7c39550729cec91b78fde54ea0ef6a4a3fee65b
+size 526932687
diff --git a/global_step1851/zero_pp_rank_10_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_10_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..613e219f275de8ab56faf2a9f6c582bd28d783b0
--- /dev/null
+++ b/global_step1851/zero_pp_rank_10_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d51aa79247167cab6c12bc19d50868cf6e463b2777ce76651c461044e0bc46b
+size 526935445
diff --git a/global_step1851/zero_pp_rank_11_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_11_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0debbf151694958cf6f602e7d4a135917cec08af
--- /dev/null
+++ b/global_step1851/zero_pp_rank_11_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce906b0d6b350a48c983d1ce236c363ae0823a4c8d71f9cfaee7883b191be4d4
+size 526935445
diff --git a/global_step1851/zero_pp_rank_12_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_12_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8990636c4d3eff25b9cd6c49d545bf14c063a506
--- /dev/null
+++ b/global_step1851/zero_pp_rank_12_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bec42fd58898de452f3986ca86f2f05eacb3ed429e8e4b48b79f59626707180d
+size 526935445
diff --git a/global_step1851/zero_pp_rank_13_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_13_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3fc3ee6ee69c2809c257b1f38fb6f0742f2a5316
--- /dev/null
+++ b/global_step1851/zero_pp_rank_13_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9521594521b888df18bc98094ae9d6d12e620f63587ca9ad94141b25b0e2f80
+size 526935445
diff --git a/global_step1851/zero_pp_rank_14_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_14_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..119b3f3504a97fbe1d8dc964400a18891aeb0ed4
--- /dev/null
+++ b/global_step1851/zero_pp_rank_14_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db95b3cc74d30eb5fa45d5e7e6b8b02d64719fa34afe27b597ff84a3becd9be8
+size 526935445
diff --git a/global_step1851/zero_pp_rank_15_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_15_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82e1d642fc51bbaf21ebba23192e6a373af56c2b
--- /dev/null
+++ b/global_step1851/zero_pp_rank_15_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eab529ea20ced93e80dca7ecab4c69a05e748341f0259daaf863895a288f1e83
+size 526935445
diff --git a/global_step1851/zero_pp_rank_16_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_16_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..95043308738c6b76bb5782d777bd8415e11e8ac0
--- /dev/null
+++ b/global_step1851/zero_pp_rank_16_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f7f1a538fdd8f7665cc455c03bdd8f6c1b8bb15062048f748ceba7f3e99dd53
+size 526935445
diff --git a/global_step1851/zero_pp_rank_17_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_17_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..39a0e073186dbb22511bd7f35c992b503aa7f29f
--- /dev/null
+++ b/global_step1851/zero_pp_rank_17_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc00c0885cdddde76db513e739f58456e95a255fe0a9d4f009463079ed0b102e
+size 526935445
diff --git a/global_step1851/zero_pp_rank_18_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_18_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b242104d60abdfa1d58c1628e7d8ef70372594b
--- /dev/null
+++ b/global_step1851/zero_pp_rank_18_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d73143ba31897864028bc99357fb7b6432573df23106a13f343bb971ee88ff9
+size 526935445
diff --git a/global_step1851/zero_pp_rank_19_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_19_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b35ba2ef968aec77d0b0e4b97d881947a6bcede7
--- /dev/null
+++ b/global_step1851/zero_pp_rank_19_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d05a6d71b9bb4d67a0941daf6834d79d8f7ad7d77cdd0fa9f5c58aec902f9d01
+size 526935445
diff --git a/global_step1851/zero_pp_rank_1_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_1_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..892b4f739b85f96b9b53245b9aa22fb1cacac923
--- /dev/null
+++ b/global_step1851/zero_pp_rank_1_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abd26f545bbaf53db05106c2d762f978c218701550b8742c63fd6569097fca55
+size 526932687
diff --git a/global_step1851/zero_pp_rank_20_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_20_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b651ce7e350049408d1ca42f8b4cd5cec7ba2a78
--- /dev/null
+++ b/global_step1851/zero_pp_rank_20_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f69acd690339f4e607dc2ed0be2f448a55cd6b14f0ec223d7fcb37bc2e8edcfd
+size 526935445
diff --git a/global_step1851/zero_pp_rank_21_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_21_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9fac9c38a49aa68a41a6b55fa6aec9d8ec7f251a
--- /dev/null
+++ b/global_step1851/zero_pp_rank_21_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48a5abec20e07c4989d51c3b011a577aea82c3d56a50ac65eb1f534e40350060
+size 526935445
diff --git a/global_step1851/zero_pp_rank_22_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_22_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d06d88ba2da877159f2c364cc5edde04da172f41
--- /dev/null
+++ b/global_step1851/zero_pp_rank_22_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d09274d1752934484470667d48d5b5738b7ca64408882b1550526fcbfc9daa5
+size 526935445
diff --git a/global_step1851/zero_pp_rank_23_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_23_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b9736da18953242276b08e6c7886a723cc505b0
--- /dev/null
+++ b/global_step1851/zero_pp_rank_23_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4399efae8098e8ad1893e3c42cbfbedd1496f1d3c87dada2469a821ed59fbf11
+size 526935445
diff --git a/global_step1851/zero_pp_rank_24_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_24_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b57cf91d653d20f5a77531690c5375d7cf3bafdf
--- /dev/null
+++ b/global_step1851/zero_pp_rank_24_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca15f6d2cb1876a05d5bdb8c5187f8a493a31cd5ba0e2caaf82398c3c7b7e564
+size 526935445
diff --git a/global_step1851/zero_pp_rank_25_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_25_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e345418ac8053db450b58ba1b8e7d9e1020aedc
--- /dev/null
+++ b/global_step1851/zero_pp_rank_25_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d47c1e2c1ebf8a617a57636759c1cd221d03252cc3dfb4c115b00ce42c9550a3
+size 526935445
diff --git a/global_step1851/zero_pp_rank_26_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_26_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..007241be1fc4d44083c96a9ce1515d151f976695
--- /dev/null
+++ b/global_step1851/zero_pp_rank_26_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1e6769213c1ffda87bd72e9bc353b4cf1f7c7545a852378758b00e345d5ee52
+size 526935445
diff --git a/global_step1851/zero_pp_rank_27_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_27_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3271f408ed35bb8ea8b87d779df050ba8c01c4a2
--- /dev/null
+++ b/global_step1851/zero_pp_rank_27_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a6f2909589355d630264ca7b8f9acc5a5a7fd69bcf3143cde7a1ee7b30e6e1c
+size 526935445
diff --git a/global_step1851/zero_pp_rank_28_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_28_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1164aecdd864affa62485ac18f15006ecfc805e2
--- /dev/null
+++ b/global_step1851/zero_pp_rank_28_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6f4b813b6d09e66b25531e4480a8f7a4f9fc260b45eab395938d790b0b9a28c
+size 526935445
diff --git a/global_step1851/zero_pp_rank_29_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_29_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc9dd247a0baeb454ae0c0880925f74811374716
--- /dev/null
+++ b/global_step1851/zero_pp_rank_29_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22e5a23f3add0750310a95475734e6a2d76d19f18af9c9a29e10377a7d3810c1
+size 526935445
diff --git a/global_step1851/zero_pp_rank_2_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_2_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c49fa4e28b5b23c9220b4da023174eb7df2eccf5
--- /dev/null
+++ b/global_step1851/zero_pp_rank_2_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2970d8224934237bfae1b9017beeff8e5f57342233d04cecf52fde75e3b305c
+size 526932687
diff --git a/global_step1851/zero_pp_rank_30_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_30_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84350b4759c4a34e62c9ba7c0843717640548554
--- /dev/null
+++ b/global_step1851/zero_pp_rank_30_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c17d67acfa564696256b7050568d17430af2ea1ef03baec47ae82bc223f5b560
+size 526935445
diff --git a/global_step1851/zero_pp_rank_31_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_31_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c8b48af75ea5525da9be07f9d7e98b570a534380
--- /dev/null
+++ b/global_step1851/zero_pp_rank_31_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e687a167ded9d8235a454730a947cd8ca582b514c01e836b27d59e0b8bee666
+size 526935445
diff --git a/global_step1851/zero_pp_rank_3_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_3_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1787ad1ccee9b93858e291197adb83e6486386f1
--- /dev/null
+++ b/global_step1851/zero_pp_rank_3_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b17986c2695f590c854178c262291e31f4d6ff704b1dfe7c80c1cf0a339f64b
+size 526932687
diff --git a/global_step1851/zero_pp_rank_4_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_4_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..592cbcd9fa23a81b49663dfdcbc4e12f16838e9b
--- /dev/null
+++ b/global_step1851/zero_pp_rank_4_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7260aa8821b3f7a7437dd8158d67ac38373a3030d4ad6f29b77e284fdba6c99a
+size 526932687
diff --git a/global_step1851/zero_pp_rank_5_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_5_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8205005fb7a245e9179006ff7367872a70a8a6cd
--- /dev/null
+++ b/global_step1851/zero_pp_rank_5_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5265307e62e4d388d91a47fbc6ed0d48be3e52e0c2a76ed48b6a0b87e1d5064e
+size 526932687
diff --git a/global_step1851/zero_pp_rank_6_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_6_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd110dbdf698b06f54bde7f57ad30669447b3a31
--- /dev/null
+++ b/global_step1851/zero_pp_rank_6_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1df927bb99fecff071ebd14907169e2523cf79edcad3bc64fb035a66f504284
+size 526932687
diff --git a/global_step1851/zero_pp_rank_7_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_7_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ab329c5b4c3300fcb993d9de8cf52d71f0fc0208
--- /dev/null
+++ b/global_step1851/zero_pp_rank_7_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ca84ed99c23834fcf799862b452bf7320fc5f6ea9d8a7faf23984e6a63dfd21
+size 526932687
diff --git a/global_step1851/zero_pp_rank_8_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_8_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed9a241e269756fdc6fe2eaddfdb56f91da7e934
--- /dev/null
+++ b/global_step1851/zero_pp_rank_8_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:316134819603de176ae0eb1add66e4aae4cc5617b30e3a5f5e57fdaa4a4ae2ba
+size 526932687
diff --git a/global_step1851/zero_pp_rank_9_mp_rank_00_model_states.pt b/global_step1851/zero_pp_rank_9_mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1fb7773444ec0acfcbf375e4ec862ec0ccde993c
--- /dev/null
+++ b/global_step1851/zero_pp_rank_9_mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa8070f1061e72d19dff675d5109c31619d898279a538da41e72f1ec8871cdf8
+size 526932687
diff --git a/latest b/latest
index 6a0c43c2922f253815f75b4d6776ab23df4abc63..e943533aea1ffb04655dc58d931ae8c9b0a271e3 100644
--- a/latest
+++ b/latest
@@ -1 +1 @@
-global_step616
\ No newline at end of file
+global_step1851
\ No newline at end of file
diff --git a/training_args.bin b/training_args.bin
index a1731ea9689751a9839f94fd573260c0ca41404f..2d5b3723b06da95eb35d06cf1ba29fe96e145731 100644
--- a/training_args.bin
+++ b/training_args.bin
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ddb992ed9512561a06ea14b1d6d982dcfdbbe277af4c5323871726ba9f42ee0
-size 5048
+oid sha256:45990b0b3aa8bc4670448eacc0828e544fff7f7ab7eb879cfad9b07a342507bc
+size 5819