Model save

README.md

@@ -1,15 +1,9 @@
 ---
 base_model: dmis-lab/selfbiorag_7b
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: selfbiorag-7b-dpo-full-wo-medication_qa-ep3
   results: []
@@ -20,17 +14,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # selfbiorag-7b-dpo-full-wo-medication_qa-ep3
 
-This model is a fine-tuned version of [dmis-lab/selfbiorag_7b](https://huggingface.co/dmis-lab/selfbiorag_7b) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [dmis-lab/selfbiorag_7b](https://huggingface.co/dmis-lab/selfbiorag_7b) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4771
-- Rewards/chosen: 0.4989
-- Rewards/rejected: -0.1013
-- Rewards/accuracies: 0.9886
-- Rewards/margins: 0.6002
-- Logps/rejected: -101.9239
-- Logps/chosen: -280.5883
-- Logits/rejected: -1.9346
-- Logits/chosen: -1.9269
+- Loss: 0.6636
+- Rewards/chosen: 0.0736
+- Rewards/rejected: 0.0155
+- Rewards/accuracies: 0.6536
+- Rewards/margins: 0.0581
+- Logps/rejected: -146.6977
+- Logps/chosen: -146.5513
+- Logits/rejected: -1.7452
+- Logits/chosen: -1.7907
 
 ## Model description
 
@@ -54,25 +48,27 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 8
+- num_devices: 4
 - gradient_accumulation_steps: 2
-- total_train_batch_size: 128
-- total_eval_batch_size: 64
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 3
+- num_epochs: 1
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.2815        | 2.94  | 100  | 0.4772          | 0.4986         | -0.1015          | 0.9886             | 0.6001          | -101.9416      | -280.6158    | -1.9363         | -1.9264       |
+| 0.6575        | 0.25  | 100  | 0.6771          | 0.0687         | 0.0368           | 0.6298             | 0.0319          | -144.5707      | -147.0498    | -1.7114         | -1.7555       |
+| 0.6216        | 0.5   | 200  | 0.6684          | 0.0831         | 0.0335           | 0.6546             | 0.0496          | -144.9030      | -145.6110    | -1.6971         | -1.7417       |
+| 0.6027        | 0.76  | 300  | 0.6636          | 0.0736         | 0.0155           | 0.6536             | 0.0581          | -146.6977      | -146.5513    | -1.7452         | -1.7907       |
 
 
 ### Framework versions
 
 - Transformers 4.39.0.dev0
-- Pytorch 2.2.1+cu121
+- Pytorch 2.1.2
 - Datasets 2.14.6
 - Tokenizers 0.15.2

all_results.json

@@ -1,21 +1,8 @@
 {
-    "epoch": 3.0,
-    "eval_logits/chosen": -1.926910400390625,
-    "eval_logits/rejected": -1.9345871210098267,
-    "eval_logps/chosen": -280.5882873535156,
-    "eval_logps/rejected": -101.92391967773438,
-    "eval_loss": 0.47706252336502075,
-    "eval_rewards/accuracies": 0.9886363744735718,
-    "eval_rewards/chosen": 0.49887382984161377,
-    "eval_rewards/margins": 0.6001514196395874,
-    "eval_rewards/rejected": -0.10127758979797363,
-    "eval_runtime": 18.012,
-    "eval_samples": 671,
-    "eval_samples_per_second": 37.253,
-    "eval_steps_per_second": 0.611,
-    "train_loss": 0.4128076667879142,
-    "train_runtime": 752.5051,
-    "train_samples": 4291,
-    "train_samples_per_second": 17.107,
-    "train_steps_per_second": 0.136
+    "epoch": 1.0,
+    "train_loss": 0.6319632794454656,
+    "train_runtime": 6470.6438,
+    "train_samples": 25393,
+    "train_samples_per_second": 3.924,
+    "train_steps_per_second": 0.061
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c912dec0b9ed03ba5f758a2e5385075a285aa86eb67be96ad9c5ed55a99604f
+oid sha256:012108005c6009c0ee9470a1c48ce2b3f03f2b4c410aa816dadce7d258947d1a
 size 4939116424

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de60efe3c1704a7c0e2cdb2ce40c9162b07183a5b045f3f4cca5680986250671
+oid sha256:63c21f85bb26f49a4999c16b57c2db86179344af09080ecbbda249f0734c0cd4
 size 4947390880

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c928d0f253a688091b666b8ab5bcb489732fa1e7bf0513abb69f23175a5f3c2
+oid sha256:b31e69ac88636c40ccf000bca535e8f07e5a5c0c10b62456193682e3cbd0d6ea
 size 3590619888

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.0,
-    "train_loss": 0.4128076667879142,
-    "train_runtime": 752.5051,
-    "train_samples": 4291,
-    "train_samples_per_second": 17.107,
-    "train_steps_per_second": 0.136
+    "epoch": 1.0,
+    "train_loss": 0.6319632794454656,
+    "train_runtime": 6470.6438,
+    "train_samples": 25393,
+    "train_samples_per_second": 3.924,
+    "train_steps_per_second": 0.061
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 102,
+  "global_step": 397,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.03,
-      "grad_norm": 17.39193372084505,
-      "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -1.7341437339782715,
-      "logits/rejected": -1.7096905708312988,
-      "logps/chosen": -317.9366149902344,
-      "logps/rejected": -80.48970794677734,
+      "epoch": 0.0,
+      "grad_norm": 6.630983756635612,
+      "learning_rate": 1.25e-08,
+      "logits/chosen": -1.8983474969863892,
+      "logits/rejected": -2.1539933681488037,
+      "logps/chosen": -157.78988647460938,
+      "logps/rejected": -181.83248901367188,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,185 +24,652 @@
       "step": 1
     },
     {
-      "epoch": 0.29,
-      "grad_norm": 16.94942731507905,
-      "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -1.6776922941207886,
-      "logits/rejected": -1.7127333879470825,
-      "logps/chosen": -384.7455139160156,
-      "logps/rejected": -92.23312377929688,
-      "loss": 0.6913,
-      "rewards/accuracies": 0.5347222089767456,
-      "rewards/chosen": 0.003319037612527609,
-      "rewards/margins": 0.0032567449379712343,
-      "rewards/rejected": 6.22929073870182e-05,
+      "epoch": 0.03,
+      "grad_norm": 6.468249821775856,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -1.6990889310836792,
+      "logits/rejected": -1.8442111015319824,
+      "logps/chosen": -139.26223754882812,
+      "logps/rejected": -170.71685791015625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.3958333432674408,
+      "rewards/chosen": -0.00010233223292743787,
+      "rewards/margins": -0.0004987753927707672,
+      "rewards/rejected": 0.0003964433853980154,
       "step": 10
     },
     {
-      "epoch": 0.59,
-      "grad_norm": 14.086392874180936,
-      "learning_rate": 4.880294164776785e-07,
-      "logits/chosen": -1.6398036479949951,
-      "logits/rejected": -1.716923713684082,
-      "logps/chosen": -350.6126403808594,
-      "logps/rejected": -87.44767761230469,
-      "loss": 0.6426,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 0.10100631415843964,
-      "rewards/margins": 0.09823404997587204,
-      "rewards/rejected": 0.002772263251245022,
+      "epoch": 0.05,
+      "grad_norm": 6.997695539118458,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -1.6994755268096924,
+      "logits/rejected": -1.91769540309906,
+      "logps/chosen": -172.0102996826172,
+      "logps/rejected": -182.40017700195312,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 1.7367385225952603e-05,
+      "rewards/margins": -0.0008358469349332154,
+      "rewards/rejected": 0.0008532142383046448,
       "step": 20
     },
     {
-      "epoch": 0.88,
-      "grad_norm": 10.504180758931643,
-      "learning_rate": 4.4811921048777256e-07,
-      "logits/chosen": -1.6433786153793335,
-      "logits/rejected": -1.8264563083648682,
-      "logps/chosen": -349.88677978515625,
-      "logps/rejected": -90.65472412109375,
-      "loss": 0.5266,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.37893790006637573,
-      "rewards/margins": 0.3780257999897003,
-      "rewards/rejected": 0.0009120916947722435,
+      "epoch": 0.08,
+      "grad_norm": 6.494722320175394,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -1.7388131618499756,
+      "logits/rejected": -1.8860772848129272,
+      "logps/chosen": -165.6354217529297,
+      "logps/rejected": -164.85092163085938,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.0055449954234063625,
+      "rewards/margins": 0.0032338635064661503,
+      "rewards/rejected": 0.002311132149770856,
       "step": 30
     },
     {
-      "epoch": 1.18,
-      "grad_norm": 8.111041238982029,
-      "learning_rate": 3.848300086247998e-07,
-      "logits/chosen": -1.5381871461868286,
-      "logits/rejected": -1.6925960779190063,
-      "logps/chosen": -315.11090087890625,
-      "logps/rejected": -93.9135971069336,
-      "loss": 0.446,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.5725741386413574,
-      "rewards/margins": 0.5963729619979858,
-      "rewards/rejected": -0.023798907175660133,
+      "epoch": 0.1,
+      "grad_norm": 6.118204042536185,
+      "learning_rate": 5e-07,
+      "logits/chosen": -1.7059710025787354,
+      "logits/rejected": -1.8278601169586182,
+      "logps/chosen": -175.22561645507812,
+      "logps/rejected": -130.00428771972656,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.016985954716801643,
+      "rewards/margins": 0.010590704157948494,
+      "rewards/rejected": 0.006395251955837011,
       "step": 40
     },
     {
-      "epoch": 1.47,
-      "grad_norm": 4.826424682339091,
-      "learning_rate": 3.056302334890786e-07,
-      "logits/chosen": -1.5278830528259277,
-      "logits/rejected": -1.7755340337753296,
-      "logps/chosen": -315.25958251953125,
-      "logps/rejected": -94.671875,
-      "loss": 0.3792,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.774041473865509,
-      "rewards/margins": 0.8471506237983704,
-      "rewards/rejected": -0.07310913503170013,
+      "epoch": 0.13,
+      "grad_norm": 6.45457328695851,
+      "learning_rate": 4.990326290427909e-07,
+      "logits/chosen": -1.6350853443145752,
+      "logits/rejected": -1.7747881412506104,
+      "logps/chosen": -149.56568908691406,
+      "logps/rejected": -141.73928833007812,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.03599601984024048,
+      "rewards/margins": 0.023123348131775856,
+      "rewards/rejected": 0.012872670777142048,
       "step": 50
     },
     {
-      "epoch": 1.76,
-      "grad_norm": 4.061581695339842,
-      "learning_rate": 2.1986582993616925e-07,
-      "logits/chosen": -1.3848717212677002,
-      "logits/rejected": -1.8300845623016357,
-      "logps/chosen": -288.1097106933594,
-      "logps/rejected": -95.28162384033203,
-      "loss": 0.3177,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.9560748338699341,
-      "rewards/margins": 1.0403478145599365,
-      "rewards/rejected": -0.08427302539348602,
+      "epoch": 0.15,
+      "grad_norm": 5.879221794352099,
+      "learning_rate": 4.961380026237147e-07,
+      "logits/chosen": -1.6536169052124023,
+      "logits/rejected": -1.6769773960113525,
+      "logps/chosen": -138.72972106933594,
+      "logps/rejected": -167.4161376953125,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.06843338906764984,
+      "rewards/margins": 0.040991514921188354,
+      "rewards/rejected": 0.02744186483323574,
       "step": 60
     },
     {
-      "epoch": 2.06,
-      "grad_norm": 3.5303974376494685,
-      "learning_rate": 1.3765739975820962e-07,
-      "logits/chosen": -1.3715473413467407,
-      "logits/rejected": -1.733302354812622,
-      "logps/chosen": -289.36297607421875,
-      "logps/rejected": -95.04505920410156,
-      "loss": 0.2956,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.9947406053543091,
-      "rewards/margins": 1.1034629344940186,
-      "rewards/rejected": -0.10872236639261246,
+      "epoch": 0.18,
+      "grad_norm": 6.533065923023092,
+      "learning_rate": 4.913385221630096e-07,
+      "logits/chosen": -1.5974829196929932,
+      "logits/rejected": -1.7232844829559326,
+      "logps/chosen": -149.51318359375,
+      "logps/rejected": -181.23458862304688,
+      "loss": 0.6731,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.07665083557367325,
+      "rewards/margins": 0.04112376645207405,
+      "rewards/rejected": 0.0355270616710186,
       "step": 70
     },
     {
-      "epoch": 2.35,
-      "grad_norm": 3.377643529586957,
-      "learning_rate": 6.870592321415594e-08,
-      "logits/chosen": -1.4003493785858154,
-      "logits/rejected": -1.76254403591156,
-      "logps/chosen": -277.57757568359375,
-      "logps/rejected": -103.76091003417969,
-      "loss": 0.288,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.0150843858718872,
-      "rewards/margins": 1.1521565914154053,
-      "rewards/rejected": -0.13707229495048523,
+      "epoch": 0.2,
+      "grad_norm": 5.782674302348476,
+      "learning_rate": 4.846713306847346e-07,
+      "logits/chosen": -1.5229326486587524,
+      "logits/rejected": -1.6296303272247314,
+      "logps/chosen": -154.80349731445312,
+      "logps/rejected": -166.1602783203125,
+      "loss": 0.6626,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.09903104603290558,
+      "rewards/margins": 0.07674197852611542,
+      "rewards/rejected": 0.022289065644145012,
       "step": 80
     },
     {
-      "epoch": 2.65,
-      "grad_norm": 3.2073192545802875,
-      "learning_rate": 2.1147997868658424e-08,
-      "logits/chosen": -1.383286714553833,
-      "logits/rejected": -1.8171049356460571,
-      "logps/chosen": -287.3675842285156,
-      "logps/rejected": -99.15599060058594,
-      "loss": 0.2833,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.0756561756134033,
-      "rewards/margins": 1.2087364196777344,
-      "rewards/rejected": -0.13308006525039673,
+      "epoch": 0.23,
+      "grad_norm": 5.82509109098671,
+      "learning_rate": 4.7618802536810756e-07,
+      "logits/chosen": -1.5917282104492188,
+      "logits/rejected": -1.6181108951568604,
+      "logps/chosen": -161.16732788085938,
+      "logps/rejected": -130.50424194335938,
+      "loss": 0.6583,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.09196507930755615,
+      "rewards/margins": 0.06623740494251251,
+      "rewards/rejected": 0.02572767809033394,
       "step": 90
     },
     {
-      "epoch": 2.94,
-      "grad_norm": 3.400759769088924,
-      "learning_rate": 5.956819246881184e-10,
-      "logits/chosen": -1.3634802103042603,
-      "logits/rejected": -1.7978252172470093,
-      "logps/chosen": -282.2854309082031,
-      "logps/rejected": -108.45909118652344,
-      "loss": 0.2815,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.0805023908615112,
-      "rewards/margins": 1.2250083684921265,
-      "rewards/rejected": -0.14450612664222717,
+      "epoch": 0.25,
+      "grad_norm": 5.5894075589728995,
+      "learning_rate": 4.65954258238604e-07,
+      "logits/chosen": -1.5196716785430908,
+      "logits/rejected": -1.6250730752944946,
+      "logps/chosen": -160.19277954101562,
+      "logps/rejected": -163.7660675048828,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.10788457095623016,
+      "rewards/margins": 0.06017876788973808,
+      "rewards/rejected": 0.04770580679178238,
       "step": 100
     },
     {
-      "epoch": 2.94,
-      "eval_logits/chosen": -1.9263789653778076,
-      "eval_logits/rejected": -1.9363350868225098,
-      "eval_logps/chosen": -280.6158447265625,
-      "eval_logps/rejected": -101.94155883789062,
-      "eval_loss": 0.47716638445854187,
-      "eval_rewards/accuracies": 0.9886363744735718,
-      "eval_rewards/chosen": 0.498598575592041,
-      "eval_rewards/margins": 0.6000526547431946,
-      "eval_rewards/rejected": -0.10145408660173416,
-      "eval_runtime": 18.2465,
-      "eval_samples_per_second": 36.774,
-      "eval_steps_per_second": 0.603,
+      "epoch": 0.25,
+      "eval_logits/chosen": -1.7555162906646729,
+      "eval_logits/rejected": -1.7114107608795166,
+      "eval_logps/chosen": -147.0498046875,
+      "eval_logps/rejected": -144.57066345214844,
+      "eval_loss": 0.6770787239074707,
+      "eval_rewards/accuracies": 0.6297709941864014,
+      "eval_rewards/chosen": 0.06866411119699478,
+      "eval_rewards/margins": 0.031887978315353394,
+      "eval_rewards/rejected": 0.036776136606931686,
+      "eval_runtime": 390.8735,
+      "eval_samples_per_second": 10.712,
+      "eval_steps_per_second": 0.335,
       "step": 100
     },
     {
-      "epoch": 3.0,
-      "step": 102,
+      "epoch": 0.28,
+      "grad_norm": 6.3167043153426885,
+      "learning_rate": 4.5404922808905543e-07,
+      "logits/chosen": -1.4683902263641357,
+      "logits/rejected": -1.5312354564666748,
+      "logps/chosen": -149.3756561279297,
+      "logps/rejected": -151.71046447753906,
+      "loss": 0.652,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.13409540057182312,
+      "rewards/margins": 0.09936924278736115,
+      "rewards/rejected": 0.03472616523504257,
+      "step": 110
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 5.6845547347503365,
+      "learning_rate": 4.405650675627526e-07,
+      "logits/chosen": -1.3839296102523804,
+      "logits/rejected": -1.5521198511123657,
+      "logps/chosen": -138.05992126464844,
+      "logps/rejected": -158.92774963378906,
+      "loss": 0.6546,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.12617681920528412,
+      "rewards/margins": 0.07908697426319122,
+      "rewards/rejected": 0.0470898374915123,
+      "step": 120
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 5.558751608445603,
+      "learning_rate": 4.256061301418996e-07,
+      "logits/chosen": -1.4178273677825928,
+      "logits/rejected": -1.487697958946228,
+      "logps/chosen": -163.39732360839844,
+      "logps/rejected": -152.8906707763672,
+      "loss": 0.6428,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.15949994325637817,
+      "rewards/margins": 0.12188158184289932,
+      "rewards/rejected": 0.037618380039930344,
+      "step": 130
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 6.307667169176028,
+      "learning_rate": 4.0928818255938946e-07,
+      "logits/chosen": -1.4612088203430176,
+      "logits/rejected": -1.3975152969360352,
+      "logps/chosen": -146.20932006835938,
+      "logps/rejected": -152.57125854492188,
+      "loss": 0.6423,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.16104832291603088,
+      "rewards/margins": 0.11473848670721054,
+      "rewards/rejected": 0.04630985110998154,
+      "step": 140
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 5.503685077368025,
+      "learning_rate": 3.9173750888380286e-07,
+      "logits/chosen": -1.4767869710922241,
+      "logits/rejected": -1.5618369579315186,
+      "logps/chosen": -164.35479736328125,
+      "logps/rejected": -175.6611785888672,
+      "loss": 0.6389,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.16007742285728455,
+      "rewards/margins": 0.11617257446050644,
+      "rewards/rejected": 0.043904852122068405,
+      "step": 150
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 5.299816790075546,
+      "learning_rate": 3.730899332110855e-07,
+      "logits/chosen": -1.4294887781143188,
+      "logits/rejected": -1.519938349723816,
+      "logps/chosen": -166.07269287109375,
+      "logps/rejected": -159.87347412109375,
+      "loss": 0.6355,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.15157723426818848,
+      "rewards/margins": 0.11712996661663055,
+      "rewards/rejected": 0.03444724529981613,
+      "step": 160
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 5.659366705167107,
+      "learning_rate": 3.5348976852626255e-07,
+      "logits/chosen": -1.471045970916748,
+      "logits/rejected": -1.5659321546554565,
+      "logps/chosen": -153.71054077148438,
+      "logps/rejected": -185.73037719726562,
+      "loss": 0.635,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.1759396344423294,
+      "rewards/margins": 0.13958297669887543,
+      "rewards/rejected": 0.036356642842292786,
+      "step": 170
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 5.37417422805351,
+      "learning_rate": 3.3308869986991487e-07,
+      "logits/chosen": -1.4533182382583618,
+      "logits/rejected": -1.4726018905639648,
+      "logps/chosen": -132.34890747070312,
+      "logps/rejected": -158.0144805908203,
+      "loss": 0.6308,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.16819921135902405,
+      "rewards/margins": 0.1448287069797516,
+      "rewards/rejected": 0.023370517417788506,
+      "step": 180
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 5.210502712289649,
+      "learning_rate": 3.1204461045255597e-07,
+      "logits/chosen": -1.4649099111557007,
+      "logits/rejected": -1.476950764656067,
+      "logps/chosen": -153.90042114257812,
+      "logps/rejected": -139.3481903076172,
+      "loss": 0.6285,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.18976205587387085,
+      "rewards/margins": 0.1642896831035614,
+      "rewards/rejected": 0.025472382083535194,
+      "step": 190
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 6.488049510520537,
+      "learning_rate": 2.905203598015718e-07,
+      "logits/chosen": -1.4083713293075562,
+      "logits/rejected": -1.566903829574585,
+      "logps/chosen": -147.81484985351562,
+      "logps/rejected": -159.68626403808594,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.1641792356967926,
+      "rewards/margins": 0.14985951781272888,
+      "rewards/rejected": 0.014319752342998981,
+      "step": 200
+    },
+    {
+      "epoch": 0.5,
+      "eval_logits/chosen": -1.7417274713516235,
+      "eval_logits/rejected": -1.6970983743667603,
+      "eval_logps/chosen": -145.6109619140625,
+      "eval_logps/rejected": -144.90298461914062,
+      "eval_loss": 0.6683931350708008,
+      "eval_rewards/accuracies": 0.6545801758766174,
+      "eval_rewards/chosen": 0.08305264264345169,
+      "eval_rewards/margins": 0.04959966242313385,
+      "eval_rewards/rejected": 0.033452972769737244,
+      "eval_runtime": 390.924,
+      "eval_samples_per_second": 10.711,
+      "eval_steps_per_second": 0.335,
+      "step": 200
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 6.638096992310944,
+      "learning_rate": 2.6868252339660607e-07,
+      "logits/chosen": -1.477495789527893,
+      "logits/rejected": -1.5919526815414429,
+      "logps/chosen": -135.86337280273438,
+      "logps/rejected": -156.9414825439453,
+      "loss": 0.6207,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.17845749855041504,
+      "rewards/margins": 0.15249738097190857,
+      "rewards/rejected": 0.025960108265280724,
+      "step": 210
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 5.51062470232046,
+      "learning_rate": 2.467001035473103e-07,
+      "logits/chosen": -1.5061748027801514,
+      "logits/rejected": -1.5650289058685303,
+      "logps/chosen": -140.81883239746094,
+      "logps/rejected": -166.7301025390625,
+      "loss": 0.618,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18335238099098206,
+      "rewards/margins": 0.17234531044960022,
+      "rewards/rejected": 0.011007057502865791,
+      "step": 220
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 6.215634434875552,
+      "learning_rate": 2.2474322148993558e-07,
+      "logits/chosen": -1.4497629404067993,
+      "logits/rejected": -1.5992558002471924,
+      "logps/chosen": -160.27186584472656,
+      "logps/rejected": -178.17044067382812,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.17792531847953796,
+      "rewards/margins": 0.15902863442897797,
+      "rewards/rejected": 0.018896691501140594,
+      "step": 230
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 5.720675729630434,
+      "learning_rate": 2.0298180082458716e-07,
+      "logits/chosen": -1.431714415550232,
+      "logits/rejected": -1.6490243673324585,
+      "logps/chosen": -152.18887329101562,
+      "logps/rejected": -188.75436401367188,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.17461131513118744,
+      "rewards/margins": 0.19742897152900696,
+      "rewards/rejected": -0.022817647084593773,
+      "step": 240
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 5.295028846449245,
+      "learning_rate": 1.8158425248197928e-07,
+      "logits/chosen": -1.5309040546417236,
+      "logits/rejected": -1.5308668613433838,
+      "logps/chosen": -160.09024047851562,
+      "logps/rejected": -176.86062622070312,
+      "loss": 0.613,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.196733757853508,
+      "rewards/margins": 0.1954016089439392,
+      "rewards/rejected": 0.0013321473961696029,
+      "step": 250
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 5.722335464218675,
+      "learning_rate": 1.607161713966888e-07,
+      "logits/chosen": -1.5420739650726318,
+      "logits/rejected": -1.5120036602020264,
+      "logps/chosen": -143.50515747070312,
+      "logps/rejected": -159.03140258789062,
+      "loss": 0.612,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.16295605897903442,
+      "rewards/margins": 0.16081510484218597,
+      "rewards/rejected": 0.002140958094969392,
+      "step": 260
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 6.301577955859591,
+      "learning_rate": 1.4053905497331248e-07,
+      "logits/chosen": -1.4922574758529663,
+      "logits/rejected": -1.7327592372894287,
+      "logps/chosen": -147.85516357421875,
+      "logps/rejected": -173.5520782470703,
+      "loss": 0.6071,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.1789577305316925,
+      "rewards/margins": 0.1764828860759735,
+      "rewards/rejected": 0.0024748395662754774,
+      "step": 270
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 5.816041254361184,
+      "learning_rate": 1.2120905326327596e-07,
+      "logits/chosen": -1.409450888633728,
+      "logits/rejected": -1.5283879041671753,
+      "logps/chosen": -136.27198791503906,
+      "logps/rejected": -181.03640747070312,
+      "loss": 0.6024,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 0.20897528529167175,
+      "rewards/margins": 0.21122315526008606,
+      "rewards/rejected": -0.002247871132567525,
+      "step": 280
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 5.362090199077954,
+      "learning_rate": 1.0287576052463592e-07,
+      "logits/chosen": -1.4695947170257568,
+      "logits/rejected": -1.603865623474121,
+      "logps/chosen": -147.73684692382812,
+      "logps/rejected": -169.95004272460938,
+      "loss": 0.6075,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.2076796591281891,
+      "rewards/margins": 0.214215949177742,
+      "rewards/rejected": -0.0065362839959561825,
+      "step": 290
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 6.01092868715098,
+      "learning_rate": 8.568105751695532e-08,
+      "logits/chosen": -1.4839708805084229,
+      "logits/rejected": -1.5717895030975342,
+      "logps/chosen": -134.413330078125,
+      "logps/rejected": -169.9065399169922,
+      "loss": 0.6027,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.17760583758354187,
+      "rewards/margins": 0.19261705875396729,
+      "rewards/rejected": -0.015011194162070751,
+      "step": 300
+    },
+    {
+      "epoch": 0.76,
+      "eval_logits/chosen": -1.7907291650772095,
+      "eval_logits/rejected": -1.7452377080917358,
+      "eval_logps/chosen": -146.55128479003906,
+      "eval_logps/rejected": -146.69766235351562,
+      "eval_loss": 0.6636182069778442,
+      "eval_rewards/accuracies": 0.6536259651184082,
+      "eval_rewards/chosen": 0.07364924252033234,
+      "eval_rewards/margins": 0.05814323574304581,
+      "eval_rewards/rejected": 0.015506002120673656,
+      "eval_runtime": 390.8302,
+      "eval_samples_per_second": 10.713,
+      "eval_steps_per_second": 0.335,
+      "step": 300
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 5.7348443918955985,
+      "learning_rate": 6.975801349069385e-08,
+      "logits/chosen": -1.5302002429962158,
+      "logits/rejected": -1.5517950057983398,
+      "logps/chosen": -145.04437255859375,
+      "logps/rejected": -148.3016357421875,
+      "loss": 0.6081,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.18793386220932007,
+      "rewards/margins": 0.18624603748321533,
+      "rewards/rejected": 0.0016878179740160704,
+      "step": 310
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 6.059333818089276,
+      "learning_rate": 5.5229856368582376e-08,
+      "logits/chosen": -1.5165385007858276,
+      "logits/rejected": -1.5692708492279053,
+      "logps/chosen": -132.45120239257812,
+      "logps/rejected": -134.09951782226562,
+      "loss": 0.6008,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.19101902842521667,
+      "rewards/margins": 0.21717305481433868,
+      "rewards/rejected": -0.02615402638912201,
+      "step": 320
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 5.082167006839253,
+      "learning_rate": 4.2209019088714684e-08,
+      "logits/chosen": -1.4818332195281982,
+      "logits/rejected": -1.5808608531951904,
+      "logps/chosen": -152.65586853027344,
+      "logps/rejected": -180.80148315429688,
+      "loss": 0.6028,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.1835198700428009,
+      "rewards/margins": 0.21374797821044922,
+      "rewards/rejected": -0.03022811934351921,
+      "step": 330
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 6.416095629136342,
+      "learning_rate": 3.079626948967534e-08,
+      "logits/chosen": -1.4870667457580566,
+      "logits/rejected": -1.5739234685897827,
+      "logps/chosen": -136.45143127441406,
+      "logps/rejected": -178.38784790039062,
+      "loss": 0.599,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.18290963768959045,
+      "rewards/margins": 0.20135290920734406,
+      "rewards/rejected": -0.018443269655108452,
+      "step": 340
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 6.669687170111576,
+      "learning_rate": 2.1079930471496448e-08,
+      "logits/chosen": -1.5168672800064087,
+      "logits/rejected": -1.6049163341522217,
+      "logps/chosen": -146.41165161132812,
+      "logps/rejected": -158.65560913085938,
+      "loss": 0.599,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.18101036548614502,
+      "rewards/margins": 0.20873501896858215,
+      "rewards/rejected": -0.02772463858127594,
+      "step": 350
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 5.5835416384132435,
+      "learning_rate": 1.3135196467590704e-08,
+      "logits/chosen": -1.4397341012954712,
+      "logits/rejected": -1.5808002948760986,
+      "logps/chosen": -147.1988525390625,
+      "logps/rejected": -172.8821258544922,
+      "loss": 0.5991,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 0.18593208491802216,
+      "rewards/margins": 0.21097011864185333,
+      "rewards/rejected": -0.02503802813589573,
+      "step": 360
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 5.924220316569174,
+      "learning_rate": 7.023551517463089e-09,
+      "logits/chosen": -1.5233843326568604,
+      "logits/rejected": -1.7039680480957031,
+      "logps/chosen": -150.74948120117188,
+      "logps/rejected": -168.566650390625,
+      "loss": 0.6,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18078921735286713,
+      "rewards/margins": 0.22719056904315948,
+      "rewards/rejected": -0.04640134796500206,
+      "step": 370
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 5.61259512399788,
+      "learning_rate": 2.7922934437178692e-09,
+      "logits/chosen": -1.5067952871322632,
+      "logits/rejected": -1.6481235027313232,
+      "logps/chosen": -161.50125122070312,
+      "logps/rejected": -179.09764099121094,
+      "loss": 0.5885,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 0.17576393485069275,
+      "rewards/margins": 0.214347243309021,
+      "rewards/rejected": -0.038583338260650635,
+      "step": 380
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 6.288741942272177,
+      "learning_rate": 4.741678157389739e-10,
+      "logits/chosen": -1.4176700115203857,
+      "logits/rejected": -1.6368234157562256,
+      "logps/chosen": -149.76397705078125,
+      "logps/rejected": -154.95748901367188,
+      "loss": 0.5995,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.15409687161445618,
+      "rewards/margins": 0.1682409644126892,
+      "rewards/rejected": -0.014144097454845905,
+      "step": 390
+    },
+    {
+      "epoch": 1.0,
+      "step": 397,
       "total_flos": 0.0,
-      "train_loss": 0.4128076667879142,
-      "train_runtime": 752.5051,
-      "train_samples_per_second": 17.107,
-      "train_steps_per_second": 0.136
+      "train_loss": 0.6319632794454656,
+      "train_runtime": 6470.6438,
+      "train_samples_per_second": 3.924,
+      "train_steps_per_second": 0.061
     }
   ],
   "logging_steps": 10,
-  "max_steps": 102,
+  "max_steps": 397,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 8,