Model save

README.md

@@ -1,5 +1,4 @@
 ---
-base_model: princeton-nlp/Mistral-7B-Base-SFT-DPO
 tags:
 - trl
 - dpo
@@ -14,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [princeton-nlp/Mistral-7B-Base-SFT-DPO](https://huggingface.co/princeton-nlp/Mistral-7B-Base-SFT-DPO) on an unknown dataset.
+This model was trained from scratch on the None dataset.
 
 ## Model description
 
@@ -34,14 +33,14 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 8
-- eval_batch_size: 8
+- train_batch_size: 4
+- eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 256
-- total_eval_batch_size: 64
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -55,5 +54,5 @@ The following hyperparameters were used during training:
 
 - Transformers 4.39.3
 - Pytorch 2.1.2+cu118
-- Datasets 2.19.1
+- Datasets 2.16.1
 - Tokenizers 0.15.2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.2785977178812027,
-    "train_runtime": 11929.9898,
-    "train_samples": 102360,
-    "train_samples_per_second": 8.58,
-    "train_steps_per_second": 0.034
+    "train_loss": 0.36373490798167696,
+    "train_runtime": 5835.088,
+    "train_samples": 50000,
+    "train_samples_per_second": 8.569,
+    "train_steps_per_second": 0.067
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41b405c6b39fce54d81023410da5b0175bc34f1b707551cc87bb57315a19139d
+oid sha256:7e5deb8630443e9400286b2122958aab3235892baddf289d4dbde4dc184fd0e5
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4fcfc77bc0cce12435a691bee318c376a963ab3c60e50f0201871ef7f9f1899
+oid sha256:5fb2f3e09768a1370109befb94e79cce23f327b3f41a617eeb25535926889cff
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec4cc12a5a582b9be0861f8da52397528a0b5094e4e53a00c5c10ad9fdc740da
+oid sha256:36444111aaaa0874ed9be86eac12cf8dca245ebd1c686a45563cfc788c09cba5
 size 4540516344

runs/Jun22_01-21-37_n136-129-074/events.out.tfevents.1718990517.n136-129-074.300436.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db3c793612bcb36a08b9f60a4bdb334073f7e777c6a4b1c21d99e49e19f625ab
-size 26099
+oid sha256:7a652924d436028cbb10a75eca64eb43de02d697386ecb827f353baa2e09f3c9
+size 32645

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.2785977178812027,
-    "train_runtime": 11929.9898,
-    "train_samples": 102360,
-    "train_samples_per_second": 8.58,
-    "train_steps_per_second": 0.034
+    "train_loss": 0.36373490798167696,
+    "train_runtime": 5835.088,
+    "train_samples": 50000,
+    "train_samples_per_second": 8.569,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
-  "eval_steps": 500,
-  "global_step": 400,
+  "epoch": 0.9980806142034548,
+  "eval_steps": 10000000,
+  "global_step": 390,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 24.862652137264853,
-      "learning_rate": 1.25e-08,
-      "logits/chosen": -0.5811702013015747,
-      "logits/rejected": -0.11655431985855103,
-      "logps/chosen": -351.5902099609375,
-      "logps/rejected": -240.969970703125,
+      "grad_norm": 851.8646963671724,
+      "learning_rate": 1.282051282051282e-08,
+      "logits/chosen": -2.5583817958831787,
+      "logits/rejected": -2.4487552642822266,
+      "logps/chosen": -258.1644592285156,
+      "logps/rejected": -216.25729370117188,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,621 +25,606 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 23.69292682023629,
-      "learning_rate": 1.25e-07,
-      "logits/chosen": 0.26120826601982117,
-      "logits/rejected": 0.23706814646720886,
-      "logps/chosen": -333.1805419921875,
-      "logps/rejected": -244.67898559570312,
-      "loss": 0.6922,
-      "rewards/accuracies": 0.5173611044883728,
-      "rewards/chosen": 0.0021614907309412956,
-      "rewards/margins": 0.0021554920822381973,
-      "rewards/rejected": 5.998538654239383e-06,
+      "grad_norm": 827.2359528491244,
+      "learning_rate": 1.2820512820512818e-07,
+      "logits/chosen": -2.605931282043457,
+      "logits/rejected": -2.552781105041504,
+      "logps/chosen": -267.6236267089844,
+      "logps/rejected": -217.6671905517578,
+      "loss": 0.6967,
+      "rewards/accuracies": 0.4097222089767456,
+      "rewards/chosen": -0.03547710180282593,
+      "rewards/margins": -0.018225612118840218,
+      "rewards/rejected": -0.01725148782134056,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 18.203526649945516,
-      "learning_rate": 2.5e-07,
-      "logits/chosen": -0.017204787582159042,
-      "logits/rejected": 0.1991611272096634,
-      "logps/chosen": -320.430908203125,
-      "logps/rejected": -234.376220703125,
-      "loss": 0.669,
+      "grad_norm": 598.05709697014,
+      "learning_rate": 2.5641025641025636e-07,
+      "logits/chosen": -2.6306538581848145,
+      "logits/rejected": -2.5675768852233887,
+      "logps/chosen": -260.528564453125,
+      "logps/rejected": -207.09140014648438,
+      "loss": 0.5351,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.033605434000492096,
-      "rewards/margins": 0.04716432839632034,
-      "rewards/rejected": -0.01355889905244112,
+      "rewards/chosen": 0.4653858244419098,
+      "rewards/margins": 0.4877452850341797,
+      "rewards/rejected": -0.022359488531947136,
       "step": 20
     },
     {
-      "epoch": 0.07,
-      "grad_norm": 10.096989474079606,
-      "learning_rate": 3.75e-07,
-      "logits/chosen": -0.2575300931930542,
-      "logits/rejected": -0.4580558240413666,
-      "logps/chosen": -300.87896728515625,
-      "logps/rejected": -255.5655517578125,
-      "loss": 0.5805,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.14600001275539398,
-      "rewards/margins": 0.2884979844093323,
-      "rewards/rejected": -0.14249801635742188,
+      "epoch": 0.08,
+      "grad_norm": 777.4500662657566,
+      "learning_rate": 3.8461538461538463e-07,
+      "logits/chosen": -2.651550769805908,
+      "logits/rejected": -2.5767629146575928,
+      "logps/chosen": -250.84542846679688,
+      "logps/rejected": -198.71180725097656,
+      "loss": 0.3391,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.5810751914978027,
+      "rewards/margins": 2.4870572090148926,
+      "rewards/rejected": 0.09401801228523254,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 9.68944337059453,
-      "learning_rate": 5e-07,
-      "logits/chosen": -0.6759181022644043,
-      "logits/rejected": -0.6345951557159424,
-      "logps/chosen": -317.50872802734375,
-      "logps/rejected": -302.39630126953125,
-      "loss": 0.4819,
-      "rewards/accuracies": 0.840624988079071,
-      "rewards/chosen": 0.07102981209754944,
-      "rewards/margins": 0.6418195366859436,
-      "rewards/rejected": -0.5707896947860718,
+      "grad_norm": 344.88554576974366,
+      "learning_rate": 4.99989986344963e-07,
+      "logits/chosen": -2.6472256183624268,
+      "logits/rejected": -2.5672099590301514,
+      "logps/chosen": -243.0611114501953,
+      "logps/rejected": -193.21621704101562,
+      "loss": 0.3063,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 4.372925758361816,
+      "rewards/margins": 4.090872287750244,
+      "rewards/rejected": 0.28205329179763794,
       "step": 40
     },
     {
-      "epoch": 0.12,
-      "grad_norm": 13.329379682299182,
-      "learning_rate": 4.990486745229364e-07,
-      "logits/chosen": -0.12263472378253937,
-      "logits/rejected": 0.44540151953697205,
-      "logps/chosen": -374.64556884765625,
-      "logps/rejected": -388.1717224121094,
-      "loss": 0.3966,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.4549541473388672,
-      "rewards/margins": 1.0250240564346313,
-      "rewards/rejected": -1.4799782037734985,
+      "epoch": 0.13,
+      "grad_norm": 481.831925522066,
+      "learning_rate": 4.987893180827479e-07,
+      "logits/chosen": -2.658193588256836,
+      "logits/rejected": -2.5846261978149414,
+      "logps/chosen": -256.97283935546875,
+      "logps/rejected": -203.25177001953125,
+      "loss": 0.342,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 6.758584499359131,
+      "rewards/margins": 5.620272636413574,
+      "rewards/rejected": 1.1383121013641357,
       "step": 50
     },
     {
       "epoch": 0.15,
-      "grad_norm": 17.333516248641253,
-      "learning_rate": 4.96201938253052e-07,
-      "logits/chosen": -0.30300790071487427,
-      "logits/rejected": 0.3122316002845764,
-      "logps/chosen": -394.78106689453125,
-      "logps/rejected": -432.4813537597656,
-      "loss": 0.3861,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.7015730142593384,
-      "rewards/margins": 1.1719900369644165,
-      "rewards/rejected": -1.8735630512237549,
+      "grad_norm": 507.1864724110635,
+      "learning_rate": 4.955969343539162e-07,
+      "logits/chosen": -2.6098527908325195,
+      "logits/rejected": -2.5346760749816895,
+      "logps/chosen": -260.9481506347656,
+      "logps/rejected": -208.94192504882812,
+      "loss": 0.3189,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.727741241455078,
+      "rewards/margins": 5.257795333862305,
+      "rewards/rejected": -0.5300543308258057,
       "step": 60
     },
     {
-      "epoch": 0.17,
-      "grad_norm": 15.677534908750197,
-      "learning_rate": 4.91481456572267e-07,
-      "logits/chosen": 0.7395630478858948,
-      "logits/rejected": 1.5376254320144653,
-      "logps/chosen": -425.17236328125,
-      "logps/rejected": -448.2694396972656,
-      "loss": 0.3474,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.8609533309936523,
-      "rewards/margins": 1.3486477136611938,
-      "rewards/rejected": -2.2096011638641357,
+      "epoch": 0.18,
+      "grad_norm": 347.65616298975976,
+      "learning_rate": 4.90438392204474e-07,
+      "logits/chosen": -2.5738308429718018,
+      "logits/rejected": -2.496386766433716,
+      "logps/chosen": -291.3803405761719,
+      "logps/rejected": -228.85986328125,
+      "loss": 0.3314,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 3.918160915374756,
+      "rewards/margins": 6.1196980476379395,
+      "rewards/rejected": -2.201537609100342,
       "step": 70
     },
     {
       "epoch": 0.2,
-      "grad_norm": 17.182808543364636,
-      "learning_rate": 4.849231551964771e-07,
-      "logits/chosen": 2.598942995071411,
-      "logits/rejected": 3.4538092613220215,
-      "logps/chosen": -448.8929748535156,
-      "logps/rejected": -540.0630493164062,
-      "loss": 0.3215,
-      "rewards/accuracies": 0.890625,
-      "rewards/chosen": -1.3736767768859863,
-      "rewards/margins": 1.7528272867202759,
-      "rewards/rejected": -3.126504421234131,
+      "grad_norm": 548.6777756471274,
+      "learning_rate": 4.83354989019146e-07,
+      "logits/chosen": -2.5354433059692383,
+      "logits/rejected": -2.4591267108917236,
+      "logps/chosen": -259.90399169921875,
+      "logps/rejected": -205.8795623779297,
+      "loss": 0.2963,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 4.5924973487854,
+      "rewards/margins": 6.274462699890137,
+      "rewards/rejected": -1.6819652318954468,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 16.648755569621386,
-      "learning_rate": 4.7657694675916247e-07,
-      "logits/chosen": 2.8463895320892334,
-      "logits/rejected": 3.732513427734375,
-      "logps/chosen": -496.74005126953125,
-      "logps/rejected": -623.58984375,
-      "loss": 0.3048,
-      "rewards/accuracies": 0.878125011920929,
-      "rewards/chosen": -1.7601783275604248,
-      "rewards/margins": 1.9939384460449219,
-      "rewards/rejected": -3.7541167736053467,
+      "grad_norm": 746.9613110272085,
+      "learning_rate": 4.7440343190975353e-07,
+      "logits/chosen": -2.572143316268921,
+      "logits/rejected": -2.514286518096924,
+      "logps/chosen": -256.91656494140625,
+      "logps/rejected": -218.2171630859375,
+      "loss": 0.3067,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.8972291946411133,
+      "rewards/margins": 5.130236625671387,
+      "rewards/rejected": -2.2330079078674316,
       "step": 90
     },
     {
-      "epoch": 0.25,
-      "grad_norm": 15.972608527062494,
-      "learning_rate": 4.6650635094610966e-07,
-      "logits/chosen": 2.0133347511291504,
-      "logits/rejected": 3.3279690742492676,
-      "logps/chosen": -554.5970458984375,
-      "logps/rejected": -683.0777587890625,
-      "loss": 0.2797,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": -2.21871018409729,
-      "rewards/margins": 2.000453233718872,
-      "rewards/rejected": -4.219162940979004,
+      "epoch": 0.26,
+      "grad_norm": 292.7469627770502,
+      "learning_rate": 4.6365538373900506e-07,
+      "logits/chosen": -2.6255955696105957,
+      "logits/rejected": -2.5519518852233887,
+      "logps/chosen": -235.6707000732422,
+      "logps/rejected": -201.62594604492188,
+      "loss": 0.4639,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.8989346027374268,
+      "rewards/margins": 5.672036170959473,
+      "rewards/rejected": -1.773101806640625,
       "step": 100
     },
     {
       "epoch": 0.28,
-      "grad_norm": 16.95927334748175,
-      "learning_rate": 4.5478801107224794e-07,
-      "logits/chosen": 2.1293346881866455,
-      "logits/rejected": 3.9433817863464355,
-      "logps/chosen": -545.55078125,
-      "logps/rejected": -698.3030395507812,
-      "loss": 0.2718,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -2.248697280883789,
-      "rewards/margins": 2.459144353866577,
-      "rewards/rejected": -4.707841873168945,
+      "grad_norm": 462.4159148288855,
+      "learning_rate": 4.5119688941406386e-07,
+      "logits/chosen": -2.6220152378082275,
+      "logits/rejected": -2.5428764820098877,
+      "logps/chosen": -256.5944519042969,
+      "logps/rejected": -210.29629516601562,
+      "loss": 0.3852,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 5.447988033294678,
+      "rewards/margins": 6.571375370025635,
+      "rewards/rejected": -1.1233874559402466,
       "step": 110
     },
     {
-      "epoch": 0.3,
-      "grad_norm": 15.769838259410646,
-      "learning_rate": 4.415111107797445e-07,
-      "logits/chosen": 2.2328364849090576,
-      "logits/rejected": 3.943868637084961,
-      "logps/chosen": -547.4822998046875,
-      "logps/rejected": -709.2218017578125,
-      "loss": 0.2597,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": -2.386432409286499,
-      "rewards/margins": 2.306048631668091,
-      "rewards/rejected": -4.692481517791748,
+      "epoch": 0.31,
+      "grad_norm": 684.7134637036779,
+      "learning_rate": 4.3712768704277524e-07,
+      "logits/chosen": -2.5924530029296875,
+      "logits/rejected": -2.523179769515991,
+      "logps/chosen": -261.6965026855469,
+      "logps/rejected": -209.29666137695312,
+      "loss": 0.3703,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 4.659531116485596,
+      "rewards/margins": 6.417691707611084,
+      "rewards/rejected": -1.758161187171936,
       "step": 120
     },
     {
       "epoch": 0.33,
-      "grad_norm": 16.240997635455848,
-      "learning_rate": 4.2677669529663686e-07,
-      "logits/chosen": 3.3713316917419434,
-      "logits/rejected": 4.970644950866699,
-      "logps/chosen": -669.5197143554688,
-      "logps/rejected": -839.8416748046875,
-      "loss": 0.2523,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -3.3710944652557373,
-      "rewards/margins": 2.5790421962738037,
-      "rewards/rejected": -5.950136661529541,
+      "grad_norm": 363.69452359861873,
+      "learning_rate": 4.2156040946718343e-07,
+      "logits/chosen": -2.5601067543029785,
+      "logits/rejected": -2.4928698539733887,
+      "logps/chosen": -251.21463012695312,
+      "logps/rejected": -198.01596069335938,
+      "loss": 0.3349,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 3.6531822681427,
+      "rewards/margins": 6.222817897796631,
+      "rewards/rejected": -2.569636583328247,
       "step": 130
     },
     {
-      "epoch": 0.35,
-      "grad_norm": 16.664869807154886,
-      "learning_rate": 4.106969024216348e-07,
-      "logits/chosen": 3.0220611095428467,
-      "logits/rejected": 4.610594749450684,
-      "logps/chosen": -647.0032958984375,
-      "logps/rejected": -834.1439208984375,
-      "loss": 0.2514,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -3.284292221069336,
-      "rewards/margins": 2.7818052768707275,
-      "rewards/rejected": -6.066097259521484,
+      "epoch": 0.36,
+      "grad_norm": 388.6865732944718,
+      "learning_rate": 4.046196825665637e-07,
+      "logits/chosen": -2.58420729637146,
+      "logits/rejected": -2.5159454345703125,
+      "logps/chosen": -269.7867431640625,
+      "logps/rejected": -217.6550750732422,
+      "loss": 0.3662,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.2337214946746826,
+      "rewards/margins": 5.692519187927246,
+      "rewards/rejected": -2.4587976932525635,
       "step": 140
     },
     {
       "epoch": 0.38,
-      "grad_norm": 17.103959159416473,
-      "learning_rate": 3.933941090877615e-07,
-      "logits/chosen": 1.9788957834243774,
-      "logits/rejected": 3.797266721725464,
-      "logps/chosen": -657.1544799804688,
-      "logps/rejected": -866.92236328125,
-      "loss": 0.2465,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -3.3205840587615967,
-      "rewards/margins": 2.8848683834075928,
-      "rewards/rejected": -6.205452919006348,
+      "grad_norm": 392.10882995088383,
+      "learning_rate": 3.864411275486261e-07,
+      "logits/chosen": -2.568713426589966,
+      "logits/rejected": -2.5009543895721436,
+      "logps/chosen": -263.0735778808594,
+      "logps/rejected": -213.8876495361328,
+      "loss": 0.3955,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 4.510898590087891,
+      "rewards/margins": 6.606268882751465,
+      "rewards/rejected": -2.095369815826416,
       "step": 150
     },
     {
-      "epoch": 0.4,
-      "grad_norm": 22.71759647433438,
-      "learning_rate": 3.75e-07,
-      "logits/chosen": 2.598877429962158,
-      "logits/rejected": 3.922821044921875,
-      "logps/chosen": -650.6119995117188,
-      "logps/rejected": -860.2496337890625,
-      "loss": 0.2424,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -3.585509777069092,
-      "rewards/margins": 2.7439045906066895,
-      "rewards/rejected": -6.3294148445129395,
+      "epoch": 0.41,
+      "grad_norm": 466.21844913881523,
+      "learning_rate": 3.671702752161759e-07,
+      "logits/chosen": -2.567312717437744,
+      "logits/rejected": -2.497217893600464,
+      "logps/chosen": -244.8683624267578,
+      "logps/rejected": -200.1343536376953,
+      "loss": 0.3547,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.441751480102539,
+      "rewards/margins": 6.709475040435791,
+      "rewards/rejected": -4.267723083496094,
       "step": 160
     },
     {
-      "epoch": 0.42,
-      "grad_norm": 17.154680074008297,
-      "learning_rate": 3.5565456543517485e-07,
-      "logits/chosen": 1.2129310369491577,
-      "logits/rejected": 3.644993543624878,
-      "logps/chosen": -650.40576171875,
-      "logps/rejected": -869.5897216796875,
-      "loss": 0.245,
-      "rewards/accuracies": 0.8656250238418579,
-      "rewards/chosen": -3.348802089691162,
-      "rewards/margins": 3.0448169708251953,
-      "rewards/rejected": -6.393619537353516,
+      "epoch": 0.44,
+      "grad_norm": 681.7420438130807,
+      "learning_rate": 3.4696140090121375e-07,
+      "logits/chosen": -2.5749361515045166,
+      "logits/rejected": -2.509636640548706,
+      "logps/chosen": -266.73236083984375,
+      "logps/rejected": -213.6396026611328,
+      "loss": 0.3163,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.3756332397460938,
+      "rewards/margins": 6.7328901290893555,
+      "rewards/rejected": -4.357257843017578,
       "step": 170
     },
     {
-      "epoch": 0.45,
-      "grad_norm": 15.288786440112402,
-      "learning_rate": 3.355050358314172e-07,
-      "logits/chosen": 2.0979018211364746,
-      "logits/rejected": 3.6165339946746826,
-      "logps/chosen": -733.31298828125,
-      "logps/rejected": -946.0720825195312,
-      "loss": 0.225,
-      "rewards/accuracies": 0.909375011920929,
-      "rewards/chosen": -3.9488494396209717,
-      "rewards/margins": 3.014504909515381,
-      "rewards/rejected": -6.963354587554932,
+      "epoch": 0.46,
+      "grad_norm": 450.6259386496744,
+      "learning_rate": 3.259762893935617e-07,
+      "logits/chosen": -2.635408878326416,
+      "logits/rejected": -2.547847270965576,
+      "logps/chosen": -237.7583465576172,
+      "logps/rejected": -188.80947875976562,
+      "loss": 0.3732,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.5650315284729004,
+      "rewards/margins": 5.900453090667725,
+      "rewards/rejected": -3.335421323776245,
       "step": 180
     },
     {
-      "epoch": 0.47,
-      "grad_norm": 15.20089211524797,
-      "learning_rate": 3.147047612756302e-07,
-      "logits/chosen": 1.049578309059143,
-      "logits/rejected": 3.2230868339538574,
-      "logps/chosen": -655.8287963867188,
-      "logps/rejected": -909.56787109375,
-      "loss": 0.2177,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": -3.187329053878784,
-      "rewards/margins": 3.4261791706085205,
-      "rewards/rejected": -6.613508701324463,
+      "epoch": 0.49,
+      "grad_norm": 393.57273774416257,
+      "learning_rate": 3.0438293975154184e-07,
+      "logits/chosen": -2.5968377590179443,
+      "logits/rejected": -2.520313024520874,
+      "logps/chosen": -259.823974609375,
+      "logps/rejected": -205.4634246826172,
+      "loss": 0.3175,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 2.9979615211486816,
+      "rewards/margins": 6.678049564361572,
+      "rewards/rejected": -3.6800880432128906,
       "step": 190
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 19.21517389497067,
-      "learning_rate": 2.934120444167326e-07,
-      "logits/chosen": 2.0917961597442627,
-      "logits/rejected": 4.381856918334961,
-      "logps/chosen": -707.9210205078125,
-      "logps/rejected": -967.8511962890625,
-      "loss": 0.2291,
+      "epoch": 0.51,
+      "grad_norm": 569.8752439327117,
+      "learning_rate": 2.823542203635138e-07,
+      "logits/chosen": -2.62797212600708,
+      "logits/rejected": -2.5373997688293457,
+      "logps/chosen": -274.06500244140625,
+      "logps/rejected": -219.7933807373047,
+      "loss": 0.36,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.029782772064209,
-      "rewards/margins": 3.380286455154419,
-      "rewards/rejected": -7.410069465637207,
+      "rewards/chosen": 3.8748526573181152,
+      "rewards/margins": 7.579891204833984,
+      "rewards/rejected": -3.705038070678711,
       "step": 200
     },
     {
-      "epoch": 0.53,
-      "grad_norm": 17.876619392703006,
-      "learning_rate": 2.717889356869146e-07,
-      "logits/chosen": 2.075894832611084,
-      "logits/rejected": 3.812873363494873,
-      "logps/chosen": -664.9110717773438,
-      "logps/rejected": -898.7711791992188,
-      "loss": 0.2335,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -3.6399245262145996,
-      "rewards/margins": 3.0123069286346436,
-      "rewards/rejected": -6.652230739593506,
+      "epoch": 0.54,
+      "grad_norm": 528.723840543068,
+      "learning_rate": 2.600664850273538e-07,
+      "logits/chosen": -2.612349033355713,
+      "logits/rejected": -2.537264347076416,
+      "logps/chosen": -266.8662109375,
+      "logps/rejected": -212.73904418945312,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 2.512554883956909,
+      "rewards/margins": 6.339517116546631,
+      "rewards/rejected": -3.8269622325897217,
       "step": 210
     },
     {
-      "epoch": 0.55,
-      "grad_norm": 16.42311250323521,
-      "learning_rate": 2.5e-07,
-      "logits/chosen": 2.077141523361206,
-      "logits/rejected": 4.0336527824401855,
-      "logps/chosen": -714.0510864257812,
-      "logps/rejected": -951.3372802734375,
-      "loss": 0.2163,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -4.000287055969238,
-      "rewards/margins": 3.160945415496826,
-      "rewards/rejected": -7.161231994628906,
+      "epoch": 0.56,
+      "grad_norm": 499.14391994879634,
+      "learning_rate": 2.3769816112703045e-07,
+      "logits/chosen": -2.6316843032836914,
+      "logits/rejected": -2.567474842071533,
+      "logps/chosen": -257.08892822265625,
+      "logps/rejected": -214.96658325195312,
+      "loss": 0.3779,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 2.75276517868042,
+      "rewards/margins": 5.872605323791504,
+      "rewards/rejected": -3.119840145111084,
       "step": 220
     },
     {
-      "epoch": 0.57,
-      "grad_norm": 21.86260854020408,
-      "learning_rate": 2.2821106431308543e-07,
-      "logits/chosen": 1.8970081806182861,
-      "logits/rejected": 3.8517441749572754,
-      "logps/chosen": -711.104248046875,
-      "logps/rejected": -952.9786987304688,
-      "loss": 0.2307,
+      "epoch": 0.59,
+      "grad_norm": 411.504863411568,
+      "learning_rate": 2.1542832120881677e-07,
+      "logits/chosen": -2.6690242290496826,
+      "logits/rejected": -2.5815441608428955,
+      "logps/chosen": -266.91094970703125,
+      "logps/rejected": -217.81674194335938,
+      "loss": 0.33,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.102777481079102,
-      "rewards/margins": 3.2044379711151123,
-      "rewards/rejected": -7.307215213775635,
+      "rewards/chosen": 3.8953208923339844,
+      "rewards/margins": 6.836805820465088,
+      "rewards/rejected": -2.9414849281311035,
       "step": 230
     },
     {
-      "epoch": 0.6,
-      "grad_norm": 17.415535830140726,
-      "learning_rate": 2.065879555832674e-07,
-      "logits/chosen": 1.93063485622406,
-      "logits/rejected": 3.716691255569458,
-      "logps/chosen": -733.0238037109375,
-      "logps/rejected": -995.0330200195312,
-      "loss": 0.2135,
-      "rewards/accuracies": 0.8968750238418579,
-      "rewards/chosen": -4.205197811126709,
-      "rewards/margins": 3.2796833515167236,
-      "rewards/rejected": -7.4848809242248535,
+      "epoch": 0.61,
+      "grad_norm": 592.3030101879839,
+      "learning_rate": 1.934352493925695e-07,
+      "logits/chosen": -2.65104341506958,
+      "logits/rejected": -2.6025872230529785,
+      "logps/chosen": -262.87799072265625,
+      "logps/rejected": -222.07803344726562,
+      "loss": 0.3337,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 2.9119772911071777,
+      "rewards/margins": 8.015997886657715,
+      "rewards/rejected": -5.104020118713379,
       "step": 240
     },
     {
-      "epoch": 0.62,
-      "grad_norm": 23.48694643420195,
-      "learning_rate": 1.8529523872436977e-07,
-      "logits/chosen": 1.754500150680542,
-      "logits/rejected": 3.7942306995391846,
-      "logps/chosen": -756.137939453125,
-      "logps/rejected": -1002.9494018554688,
-      "loss": 0.2284,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -4.309305667877197,
-      "rewards/margins": 3.231421947479248,
-      "rewards/rejected": -7.5407280921936035,
+      "epoch": 0.64,
+      "grad_norm": 484.1153834763422,
+      "learning_rate": 1.7189501409486059e-07,
+      "logits/chosen": -2.6601402759552,
+      "logits/rejected": -2.5887746810913086,
+      "logps/chosen": -267.6604309082031,
+      "logps/rejected": -223.6746826171875,
+      "loss": 0.3508,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 2.4709649085998535,
+      "rewards/margins": 6.8468828201293945,
+      "rewards/rejected": -4.375916957855225,
       "step": 250
     },
     {
-      "epoch": 0.65,
-      "grad_norm": 16.97115932824073,
-      "learning_rate": 1.6449496416858282e-07,
-      "logits/chosen": 2.1520519256591797,
-      "logits/rejected": 4.258932590484619,
-      "logps/chosen": -750.8233642578125,
-      "logps/rejected": -1022.2374267578125,
-      "loss": 0.2136,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": -4.413332939147949,
-      "rewards/margins": 3.5410499572753906,
-      "rewards/rejected": -7.954381465911865,
+      "epoch": 0.67,
+      "grad_norm": 369.72495233490616,
+      "learning_rate": 1.5098005849021078e-07,
+      "logits/chosen": -2.6492981910705566,
+      "logits/rejected": -2.5904128551483154,
+      "logps/chosen": -261.8894958496094,
+      "logps/rejected": -209.9693145751953,
+      "loss": 0.3216,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.382293224334717,
+      "rewards/margins": 6.440347194671631,
+      "rewards/rejected": -4.058054447174072,
       "step": 260
     },
     {
-      "epoch": 0.68,
-      "grad_norm": 20.50346815073402,
-      "learning_rate": 1.4434543456482518e-07,
-      "logits/chosen": 1.4445512294769287,
-      "logits/rejected": 3.574235439300537,
-      "logps/chosen": -682.8297729492188,
-      "logps/rejected": -979.3341674804688,
-      "loss": 0.1978,
-      "rewards/accuracies": 0.9156249761581421,
-      "rewards/chosen": -3.6296730041503906,
-      "rewards/margins": 3.7298316955566406,
-      "rewards/rejected": -7.359505653381348,
+      "epoch": 0.69,
+      "grad_norm": 423.81065150105087,
+      "learning_rate": 1.30857819994673e-07,
+      "logits/chosen": -2.626788377761841,
+      "logits/rejected": -2.541728973388672,
+      "logps/chosen": -270.8172912597656,
+      "logps/rejected": -231.0417022705078,
+      "loss": 0.4168,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 3.459994077682495,
+      "rewards/margins": 9.697509765625,
+      "rewards/rejected": -6.237515926361084,
       "step": 270
     },
     {
-      "epoch": 0.7,
-      "grad_norm": 16.82073433691609,
-      "learning_rate": 1.2500000000000005e-07,
-      "logits/chosen": 1.172753930091858,
-      "logits/rejected": 3.4942619800567627,
-      "logps/chosen": -675.3418579101562,
-      "logps/rejected": -974.0270385742188,
-      "loss": 0.2189,
-      "rewards/accuracies": 0.934374988079071,
-      "rewards/chosen": -3.6095943450927734,
-      "rewards/margins": 3.8261497020721436,
-      "rewards/rejected": -7.435744285583496,
+      "epoch": 0.72,
+      "grad_norm": 328.5138374152781,
+      "learning_rate": 1.116893898236716e-07,
+      "logits/chosen": -2.656203508377075,
+      "logits/rejected": -2.6001851558685303,
+      "logps/chosen": -269.64996337890625,
+      "logps/rejected": -220.4279327392578,
+      "loss": 0.3201,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.4338252544403076,
+      "rewards/margins": 6.923757076263428,
+      "rewards/rejected": -4.489931583404541,
       "step": 280
     },
     {
-      "epoch": 0.72,
-      "grad_norm": 27.148115499609514,
-      "learning_rate": 1.0660589091223854e-07,
-      "logits/chosen": 1.3045436143875122,
-      "logits/rejected": 3.874147891998291,
-      "logps/chosen": -724.1644287109375,
-      "logps/rejected": -1020.5606689453125,
-      "loss": 0.2159,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -3.9626336097717285,
-      "rewards/margins": 3.8746466636657715,
-      "rewards/rejected": -7.837281227111816,
+      "epoch": 0.74,
+      "grad_norm": 382.1224954400422,
+      "learning_rate": 9.362822335518062e-08,
+      "logits/chosen": -2.6185498237609863,
+      "logits/rejected": -2.572524309158325,
+      "logps/chosen": -267.46600341796875,
+      "logps/rejected": -217.64089965820312,
+      "loss": 0.319,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.6880507469177246,
+      "rewards/margins": 6.601712703704834,
+      "rewards/rejected": -3.9136624336242676,
       "step": 290
     },
     {
-      "epoch": 0.75,
-      "grad_norm": 18.896563678409045,
-      "learning_rate": 8.930309757836516e-08,
-      "logits/chosen": 1.5070204734802246,
-      "logits/rejected": 3.8179619312286377,
-      "logps/chosen": -756.2379150390625,
-      "logps/rejected": -1010.8900146484375,
-      "loss": 0.2118,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.342662811279297,
-      "rewards/margins": 3.381243944168091,
-      "rewards/rejected": -7.72390604019165,
+      "epoch": 0.77,
+      "grad_norm": 546.200687023906,
+      "learning_rate": 7.681891162260015e-08,
+      "logits/chosen": -2.6388938426971436,
+      "logits/rejected": -2.5842864513397217,
+      "logps/chosen": -273.81378173828125,
+      "logps/rejected": -221.57211303710938,
+      "loss": 0.3443,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.8565659523010254,
+      "rewards/margins": 6.786903381347656,
+      "rewards/rejected": -3.9303371906280518,
       "step": 300
     },
     {
-      "epoch": 0.78,
-      "grad_norm": 18.302278636631012,
-      "learning_rate": 7.322330470336313e-08,
-      "logits/chosen": 2.013995885848999,
-      "logits/rejected": 4.006863117218018,
-      "logps/chosen": -755.84326171875,
-      "logps/rejected": -1039.4290771484375,
-      "loss": 0.2007,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -4.390773296356201,
-      "rewards/margins": 3.617499589920044,
-      "rewards/rejected": -8.008273124694824,
+      "epoch": 0.79,
+      "grad_norm": 307.4415182886001,
+      "learning_rate": 6.139602377230247e-08,
+      "logits/chosen": -2.604128360748291,
+      "logits/rejected": -2.5361759662628174,
+      "logps/chosen": -277.45074462890625,
+      "logps/rejected": -216.7763671875,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.8939478397369385,
+      "rewards/margins": 7.20609188079834,
+      "rewards/rejected": -4.3121442794799805,
       "step": 310
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 19.269057964941258,
-      "learning_rate": 5.848888922025552e-08,
-      "logits/chosen": 1.4602447748184204,
-      "logits/rejected": 3.709857940673828,
-      "logps/chosen": -771.56640625,
-      "logps/rejected": -1046.9013671875,
-      "loss": 0.2211,
-      "rewards/accuracies": 0.903124988079071,
-      "rewards/chosen": -4.392203330993652,
-      "rewards/margins": 3.6757659912109375,
-      "rewards/rejected": -8.067970275878906,
+      "epoch": 0.82,
+      "grad_norm": 438.0667457692161,
+      "learning_rate": 4.748302975270837e-08,
+      "logits/chosen": -2.6282718181610107,
+      "logits/rejected": -2.5831058025360107,
+      "logps/chosen": -260.8962707519531,
+      "logps/rejected": -205.3025665283203,
+      "loss": 0.3343,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.4093146324157715,
+      "rewards/margins": 6.292300701141357,
+      "rewards/rejected": -3.882986068725586,
       "step": 320
     },
     {
-      "epoch": 0.82,
-      "grad_norm": 16.212116381856944,
-      "learning_rate": 4.521198892775202e-08,
-      "logits/chosen": 1.5877026319503784,
-      "logits/rejected": 3.5275306701660156,
-      "logps/chosen": -748.9736328125,
-      "logps/rejected": -1030.6241455078125,
-      "loss": 0.1902,
-      "rewards/accuracies": 0.909375011920929,
-      "rewards/chosen": -4.198099613189697,
-      "rewards/margins": 3.724585771560669,
-      "rewards/rejected": -7.922685146331787,
+      "epoch": 0.84,
+      "grad_norm": 997.9385970550169,
+      "learning_rate": 3.5191311859445795e-08,
+      "logits/chosen": -2.6487433910369873,
+      "logits/rejected": -2.5906193256378174,
+      "logps/chosen": -264.0924377441406,
+      "logps/rejected": -218.33743286132812,
+      "loss": 0.3351,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.3637747764587402,
+      "rewards/margins": 6.7286200523376465,
+      "rewards/rejected": -3.364844799041748,
       "step": 330
     },
     {
-      "epoch": 0.85,
-      "grad_norm": 23.048514547738275,
-      "learning_rate": 3.349364905389032e-08,
-      "logits/chosen": 1.2227389812469482,
-      "logits/rejected": 3.192277193069458,
-      "logps/chosen": -744.6568603515625,
-      "logps/rejected": -1009.3792724609375,
-      "loss": 0.2061,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -4.285494804382324,
-      "rewards/margins": 3.405372142791748,
-      "rewards/rejected": -7.690866947174072,
+      "epoch": 0.87,
+      "grad_norm": 381.15409194754363,
+      "learning_rate": 2.4619273049795996e-08,
+      "logits/chosen": -2.6333811283111572,
+      "logits/rejected": -2.577167510986328,
+      "logps/chosen": -259.5494689941406,
+      "logps/rejected": -211.8303680419922,
+      "loss": 0.2972,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 3.1570422649383545,
+      "rewards/margins": 7.332770347595215,
+      "rewards/rejected": -4.175727844238281,
       "step": 340
     },
     {
-      "epoch": 0.88,
-      "grad_norm": 18.4900810885199,
-      "learning_rate": 2.3423053240837514e-08,
-      "logits/chosen": 1.2598426342010498,
-      "logits/rejected": 3.358072280883789,
-      "logps/chosen": -737.2872314453125,
-      "logps/rejected": -1013.8024291992188,
-      "loss": 0.2239,
-      "rewards/accuracies": 0.871874988079071,
-      "rewards/chosen": -4.232865333557129,
-      "rewards/margins": 3.510840892791748,
-      "rewards/rejected": -7.743706703186035,
+      "epoch": 0.9,
+      "grad_norm": 832.2502586276644,
+      "learning_rate": 1.5851549164932115e-08,
+      "logits/chosen": -2.6442418098449707,
+      "logits/rejected": -2.595158576965332,
+      "logps/chosen": -269.0829162597656,
+      "logps/rejected": -227.48281860351562,
+      "loss": 0.3277,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.1007628440856934,
+      "rewards/margins": 6.698919773101807,
+      "rewards/rejected": -3.598156452178955,
       "step": 350
     },
     {
-      "epoch": 0.9,
-      "grad_norm": 16.48305540217272,
-      "learning_rate": 1.507684480352292e-08,
-      "logits/chosen": 1.188299298286438,
-      "logits/rejected": 3.3616530895233154,
-      "logps/chosen": -726.2432861328125,
-      "logps/rejected": -1018.1263427734375,
-      "loss": 0.1958,
+      "epoch": 0.92,
+      "grad_norm": 372.63658491482676,
+      "learning_rate": 8.958331366609423e-09,
+      "logits/chosen": -2.6463985443115234,
+      "logits/rejected": -2.5795321464538574,
+      "logps/chosen": -274.5993957519531,
+      "logps/rejected": -220.4040069580078,
+      "loss": 0.3835,
       "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.075113773345947,
-      "rewards/margins": 3.6948330402374268,
-      "rewards/rejected": -7.769946098327637,
+      "rewards/chosen": 2.770512819290161,
+      "rewards/margins": 6.775577545166016,
+      "rewards/rejected": -4.005064964294434,
       "step": 360
     },
-    {
-      "epoch": 0.93,
-      "grad_norm": 20.27854050236199,
-      "learning_rate": 8.518543427732949e-09,
-      "logits/chosen": 1.1218559741973877,
-      "logits/rejected": 3.376429319381714,
-      "logps/chosen": -706.4921875,
-      "logps/rejected": -1009.96533203125,
-      "loss": 0.199,
-      "rewards/accuracies": 0.9156249761581421,
-      "rewards/chosen": -4.078927516937256,
-      "rewards/margins": 3.7843894958496094,
-      "rewards/rejected": -7.863317966461182,
-      "step": 370
-    },
     {
       "epoch": 0.95,
-      "grad_norm": 21.44365501017452,
-      "learning_rate": 3.798061746947995e-09,
-      "logits/chosen": 1.2741193771362305,
-      "logits/rejected": 3.5388190746307373,
-      "logps/chosen": -727.7400512695312,
-      "logps/rejected": -996.4246215820312,
-      "loss": 0.2143,
-      "rewards/accuracies": 0.9281250238418579,
-      "rewards/chosen": -4.139595985412598,
-      "rewards/margins": 3.5642218589782715,
-      "rewards/rejected": -7.703817844390869,
-      "step": 380
+      "grad_norm": 695.4821075832409,
+      "learning_rate": 3.994804212627461e-09,
+      "logits/chosen": -2.6053659915924072,
+      "logits/rejected": -2.5696444511413574,
+      "logps/chosen": -273.0481262207031,
+      "logps/rejected": -229.8125762939453,
+      "loss": 0.3476,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.7336528301239014,
+      "rewards/margins": 6.8774590492248535,
+      "rewards/rejected": -3.143805980682373,
+      "step": 370
     },
     {
       "epoch": 0.97,
-      "grad_norm": 19.980658149789488,
-      "learning_rate": 9.513254770636137e-10,
-      "logits/chosen": 1.3589586019515991,
-      "logits/rejected": 3.3931171894073486,
-      "logps/chosen": -751.2428588867188,
-      "logps/rejected": -1019.2142333984375,
-      "loss": 0.2117,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -4.217888832092285,
-      "rewards/margins": 3.668452739715576,
-      "rewards/rejected": -7.8863420486450195,
-      "step": 390
+      "grad_norm": 284.5109598747135,
+      "learning_rate": 1.0007038696262516e-09,
+      "logits/chosen": -2.65440034866333,
+      "logits/rejected": -2.61323618888855,
+      "logps/chosen": -262.66998291015625,
+      "logps/rejected": -231.3791961669922,
+      "loss": 0.3209,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.4292311668395996,
+      "rewards/margins": 6.774644374847412,
+      "rewards/rejected": -3.3454127311706543,
+      "step": 380
     },
     {
       "epoch": 1.0,
-      "grad_norm": 19.223012927780225,
+      "grad_norm": 490.88652867728223,
       "learning_rate": 0.0,
-      "logits/chosen": 1.3647325038909912,
-      "logits/rejected": 3.651308536529541,
-      "logps/chosen": -756.8760986328125,
-      "logps/rejected": -1033.67333984375,
-      "loss": 0.2003,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -4.446074485778809,
-      "rewards/margins": 3.6355972290039062,
-      "rewards/rejected": -8.081671714782715,
-      "step": 400
+      "logits/chosen": -2.6619739532470703,
+      "logits/rejected": -2.6010966300964355,
+      "logps/chosen": -250.47537231445312,
+      "logps/rejected": -211.33810424804688,
+      "loss": 0.3426,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.711453914642334,
+      "rewards/margins": 6.628802299499512,
+      "rewards/rejected": -3.917348861694336,
+      "step": 390
     },
     {
       "epoch": 1.0,
-      "step": 400,
+      "step": 390,
       "total_flos": 0.0,
-      "train_loss": 0.2785977178812027,
-      "train_runtime": 11929.9898,
-      "train_samples_per_second": 8.58,
-      "train_steps_per_second": 0.034
+      "train_loss": 0.36373490798167696,
+      "train_runtime": 5835.088,
+      "train_samples_per_second": 8.569,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 10,
-  "max_steps": 400,
+  "max_steps": 390,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
-  "train_batch_size": 8,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c46a043ab303535917560fd89542c405187453fe88a04c8c789ec363f76563bc
-size 6328
+oid sha256:527970ec9a0d5db7ce1f62ba6ec3a4699cc73881b2dacbb36f612ad48ab1823d
+size 6264