Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: mit
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: microsoft/phi-2
+model-index:
+- name: phi-2-gpo-renew2-b0.001-v2-i1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# phi-2-gpo-renew2-b0.001-v2-i1
+
+This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0539
+- Rewards/chosen: 0.0016
+- Rewards/rejected: 0.0019
+- Rewards/accuracies: 0.4295
+- Rewards/margins: -0.0003
+- Logps/rejected: -365.2908
+- Logps/chosen: -394.6620
+- Logits/rejected: -0.7531
+- Logits/chosen: -0.8487
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.1204        | 0.32  | 100  | 0.0538          | 0.0003         | 0.0005           | 0.4400             | -0.0002         | -366.7078      | -395.9741    | -0.7404         | -0.8344       |
+| 0.1673        | 0.64  | 200  | 0.0538          | 0.0014         | 0.0014           | 0.4465             | -0.0001         | -365.8175      | -394.9525    | -0.7452         | -0.8413       |
+| 0.1395        | 0.96  | 300  | 0.0538          | 0.0016         | 0.0018           | 0.4410             | -0.0002         | -365.4079      | -394.7079    | -0.7549         | -0.8511       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.15.2

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e177e05cff5f0817fac9232c082b983b7c50d8b8da0b885df74bb53bea61c94a
+oid sha256:4d06ce3bad599246fbbd56cba43c010341421ff96160c0f82e9618588877f88f
 size 167807296

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -0.8487240672111511,
+    "eval_logits/rejected": -0.7531492114067078,
+    "eval_logps/chosen": -394.66204833984375,
+    "eval_logps/rejected": -365.29083251953125,
+    "eval_loss": 0.053853537887334824,
+    "eval_rewards/accuracies": 0.429500013589859,
+    "eval_rewards/chosen": 0.0016431997064501047,
+    "eval_rewards/margins": -0.00030138425063341856,
+    "eval_rewards/rejected": 0.0019445838406682014,
+    "eval_runtime": 545.5213,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 3.666,
+    "eval_steps_per_second": 0.917,
+    "train_loss": 0.14698060888510484,
+    "train_runtime": 3810.9697,
+    "train_samples": 61135,
+    "train_samples_per_second": 1.312,
+    "train_steps_per_second": 0.082
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -0.8487240672111511,
+    "eval_logits/rejected": -0.7531492114067078,
+    "eval_logps/chosen": -394.66204833984375,
+    "eval_logps/rejected": -365.29083251953125,
+    "eval_loss": 0.053853537887334824,
+    "eval_rewards/accuracies": 0.429500013589859,
+    "eval_rewards/chosen": 0.0016431997064501047,
+    "eval_rewards/margins": -0.00030138425063341856,
+    "eval_rewards/rejected": 0.0019445838406682014,
+    "eval_runtime": 545.5213,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 3.666,
+    "eval_steps_per_second": 0.917
+}

runs/Apr22_16-59-25_gpu4-119-5/events.out.tfevents.1713769246.gpu4-119-5.3043320.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b8dcdfcffa0a82e78cf561a9bf54533c8804030225457f4e01a36c9ac2a3980
-size 26568
+oid sha256:51fd6d7e89f1e87c73046e5a642b494bd9d128d44c5a908b46b82e246b3696db
+size 27556

runs/Apr22_16-59-25_gpu4-119-5/events.out.tfevents.1713773602.gpu4-119-5.3043320.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec3d15566911215e4198b017a87b68d7faf27c122046b77aaa143112a3f195bc
+size 828

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.14698060888510484,
+    "train_runtime": 3810.9697,
+    "train_samples": 61135,
+    "train_samples_per_second": 1.312,
+    "train_steps_per_second": 0.082
+}

trainer_state.json

@@ -0,0 +1,526 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9984,
+  "eval_steps": 100,
+  "global_step": 312,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5625e-07,
+      "logits/chosen": -0.8750922679901123,
+      "logits/rejected": -0.7979680895805359,
+      "logps/chosen": -60.00876235961914,
+      "logps/rejected": -56.38082504272461,
+      "loss": 0.1482,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5625e-06,
+      "logits/chosen": -1.0051445960998535,
+      "logits/rejected": -0.9556077122688293,
+      "logps/chosen": -30.64621925354004,
+      "logps/rejected": -24.19979476928711,
+      "loss": 0.1565,
+      "rewards/accuracies": 0.02083333395421505,
+      "rewards/chosen": -8.15845123725012e-05,
+      "rewards/margins": -4.479259951040149e-05,
+      "rewards/rejected": -3.679191286209971e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.125e-06,
+      "logits/chosen": -1.0498403310775757,
+      "logits/rejected": -0.992475152015686,
+      "logps/chosen": -40.17894744873047,
+      "logps/rejected": -31.744924545288086,
+      "loss": 0.1421,
+      "rewards/accuracies": 0.03125,
+      "rewards/chosen": -3.2624266168568283e-06,
+      "rewards/margins": -6.817355460952967e-05,
+      "rewards/rejected": 6.491113163065165e-05,
+      "step": 20
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.6875000000000004e-06,
+      "logits/chosen": -1.0504692792892456,
+      "logits/rejected": -1.0281661748886108,
+      "logps/chosen": -23.273496627807617,
+      "logps/rejected": -18.378360748291016,
+      "loss": 0.1594,
+      "rewards/accuracies": 0.01875000074505806,
+      "rewards/chosen": -7.991820166353136e-05,
+      "rewards/margins": -0.00010900965571636334,
+      "rewards/rejected": 2.909145587182138e-05,
+      "step": 30
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989935734988098e-06,
+      "logits/chosen": -1.0432171821594238,
+      "logits/rejected": -1.0193238258361816,
+      "logps/chosen": -14.588598251342773,
+      "logps/rejected": -13.306127548217773,
+      "loss": 0.1303,
+      "rewards/accuracies": 0.012500000186264515,
+      "rewards/chosen": 4.2490195482969284e-05,
+      "rewards/margins": -2.1462084987433627e-05,
+      "rewards/rejected": 6.39522768324241e-05,
+      "step": 40
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.949188496058089e-06,
+      "logits/chosen": -1.0302413702011108,
+      "logits/rejected": -0.986968994140625,
+      "logps/chosen": -9.379155158996582,
+      "logps/rejected": -9.404864311218262,
+      "loss": 0.1469,
+      "rewards/accuracies": 0.02500000037252903,
+      "rewards/chosen": 3.0187036827555858e-05,
+      "rewards/margins": 4.167082079220563e-05,
+      "rewards/rejected": -1.1483787602628581e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8776412907378845e-06,
+      "logits/chosen": -0.9920421838760376,
+      "logits/rejected": -0.9573459625244141,
+      "logps/chosen": -17.296560287475586,
+      "logps/rejected": -15.45374584197998,
+      "loss": 0.1519,
+      "rewards/accuracies": 0.012500000186264515,
+      "rewards/chosen": 2.950535053969361e-05,
+      "rewards/margins": -1.730041731207166e-05,
+      "rewards/rejected": 4.680576239479706e-05,
+      "step": 60
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.7761938666470405e-06,
+      "logits/chosen": -1.0576204061508179,
+      "logits/rejected": -1.0229989290237427,
+      "logps/chosen": -30.714122772216797,
+      "logps/rejected": -24.89596939086914,
+      "loss": 0.1346,
+      "rewards/accuracies": 0.05000000074505806,
+      "rewards/chosen": 0.00010937159095192328,
+      "rewards/margins": -1.5012978110462427e-05,
+      "rewards/rejected": 0.0001243845617864281,
+      "step": 70
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.646121984004666e-06,
+      "logits/chosen": -1.0434526205062866,
+      "logits/rejected": -1.0283691883087158,
+      "logps/chosen": -12.912440299987793,
+      "logps/rejected": -10.21430778503418,
+      "loss": 0.1494,
+      "rewards/accuracies": 0.01875000074505806,
+      "rewards/chosen": 6.350317562464625e-05,
+      "rewards/margins": 3.430190190556459e-05,
+      "rewards/rejected": 2.9201270081102848e-05,
+      "step": 80
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4890613722044526e-06,
+      "logits/chosen": -1.005961298942566,
+      "logits/rejected": -0.9507063627243042,
+      "logps/chosen": -29.203853607177734,
+      "logps/rejected": -24.27176856994629,
+      "loss": 0.1563,
+      "rewards/accuracies": 0.02500000037252903,
+      "rewards/chosen": 0.00019098483608104289,
+      "rewards/margins": -0.00010294451203662902,
+      "rewards/rejected": 0.00029392936266958714,
+      "step": 90
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3069871595684795e-06,
+      "logits/chosen": -1.0817575454711914,
+      "logits/rejected": -1.0109978914260864,
+      "logps/chosen": -23.601749420166016,
+      "logps/rejected": -21.154720306396484,
+      "loss": 0.1204,
+      "rewards/accuracies": 0.03750000149011612,
+      "rewards/chosen": 0.0001659760600887239,
+      "rewards/margins": 1.1245434507145546e-05,
+      "rewards/rejected": 0.00015473061648663133,
+      "step": 100
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/chosen": -0.8344239592552185,
+      "eval_logits/rejected": -0.7404021620750427,
+      "eval_logps/chosen": -395.97412109375,
+      "eval_logps/rejected": -366.70782470703125,
+      "eval_loss": 0.05379165709018707,
+      "eval_rewards/accuracies": 0.4399999976158142,
+      "eval_rewards/chosen": 0.0003311456530354917,
+      "eval_rewards/margins": -0.0001964901020983234,
+      "eval_rewards/rejected": 0.0005276357405818999,
+      "eval_runtime": 545.7797,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 100
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.102189034962561e-06,
+      "logits/chosen": -1.0390229225158691,
+      "logits/rejected": -1.025102972984314,
+      "logps/chosen": -21.16196060180664,
+      "logps/rejected": -22.110755920410156,
+      "loss": 0.1488,
+      "rewards/accuracies": 0.03125,
+      "rewards/chosen": 0.00030253888689912856,
+      "rewards/margins": 5.533280273084529e-05,
+      "rewards/rejected": 0.0002472061023581773,
+      "step": 110
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8772424536302565e-06,
+      "logits/chosen": -1.029483437538147,
+      "logits/rejected": -0.9574755430221558,
+      "logps/chosen": -19.494661331176758,
+      "logps/rejected": -17.480262756347656,
+      "loss": 0.145,
+      "rewards/accuracies": 0.04374999925494194,
+      "rewards/chosen": 0.0003196864854544401,
+      "rewards/margins": 0.00018756816280074418,
+      "rewards/rejected": 0.00013211832265369594,
+      "step": 120
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.634976249348867e-06,
+      "logits/chosen": -1.107649564743042,
+      "logits/rejected": -1.076535701751709,
+      "logps/chosen": -11.74854850769043,
+      "logps/rejected": -9.875129699707031,
+      "loss": 0.1365,
+      "rewards/accuracies": 0.01875000074505806,
+      "rewards/chosen": 0.00019255562801845372,
+      "rewards/margins": 3.773098433157429e-05,
+      "rewards/rejected": 0.0001548246364109218,
+      "step": 130
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3784370602033572e-06,
+      "logits/chosen": -1.0260846614837646,
+      "logits/rejected": -0.9438373446464539,
+      "logps/chosen": -17.046716690063477,
+      "logps/rejected": -12.897377014160156,
+      "loss": 0.1655,
+      "rewards/accuracies": 0.03125,
+      "rewards/chosen": 0.00021877660765312612,
+      "rewards/margins": 0.00010056849714601412,
+      "rewards/rejected": 0.00011820811778306961,
+      "step": 140
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1108510153447352e-06,
+      "logits/chosen": -1.0065504312515259,
+      "logits/rejected": -0.9879522323608398,
+      "logps/chosen": -14.441717147827148,
+      "logps/rejected": -11.689096450805664,
+      "loss": 0.1525,
+      "rewards/accuracies": 0.02500000037252903,
+      "rewards/chosen": 0.00024849834153428674,
+      "rewards/margins": 9.472499368712306e-05,
+      "rewards/rejected": 0.0001537733623990789,
+      "step": 150
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.835583164544139e-06,
+      "logits/chosen": -1.0789475440979004,
+      "logits/rejected": -1.0655437707901,
+      "logps/chosen": -14.895776748657227,
+      "logps/rejected": -14.265124320983887,
+      "loss": 0.1408,
+      "rewards/accuracies": 0.02500000037252903,
+      "rewards/chosen": 0.00023004882677923888,
+      "rewards/margins": -1.2942269677296281e-05,
+      "rewards/rejected": 0.0002429911110084504,
+      "step": 160
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.556095160739513e-06,
+      "logits/chosen": -1.059157371520996,
+      "logits/rejected": -0.9842355847358704,
+      "logps/chosen": -23.611387252807617,
+      "logps/rejected": -22.636394500732422,
+      "loss": 0.1427,
+      "rewards/accuracies": 0.03125,
+      "rewards/chosen": 0.00042936400859616697,
+      "rewards/margins": 1.9614741177065298e-05,
+      "rewards/rejected": 0.0004097492783330381,
+      "step": 170
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2759017277414165e-06,
+      "logits/chosen": -1.0258867740631104,
+      "logits/rejected": -0.9551021456718445,
+      "logps/chosen": -26.574390411376953,
+      "logps/rejected": -17.66288948059082,
+      "loss": 0.1366,
+      "rewards/accuracies": 0.02500000037252903,
+      "rewards/chosen": 0.0002309099363628775,
+      "rewards/margins": -0.00013474392471835017,
+      "rewards/rejected": 0.00036565386108122766,
+      "step": 180
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9985264605418185e-06,
+      "logits/chosen": -1.0549209117889404,
+      "logits/rejected": -1.0101747512817383,
+      "logps/chosen": -13.79809284210205,
+      "logps/rejected": -12.155397415161133,
+      "loss": 0.1457,
+      "rewards/accuracies": 0.012500000186264515,
+      "rewards/chosen": 0.00018917236593551934,
+      "rewards/margins": -1.1611557965807151e-05,
+      "rewards/rejected": 0.00020078392117284238,
+      "step": 190
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7274575140626318e-06,
+      "logits/chosen": -1.0925042629241943,
+      "logits/rejected": -1.0523203611373901,
+      "logps/chosen": -22.70647621154785,
+      "logps/rejected": -21.175107955932617,
+      "loss": 0.1673,
+      "rewards/accuracies": 0.02500000037252903,
+      "rewards/chosen": 0.00025192610337398946,
+      "rewards/margins": 6.463247700594366e-06,
+      "rewards/rejected": 0.0002454628120176494,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": -0.8413296937942505,
+      "eval_logits/rejected": -0.7451605796813965,
+      "eval_logps/chosen": -394.95245361328125,
+      "eval_logps/rejected": -365.8174743652344,
+      "eval_loss": 0.05375674366950989,
+      "eval_rewards/accuracies": 0.4465000033378601,
+      "eval_rewards/chosen": 0.0013527950504794717,
+      "eval_rewards/margins": -6.51527734589763e-05,
+      "eval_rewards/rejected": 0.0014179478166624904,
+      "eval_runtime": 545.9031,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.466103737583699e-06,
+      "logits/chosen": -1.0759646892547607,
+      "logits/rejected": -1.0574228763580322,
+      "logps/chosen": -21.28565216064453,
+      "logps/rejected": -18.393810272216797,
+      "loss": 0.1471,
+      "rewards/accuracies": 0.01875000074505806,
+      "rewards/chosen": 0.0002163530298275873,
+      "rewards/margins": -0.00010527259291848168,
+      "rewards/rejected": 0.0003216256445739418,
+      "step": 210
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.217751806485235e-06,
+      "logits/chosen": -1.036292314529419,
+      "logits/rejected": -0.9575215578079224,
+      "logps/chosen": -19.784107208251953,
+      "logps/rejected": -16.992115020751953,
+      "loss": 0.1377,
+      "rewards/accuracies": 0.03125,
+      "rewards/chosen": 0.00023899655207060277,
+      "rewards/margins": 5.950551712885499e-05,
+      "rewards/rejected": 0.000179491049493663,
+      "step": 220
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.855248903979505e-07,
+      "logits/chosen": -1.042037844657898,
+      "logits/rejected": -1.0194261074066162,
+      "logps/chosen": -29.382129669189453,
+      "logps/rejected": -24.937179565429688,
+      "loss": 0.1457,
+      "rewards/accuracies": 0.05000000074505806,
+      "rewards/chosen": 0.0005346836405806243,
+      "rewards/margins": 4.8290185077348724e-05,
+      "rewards/rejected": 0.0004863934591412544,
+      "step": 230
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.723433775328385e-07,
+      "logits/chosen": -1.0891209840774536,
+      "logits/rejected": -1.0485570430755615,
+      "logps/chosen": -24.730152130126953,
+      "logps/rejected": -23.654470443725586,
+      "loss": 0.157,
+      "rewards/accuracies": 0.03125,
+      "rewards/chosen": 0.00022753537632524967,
+      "rewards/margins": -0.00011534886289155111,
+      "rewards/rejected": 0.0003428841882850975,
+      "step": 240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.808881491049723e-07,
+      "logits/chosen": -1.1149171590805054,
+      "logits/rejected": -1.0429071187973022,
+      "logps/chosen": -38.763633728027344,
+      "logps/rejected": -27.425064086914062,
+      "loss": 0.1685,
+      "rewards/accuracies": 0.05000000074505806,
+      "rewards/chosen": 0.0005215436685830355,
+      "rewards/margins": 2.6613433874445036e-05,
+      "rewards/rejected": 0.0004949303111061454,
+      "step": 250
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1356686569674344e-07,
+      "logits/chosen": -1.0859912633895874,
+      "logits/rejected": -1.0464859008789062,
+      "logps/chosen": -27.60003662109375,
+      "logps/rejected": -23.53853988647461,
+      "loss": 0.1439,
+      "rewards/accuracies": 0.03125,
+      "rewards/chosen": 0.000496146094519645,
+      "rewards/margins": 2.9071514290990308e-05,
+      "rewards/rejected": 0.00046707456931471825,
+      "step": 260
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7248368952908055e-07,
+      "logits/chosen": -1.0678870677947998,
+      "logits/rejected": -1.0435296297073364,
+      "logps/chosen": -32.16819763183594,
+      "logps/rejected": -28.180551528930664,
+      "loss": 0.1724,
+      "rewards/accuracies": 0.03750000149011612,
+      "rewards/chosen": 0.0004276956315152347,
+      "rewards/margins": 2.2596535927732475e-05,
+      "rewards/rejected": 0.00040509915561415255,
+      "step": 270
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.59412823400657e-07,
+      "logits/chosen": -1.0548503398895264,
+      "logits/rejected": -1.0008891820907593,
+      "logps/chosen": -25.87237548828125,
+      "logps/rejected": -24.495941162109375,
+      "loss": 0.1347,
+      "rewards/accuracies": 0.03750000149011612,
+      "rewards/chosen": 0.0003715341736096889,
+      "rewards/margins": -5.351124127628282e-05,
+      "rewards/rejected": 0.00042504543671384454,
+      "step": 280
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.577619905828281e-08,
+      "logits/chosen": -1.0749586820602417,
+      "logits/rejected": -1.0013477802276611,
+      "logps/chosen": -24.45828628540039,
+      "logps/rejected": -17.479082107543945,
+      "loss": 0.1346,
+      "rewards/accuracies": 0.03125,
+      "rewards/chosen": 0.0003076163411606103,
+      "rewards/margins": 9.37794684432447e-05,
+      "rewards/rejected": 0.00021383687271736562,
+      "step": 290
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.262559558016325e-08,
+      "logits/chosen": -1.0697089433670044,
+      "logits/rejected": -1.0091029405593872,
+      "logps/chosen": -17.250530242919922,
+      "logps/rejected": -16.223955154418945,
+      "loss": 0.1395,
+      "rewards/accuracies": 0.01875000074505806,
+      "rewards/chosen": 0.00032176534296013415,
+      "rewards/margins": -2.4562166800023988e-05,
+      "rewards/rejected": 0.00034632752067409456,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/chosen": -0.8510691523551941,
+      "eval_logits/rejected": -0.7548762559890747,
+      "eval_logps/chosen": -394.70794677734375,
+      "eval_logps/rejected": -365.4079284667969,
+      "eval_loss": 0.053816698491573334,
+      "eval_rewards/accuracies": 0.4410000145435333,
+      "eval_rewards/chosen": 0.0015973460394889116,
+      "eval_rewards/margins": -0.0002301189670106396,
+      "eval_rewards/rejected": 0.0018274650210514665,
+      "eval_runtime": 545.8177,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 300
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.294126437336734e-10,
+      "logits/chosen": -1.0546523332595825,
+      "logits/rejected": -0.9914261698722839,
+      "logps/chosen": -20.59957504272461,
+      "logps/rejected": -15.304788589477539,
+      "loss": 0.1463,
+      "rewards/accuracies": 0.02500000037252903,
+      "rewards/chosen": 0.0003310924512334168,
+      "rewards/margins": 6.335419311653823e-05,
+      "rewards/rejected": 0.00026773830177262425,
+      "step": 310
+    },
+    {
+      "epoch": 1.0,
+      "step": 312,
+      "total_flos": 0.0,
+      "train_loss": 0.14698060888510484,
+      "train_runtime": 3810.9697,
+      "train_samples_per_second": 1.312,
+      "train_steps_per_second": 0.082
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 312,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}