Model save

Browse files

Files changed (16) hide show

README.md +72 -0
adapter_config.json +25 -0
adapter_model.safetensors +3 -0
all_results.json +21 -0
eval_results.json +16 -0
runs/Dec04_10-45-13_mll-a40-5/events.out.tfevents.1701708335.mll-a40-5.2437352.0 +3 -0
runs/Dec04_10-47-50_mll-a40-5/events.out.tfevents.1701708489.mll-a40-5.2439496.0 +3 -0
runs/Dec04_21-14-17_mll-a40-5/events.out.tfevents.1701746073.mll-a40-5.2649641.0 +3 -0
runs/Dec04_21-14-17_mll-a40-5/events.out.tfevents.1701795732.mll-a40-5.2649641.1 +3 -0
special_tokens_map.json +24 -0
tokenizer.json +0 -0
tokenizer.model +3 -0
tokenizer_config.json +42 -0
train_results.json +8 -0
trainer_state.json +2792 -0
training_args.bin +3 -0

README.md ADDED Viewed

	@@ -0,0 +1,72 @@

+---
+base_model: allenai/tulu-2-7b
+tags:
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-lora
+  results: []
+---
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+# zephyr-7b-dpo-lora
+This model is a fine-tuned version of [allenai/tulu-2-7b](https://huggingface.co/allenai/tulu-2-7b) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6551
+- Rewards/chosen: -0.0635
+- Rewards/rejected: -0.1612
+- Rewards/accuracies: 0.6905
+- Rewards/margins: 0.0978
+- Logps/rejected: -241.7965
+- Logps/chosen: -271.5237
+- Logits/rejected: -1.4412
+- Logits/chosen: -1.2746
+## Model description
+More information needed
+## Intended uses & limitations
+More information needed
+## Training and evaluation data
+More information needed
+## Training procedure
+### Training hyperparameters
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 6
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 96
+- total_eval_batch_size: 24
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 3
+### Training results
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6812        | 1.0   | 645  | 0.6832          | -0.0153        | -0.0425          | 0.6190             | 0.0271          | -240.6086      | -271.0424    | -1.4556         | -1.2853       |
+| 0.667         | 2.0   | 1291 | 0.6636          | -0.0473        | -0.1169          | 0.6518             | 0.0695          | -241.3527      | -271.3625    | -1.4457         | -1.2786       |
+| 0.6518        | 3.0   | 1935 | 0.6551          | -0.0635        | -0.1612          | 0.6905             | 0.0978          | -241.7965      | -271.5237    | -1.4412         | -1.2746       |
+### Framework versions
+- Transformers 4.35.0
+- Pytorch 2.1.1+cu121
+- Datasets 2.14.6
+- Tokenizers 0.14.1

adapter_config.json ADDED Viewed

	@@ -0,0 +1,25 @@

+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "allenai/tulu-2-7b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "o_proj",
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:7703147c798d9a423ad7d65471d465de5d9d541f377ac003df2689f84a523ea4
+size 268470272

all_results.json ADDED Viewed

	@@ -0,0 +1,21 @@

+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -1.2746007442474365,
+    "eval_logits/rejected": -1.4412107467651367,
+    "eval_logps/chosen": -271.52374267578125,
+    "eval_logps/rejected": -241.7964630126953,
+    "eval_loss": 0.6550792455673218,
+    "eval_rewards/accuracies": 0.6904761791229248,
+    "eval_rewards/chosen": -0.0634559839963913,
+    "eval_rewards/margins": 0.09778770059347153,
+    "eval_rewards/rejected": -0.16124369204044342,
+    "eval_runtime": 341.5018,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 5.856,
+    "eval_steps_per_second": 0.246,
+    "train_loss": 0.6738453804676539,
+    "train_runtime": 49317.7491,
+    "train_samples": 61966,
+    "train_samples_per_second": 3.769,
+    "train_steps_per_second": 0.039
+}

eval_results.json ADDED Viewed

	@@ -0,0 +1,16 @@

+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -1.2746007442474365,
+    "eval_logits/rejected": -1.4412107467651367,
+    "eval_logps/chosen": -271.52374267578125,
+    "eval_logps/rejected": -241.7964630126953,
+    "eval_loss": 0.6550792455673218,
+    "eval_rewards/accuracies": 0.6904761791229248,
+    "eval_rewards/chosen": -0.0634559839963913,
+    "eval_rewards/margins": 0.09778770059347153,
+    "eval_rewards/rejected": -0.16124369204044342,
+    "eval_runtime": 341.5018,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 5.856,
+    "eval_steps_per_second": 0.246
+}

runs/Dec04_10-45-13_mll-a40-5/events.out.tfevents.1701708335.mll-a40-5.2437352.0 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c25c4d6b00e6a849bb3a71dca65a5f710712faf276584c24847961d4c3880440
+size 4361

runs/Dec04_10-47-50_mll-a40-5/events.out.tfevents.1701708489.mll-a40-5.2439496.0 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:013a6dc44536c1a126bea97d67bf5e94b09294118fa49402486a98a16f929bbb
+size 72795

runs/Dec04_21-14-17_mll-a40-5/events.out.tfevents.1701746073.mll-a40-5.2649641.0 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:49279d319e6854f508385b0ee5113138ffbd8f0c69b84677bcec8ef9471bdbb9
+size 129787

runs/Dec04_21-14-17_mll-a40-5/events.out.tfevents.1701795732.mll-a40-5.2649641.1 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a684bed386e9a61a2da4f6f24fe36e6a2984d831f64c595038adce89a78d738e
+size 828

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,24 @@

+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,42 @@

+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json ADDED Viewed

	@@ -0,0 +1,8 @@

+{
+    "epoch": 3.0,
+    "train_loss": 0.6738453804676539,
+    "train_runtime": 49317.7491,
+    "train_samples": 61966,
+    "train_samples_per_second": 3.769,
+    "train_steps_per_second": 0.039
+}

trainer_state.json ADDED Viewed

	@@ -0,0 +1,2792 @@

+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.997676219984508,
+  "eval_steps": 100,
+  "global_step": 1935,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.5773195876288657e-09,
+      "logits/chosen": -1.2259318828582764,
+      "logits/rejected": -1.2518267631530762,
+      "logps/chosen": -300.0604248046875,
+      "logps/rejected": -269.6971130371094,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 2.5773195876288656e-08,
+      "logits/chosen": -1.137500286102295,
+      "logits/rejected": -1.2303526401519775,
+      "logps/chosen": -301.8792724609375,
+      "logps/rejected": -238.36328125,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0015259875217452645,
+      "rewards/margins": -0.003614760935306549,
+      "rewards/rejected": 0.005140749271959066,
+      "step": 10
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 5.154639175257731e-08,
+      "logits/chosen": -1.1808393001556396,
+      "logits/rejected": -1.2743797302246094,
+      "logps/chosen": -314.679931640625,
+      "logps/rejected": -244.7173614501953,
+      "loss": 0.6962,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.004601674620062113,
+      "rewards/margins": -0.0029529244638979435,
+      "rewards/rejected": 0.007554598152637482,
+      "step": 20
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 7.731958762886598e-08,
+      "logits/chosen": -1.069253921508789,
+      "logits/rejected": -1.2187018394470215,
+      "logps/chosen": -323.1502380371094,
+      "logps/rejected": -268.97393798828125,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.005449424032121897,
+      "rewards/margins": -0.0008543418953195214,
+      "rewards/rejected": -0.004595082253217697,
+      "step": 30
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.0309278350515462e-07,
+      "logits/chosen": -1.1713712215423584,
+      "logits/rejected": -1.2351148128509521,
+      "logps/chosen": -255.2301025390625,
+      "logps/rejected": -231.7521514892578,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.004361768253147602,
+      "rewards/margins": 0.002766171470284462,
+      "rewards/rejected": -0.007127939257770777,
+      "step": 40
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.2886597938144328e-07,
+      "logits/chosen": -1.14518404006958,
+      "logits/rejected": -1.2944872379302979,
+      "logps/chosen": -297.48272705078125,
+      "logps/rejected": -256.091796875,
+      "loss": 0.6953,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.005987298674881458,
+      "rewards/margins": 0.006507495883852243,
+      "rewards/rejected": -0.0005201972089707851,
+      "step": 50
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.5463917525773197e-07,
+      "logits/chosen": -1.1941990852355957,
+      "logits/rejected": -1.266606092453003,
+      "logps/chosen": -308.13873291015625,
+      "logps/rejected": -251.240966796875,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.0025153695605695248,
+      "rewards/margins": 0.005558790173381567,
+      "rewards/rejected": -0.008074158802628517,
+      "step": 60
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.804123711340206e-07,
+      "logits/chosen": -1.214543104171753,
+      "logits/rejected": -1.2368158102035522,
+      "logps/chosen": -308.1925048828125,
+      "logps/rejected": -258.2166748046875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.001422025030478835,
+      "rewards/margins": 0.00037188082933425903,
+      "rewards/rejected": -0.0017939055105671287,
+      "step": 70
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 2.0618556701030925e-07,
+      "logits/chosen": -1.1733652353286743,
+      "logits/rejected": -1.2068002223968506,
+      "logps/chosen": -339.65484619140625,
+      "logps/rejected": -251.48757934570312,
+      "loss": 0.6954,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.000313587486743927,
+      "rewards/margins": -0.0003553667920641601,
+      "rewards/rejected": 0.0006689537549391389,
+      "step": 80
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 2.3195876288659794e-07,
+      "logits/chosen": -1.1699542999267578,
+      "logits/rejected": -1.2276877164840698,
+      "logps/chosen": -309.55816650390625,
+      "logps/rejected": -253.14212036132812,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.008602892979979515,
+      "rewards/margins": -0.002404768718406558,
+      "rewards/rejected": -0.006198124028742313,
+      "step": 90
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 2.5773195876288655e-07,
+      "logits/chosen": -1.2193819284439087,
+      "logits/rejected": -1.2733923196792603,
+      "logps/chosen": -284.50494384765625,
+      "logps/rejected": -257.36614990234375,
+      "loss": 0.6944,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.008577315136790276,
+      "rewards/margins": -0.007729054894298315,
+      "rewards/rejected": -0.0008482601260766387,
+      "step": 100
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 2.835051546391752e-07,
+      "logits/chosen": -1.1101040840148926,
+      "logits/rejected": -1.2103769779205322,
+      "logps/chosen": -272.78472900390625,
+      "logps/rejected": -236.69888305664062,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.008216220885515213,
+      "rewards/margins": -0.006476385984569788,
+      "rewards/rejected": -0.001739834202453494,
+      "step": 110
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 3.0927835051546394e-07,
+      "logits/chosen": -1.1474252939224243,
+      "logits/rejected": -1.2477691173553467,
+      "logps/chosen": -313.0333557128906,
+      "logps/rejected": -276.30560302734375,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.0040068114176392555,
+      "rewards/margins": -0.0006697230273857713,
+      "rewards/rejected": -0.0033370896708220243,
+      "step": 120
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 3.3505154639175255e-07,
+      "logits/chosen": -1.1452791690826416,
+      "logits/rejected": -1.2140785455703735,
+      "logps/chosen": -306.3499755859375,
+      "logps/rejected": -241.6109161376953,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.002921496983617544,
+      "rewards/margins": -0.0032920476514846087,
+      "rewards/rejected": 0.0003705514536704868,
+      "step": 130
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 3.608247422680412e-07,
+      "logits/chosen": -1.2596957683563232,
+      "logits/rejected": -1.3245229721069336,
+      "logps/chosen": -281.8450012207031,
+      "logps/rejected": -239.2882537841797,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.006516546942293644,
+      "rewards/margins": 0.0012826811289414763,
+      "rewards/rejected": -0.007799227721989155,
+      "step": 140
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 3.865979381443299e-07,
+      "logits/chosen": -1.1537089347839355,
+      "logits/rejected": -1.1991795301437378,
+      "logps/chosen": -282.4929504394531,
+      "logps/rejected": -248.6052703857422,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.002510285470634699,
+      "rewards/margins": 0.007546475622802973,
+      "rewards/rejected": -0.005036188755184412,
+      "step": 150
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.123711340206185e-07,
+      "logits/chosen": -1.0950748920440674,
+      "logits/rejected": -1.2696372270584106,
+      "logps/chosen": -284.0737609863281,
+      "logps/rejected": -226.47195434570312,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.0014800997450947762,
+      "rewards/margins": 0.0013229111209511757,
+      "rewards/rejected": -0.002803010866045952,
+      "step": 160
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.381443298969072e-07,
+      "logits/chosen": -1.231684923171997,
+      "logits/rejected": -1.1929103136062622,
+      "logps/chosen": -278.35870361328125,
+      "logps/rejected": -222.3286590576172,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.004064156208187342,
+      "rewards/margins": -0.0029725716449320316,
+      "rewards/rejected": -0.0010915847960859537,
+      "step": 170
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.639175257731959e-07,
+      "logits/chosen": -1.1976374387741089,
+      "logits/rejected": -1.228857398033142,
+      "logps/chosen": -254.65087890625,
+      "logps/rejected": -227.2077178955078,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.004570499062538147,
+      "rewards/margins": -0.005624748300760984,
+      "rewards/rejected": 0.0010542500531300902,
+      "step": 180
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.896907216494845e-07,
+      "logits/chosen": -1.1410865783691406,
+      "logits/rejected": -1.2138147354125977,
+      "logps/chosen": -312.42230224609375,
+      "logps/rejected": -247.4111785888672,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.0012876918772235513,
+      "rewards/margins": 0.005938920192420483,
+      "rewards/rejected": -0.0046512288972735405,
+      "step": 190
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.982768523836875e-07,
+      "logits/chosen": -1.1175594329833984,
+      "logits/rejected": -1.2174949645996094,
+      "logps/chosen": -300.48736572265625,
+      "logps/rejected": -253.6021728515625,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.010855994187295437,
+      "rewards/margins": -0.0076787760481238365,
+      "rewards/rejected": -0.0031772174406796694,
+      "step": 200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.954049396898335e-07,
+      "logits/chosen": -1.1029257774353027,
+      "logits/rejected": -1.266903042793274,
+      "logps/chosen": -292.01513671875,
+      "logps/rejected": -248.255615234375,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.003372871782630682,
+      "rewards/margins": 0.01588493585586548,
+      "rewards/rejected": -0.012512065470218658,
+      "step": 210
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.925330269959793e-07,
+      "logits/chosen": -1.1873070001602173,
+      "logits/rejected": -1.2105329036712646,
+      "logps/chosen": -294.33184814453125,
+      "logps/rejected": -248.6623077392578,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0029001368675380945,
+      "rewards/margins": 0.0060717323794960976,
+      "rewards/rejected": -0.008971868082880974,
+      "step": 220
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.896611143021252e-07,
+      "logits/chosen": -1.126260757446289,
+      "logits/rejected": -1.1767059564590454,
+      "logps/chosen": -283.59649658203125,
+      "logps/rejected": -265.82476806640625,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.0003154706209897995,
+      "rewards/margins": 0.011949801817536354,
+      "rewards/rejected": -0.01163433212786913,
+      "step": 230
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.867892016082711e-07,
+      "logits/chosen": -1.0998083353042603,
+      "logits/rejected": -1.197954773902893,
+      "logps/chosen": -297.81396484375,
+      "logps/rejected": -251.5466766357422,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.001978257205337286,
+      "rewards/margins": 0.006527910940349102,
+      "rewards/rejected": -0.008506168611347675,
+      "step": 240
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 4.839172889144169e-07,
+      "logits/chosen": -1.1897470951080322,
+      "logits/rejected": -1.2483431100845337,
+      "logps/chosen": -278.9944763183594,
+      "logps/rejected": -237.2654571533203,
+      "loss": 0.693,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.013788111507892609,
+      "rewards/margins": -0.006659683771431446,
+      "rewards/rejected": -0.007128427270799875,
+      "step": 250
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 4.810453762205628e-07,
+      "logits/chosen": -1.1891684532165527,
+      "logits/rejected": -1.2836068868637085,
+      "logps/chosen": -286.28570556640625,
+      "logps/rejected": -255.8869171142578,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0036178447771817446,
+      "rewards/margins": 0.003644808428362012,
+      "rewards/rejected": -0.0072626532055437565,
+      "step": 260
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 4.781734635267088e-07,
+      "logits/chosen": -1.1823965311050415,
+      "logits/rejected": -1.2820310592651367,
+      "logps/chosen": -270.109375,
+      "logps/rejected": -234.67568969726562,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.007160546723753214,
+      "rewards/margins": -0.003257888136431575,
+      "rewards/rejected": -0.0039026569575071335,
+      "step": 270
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 4.7530155083285466e-07,
+      "logits/chosen": -1.1655725240707397,
+      "logits/rejected": -1.2631051540374756,
+      "logps/chosen": -290.2110290527344,
+      "logps/rejected": -247.9181671142578,
+      "loss": 0.6945,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.005572388414293528,
+      "rewards/margins": 0.005412190221250057,
+      "rewards/rejected": -0.010984579101204872,
+      "step": 280
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.7242963813900055e-07,
+      "logits/chosen": -1.1850521564483643,
+      "logits/rejected": -1.2781732082366943,
+      "logps/chosen": -305.34063720703125,
+      "logps/rejected": -231.7410430908203,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.008810663595795631,
+      "rewards/margins": 0.002493415493518114,
+      "rewards/rejected": -0.011304078623652458,
+      "step": 290
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 4.6955772544514645e-07,
+      "logits/chosen": -1.2146244049072266,
+      "logits/rejected": -1.3335716724395752,
+      "logps/chosen": -260.8929748535156,
+      "logps/rejected": -225.0554656982422,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.0020951449405401945,
+      "rewards/margins": 0.0035616860259324312,
+      "rewards/rejected": -0.005656830966472626,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 4.6668581275129234e-07,
+      "logits/chosen": -1.174975872039795,
+      "logits/rejected": -1.2588573694229126,
+      "logps/chosen": -304.65008544921875,
+      "logps/rejected": -261.6839599609375,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.012646270915865898,
+      "rewards/margins": -0.005786383058875799,
+      "rewards/rejected": -0.006859886460006237,
+      "step": 310
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 4.6381390005743824e-07,
+      "logits/chosen": -1.1501063108444214,
+      "logits/rejected": -1.244741678237915,
+      "logps/chosen": -287.64208984375,
+      "logps/rejected": -252.3353729248047,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.006176070310175419,
+      "rewards/margins": 0.004587090574204922,
+      "rewards/rejected": -0.01076316274702549,
+      "step": 320
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.6094198736358413e-07,
+      "logits/chosen": -1.2028628587722778,
+      "logits/rejected": -1.2731959819793701,
+      "logps/chosen": -297.4624938964844,
+      "logps/rejected": -271.55548095703125,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.006290009710937738,
+      "rewards/margins": 0.0007669615442864597,
+      "rewards/rejected": -0.007056971546262503,
+      "step": 330
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.5807007466973003e-07,
+      "logits/chosen": -1.1701081991195679,
+      "logits/rejected": -1.245665431022644,
+      "logps/chosen": -285.584716796875,
+      "logps/rejected": -249.4613494873047,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.011785658076405525,
+      "rewards/margins": 0.0033443807624280453,
+      "rewards/rejected": -0.015130040235817432,
+      "step": 340
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.551981619758759e-07,
+      "logits/chosen": -1.0303632020950317,
+      "logits/rejected": -1.1990408897399902,
+      "logps/chosen": -314.17156982421875,
+      "logps/rejected": -267.7991638183594,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0035480200313031673,
+      "rewards/margins": 0.006092972122132778,
+      "rewards/rejected": -0.009640992619097233,
+      "step": 350
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.5232624928202176e-07,
+      "logits/chosen": -1.1599308252334595,
+      "logits/rejected": -1.2161757946014404,
+      "logps/chosen": -302.0650634765625,
+      "logps/rejected": -254.35116577148438,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.006021351087838411,
+      "rewards/margins": 0.019538475200533867,
+      "rewards/rejected": -0.02555982768535614,
+      "step": 360
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.494543365881677e-07,
+      "logits/chosen": -1.1341716051101685,
+      "logits/rejected": -1.2106345891952515,
+      "logps/chosen": -268.8562927246094,
+      "logps/rejected": -254.6302490234375,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.01141696609556675,
+      "rewards/margins": 0.0022176806814968586,
+      "rewards/rejected": -0.013634647242724895,
+      "step": 370
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.465824238943136e-07,
+      "logits/chosen": -1.1235488653182983,
+      "logits/rejected": -1.2134369611740112,
+      "logps/chosen": -294.3124084472656,
+      "logps/rejected": -249.931884765625,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0007959249196574092,
+      "rewards/margins": 0.014344396069645882,
+      "rewards/rejected": -0.013548469170928001,
+      "step": 380
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.437105112004595e-07,
+      "logits/chosen": -1.174169659614563,
+      "logits/rejected": -1.2081773281097412,
+      "logps/chosen": -268.9616394042969,
+      "logps/rejected": -272.663818359375,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -8.791415166342631e-05,
+      "rewards/margins": 0.008916514925658703,
+      "rewards/rejected": -0.009004429914057255,
+      "step": 390
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 4.408385985066054e-07,
+      "logits/chosen": -1.2326834201812744,
+      "logits/rejected": -1.2733819484710693,
+      "logps/chosen": -279.9195251464844,
+      "logps/rejected": -262.4093933105469,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.00013022082566749305,
+      "rewards/margins": 0.019076677039265633,
+      "rewards/rejected": -0.019206896424293518,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 4.3796668581275124e-07,
+      "logits/chosen": -1.0889310836791992,
+      "logits/rejected": -1.1624057292938232,
+      "logps/chosen": -342.1612243652344,
+      "logps/rejected": -292.48516845703125,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.00807907897979021,
+      "rewards/margins": 0.017683448269963264,
+      "rewards/rejected": -0.02576252445578575,
+      "step": 410
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 4.3509477311889713e-07,
+      "logits/chosen": -1.1696580648422241,
+      "logits/rejected": -1.2402842044830322,
+      "logps/chosen": -318.536376953125,
+      "logps/rejected": -249.26010131835938,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.011863854713737965,
+      "rewards/margins": 0.004245345946401358,
+      "rewards/rejected": -0.016109202057123184,
+      "step": 420
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 4.322228604250431e-07,
+      "logits/chosen": -1.1121430397033691,
+      "logits/rejected": -1.1772434711456299,
+      "logps/chosen": -316.11346435546875,
+      "logps/rejected": -265.7353210449219,
+      "loss": 0.688,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.00796548929065466,
+      "rewards/margins": 0.01683664321899414,
+      "rewards/rejected": -0.024802135303616524,
+      "step": 430
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.2935094773118897e-07,
+      "logits/chosen": -1.0895384550094604,
+      "logits/rejected": -1.235265851020813,
+      "logps/chosen": -299.3586120605469,
+      "logps/rejected": -271.97650146484375,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.013709778897464275,
+      "rewards/margins": 0.007505470421165228,
+      "rewards/rejected": -0.021215252578258514,
+      "step": 440
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.2647903503733487e-07,
+      "logits/chosen": -1.1224366426467896,
+      "logits/rejected": -1.21951425075531,
+      "logps/chosen": -290.92523193359375,
+      "logps/rejected": -250.52157592773438,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.009392457082867622,
+      "rewards/margins": 0.012002045288681984,
+      "rewards/rejected": -0.021394502371549606,
+      "step": 450
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.236071223434807e-07,
+      "logits/chosen": -1.1139185428619385,
+      "logits/rejected": -1.230420708656311,
+      "logps/chosen": -293.9082336425781,
+      "logps/rejected": -263.75726318359375,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.015224998816847801,
+      "rewards/margins": 0.006627705879509449,
+      "rewards/rejected": -0.021852705627679825,
+      "step": 460
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 4.207352096496266e-07,
+      "logits/chosen": -1.2127584218978882,
+      "logits/rejected": -1.2580344676971436,
+      "logps/chosen": -308.8389892578125,
+      "logps/rejected": -267.56787109375,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.010395748540759087,
+      "rewards/margins": 0.010451807640492916,
+      "rewards/rejected": -0.020847555249929428,
+      "step": 470
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 4.1786329695577255e-07,
+      "logits/chosen": -1.151767373085022,
+      "logits/rejected": -1.2307831048965454,
+      "logps/chosen": -312.295166015625,
+      "logps/rejected": -272.08050537109375,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.015018184669315815,
+      "rewards/margins": 0.0127108721062541,
+      "rewards/rejected": -0.027729053050279617,
+      "step": 480
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4.1499138426191845e-07,
+      "logits/chosen": -1.128575086593628,
+      "logits/rejected": -1.1787781715393066,
+      "logps/chosen": -291.2005310058594,
+      "logps/rejected": -258.6954040527344,
+      "loss": 0.688,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.006343941204249859,
+      "rewards/margins": 0.01303582638502121,
+      "rewards/rejected": -0.019379768520593643,
+      "step": 490
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.121194715680643e-07,
+      "logits/chosen": -1.1914628744125366,
+      "logits/rejected": -1.2668449878692627,
+      "logps/chosen": -261.34039306640625,
+      "logps/rejected": -213.05917358398438,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.013653025031089783,
+      "rewards/margins": 0.012117515318095684,
+      "rewards/rejected": -0.02577054128050804,
+      "step": 500
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 4.092475588742102e-07,
+      "logits/chosen": -1.1350027322769165,
+      "logits/rejected": -1.2238458395004272,
+      "logps/chosen": -315.3155517578125,
+      "logps/rejected": -300.2586975097656,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.009474663995206356,
+      "rewards/margins": 0.014838702976703644,
+      "rewards/rejected": -0.024313367903232574,
+      "step": 510
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 4.063756461803561e-07,
+      "logits/chosen": -1.137836217880249,
+      "logits/rejected": -1.269636631011963,
+      "logps/chosen": -295.51776123046875,
+      "logps/rejected": -245.64749145507812,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.01145818829536438,
+      "rewards/margins": 0.014280671253800392,
+      "rewards/rejected": -0.025738855823874474,
+      "step": 520
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.03503733486502e-07,
+      "logits/chosen": -1.0953731536865234,
+      "logits/rejected": -1.1840622425079346,
+      "logps/chosen": -302.6564025878906,
+      "logps/rejected": -265.1612854003906,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.005975248292088509,
+      "rewards/margins": 0.02241487428545952,
+      "rewards/rejected": -0.028390124440193176,
+      "step": 530
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4.006318207926479e-07,
+      "logits/chosen": -1.1384605169296265,
+      "logits/rejected": -1.2369248867034912,
+      "logps/chosen": -302.2364196777344,
+      "logps/rejected": -282.00164794921875,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.021240079775452614,
+      "rewards/margins": 0.015024731867015362,
+      "rewards/rejected": -0.0362648107111454,
+      "step": 540
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.9775990809879376e-07,
+      "logits/chosen": -1.1440156698226929,
+      "logits/rejected": -1.2083277702331543,
+      "logps/chosen": -297.77606201171875,
+      "logps/rejected": -257.05352783203125,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.00584394671022892,
+      "rewards/margins": 0.024538297206163406,
+      "rewards/rejected": -0.030382240191102028,
+      "step": 550
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 3.9488799540493966e-07,
+      "logits/chosen": -1.1610389947891235,
+      "logits/rejected": -1.2373404502868652,
+      "logps/chosen": -299.90802001953125,
+      "logps/rejected": -255.9143524169922,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.008234420791268349,
+      "rewards/margins": 0.02389439567923546,
+      "rewards/rejected": -0.03212881460785866,
+      "step": 560
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 3.9201608271108555e-07,
+      "logits/chosen": -1.1905468702316284,
+      "logits/rejected": -1.2244651317596436,
+      "logps/chosen": -341.0932922363281,
+      "logps/rejected": -253.48916625976562,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.007996906526386738,
+      "rewards/margins": 0.019794629886746407,
+      "rewards/rejected": -0.02779153548181057,
+      "step": 570
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 3.8914417001723144e-07,
+      "logits/chosen": -1.1463348865509033,
+      "logits/rejected": -1.2082992792129517,
+      "logps/chosen": -278.6533203125,
+      "logps/rejected": -247.3466796875,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.013167768716812134,
+      "rewards/margins": 0.013775348663330078,
+      "rewards/rejected": -0.026943117380142212,
+      "step": 580
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.862722573233774e-07,
+      "logits/chosen": -1.1172596216201782,
+      "logits/rejected": -1.2871202230453491,
+      "logps/chosen": -268.90435791015625,
+      "logps/rejected": -233.5662384033203,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.017810124903917313,
+      "rewards/margins": 0.01590743288397789,
+      "rewards/rejected": -0.0337175577878952,
+      "step": 590
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.8340034462952323e-07,
+      "logits/chosen": -1.1374852657318115,
+      "logits/rejected": -1.2383670806884766,
+      "logps/chosen": -309.21331787109375,
+      "logps/rejected": -242.3319091796875,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.02574210986495018,
+      "rewards/margins": 0.018136167898774147,
+      "rewards/rejected": -0.043878279626369476,
+      "step": 600
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.8052843193566913e-07,
+      "logits/chosen": -1.1285089254379272,
+      "logits/rejected": -1.1931793689727783,
+      "logps/chosen": -278.48101806640625,
+      "logps/rejected": -242.2903289794922,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.017196612432599068,
+      "rewards/margins": 0.032739974558353424,
+      "rewards/rejected": -0.04993658885359764,
+      "step": 610
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.77656519241815e-07,
+      "logits/chosen": -1.1930526494979858,
+      "logits/rejected": -1.3126742839813232,
+      "logps/chosen": -286.9619140625,
+      "logps/rejected": -246.289794921875,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.014147510752081871,
+      "rewards/margins": 0.016736771911382675,
+      "rewards/rejected": -0.030884284526109695,
+      "step": 620
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.747846065479609e-07,
+      "logits/chosen": -1.2015870809555054,
+      "logits/rejected": -1.2615153789520264,
+      "logps/chosen": -286.06475830078125,
+      "logps/rejected": -245.6126708984375,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.00835242960602045,
+      "rewards/margins": 0.02643100544810295,
+      "rewards/rejected": -0.03478343412280083,
+      "step": 630
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.7191269385410686e-07,
+      "logits/chosen": -1.092500925064087,
+      "logits/rejected": -1.2248862981796265,
+      "logps/chosen": -280.76324462890625,
+      "logps/rejected": -244.59375,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.004551100078970194,
+      "rewards/margins": 0.026827871799468994,
+      "rewards/rejected": -0.03137897327542305,
+      "step": 640
+    },
+    {
+      "epoch": 1.0,
+      "eval_logits/chosen": -1.2853448390960693,
+      "eval_logits/rejected": -1.455635905265808,
+      "eval_logps/chosen": -271.04241943359375,
+      "eval_logps/rejected": -240.60862731933594,
+      "eval_loss": 0.6831629276275635,
+      "eval_rewards/accuracies": 0.6190476417541504,
+      "eval_rewards/chosen": -0.015324260108172894,
+      "eval_rewards/margins": 0.027137557044625282,
+      "eval_rewards/rejected": -0.0424618162214756,
+      "eval_runtime": 336.2787,
+      "eval_samples_per_second": 5.947,
+      "eval_steps_per_second": 0.25,
+      "step": 645
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 3.690407811602527e-07,
+      "logits/chosen": -1.1837308406829834,
+      "logits/rejected": -1.2207152843475342,
+      "logps/chosen": -269.86431884765625,
+      "logps/rejected": -258.81109619140625,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.02447674050927162,
+      "rewards/margins": 0.018924027681350708,
+      "rewards/rejected": -0.04340077191591263,
+      "step": 650
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 3.661688684663986e-07,
+      "logits/chosen": -1.0795660018920898,
+      "logits/rejected": -1.1991580724716187,
+      "logps/chosen": -280.2117919921875,
+      "logps/rejected": -253.22000122070312,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.018620852380990982,
+      "rewards/margins": 0.016325894743204117,
+      "rewards/rejected": -0.0349467471241951,
+      "step": 660
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 3.632969557725445e-07,
+      "logits/chosen": -1.1959303617477417,
+      "logits/rejected": -1.268479824066162,
+      "logps/chosen": -282.8076477050781,
+      "logps/rejected": -270.8659973144531,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.014843074604868889,
+      "rewards/margins": 0.031116703525185585,
+      "rewards/rejected": -0.045959778130054474,
+      "step": 670
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 3.604250430786904e-07,
+      "logits/chosen": -1.131414532661438,
+      "logits/rejected": -1.2395284175872803,
+      "logps/chosen": -268.8525085449219,
+      "logps/rejected": -249.85302734375,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.019633300602436066,
+      "rewards/margins": 0.028578415513038635,
+      "rewards/rejected": -0.0482117161154747,
+      "step": 680
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 3.575531303848363e-07,
+      "logits/chosen": -1.1759953498840332,
+      "logits/rejected": -1.2496672868728638,
+      "logps/chosen": -283.6187438964844,
+      "logps/rejected": -251.77706909179688,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.02356325089931488,
+      "rewards/margins": 0.026736527681350708,
+      "rewards/rejected": -0.05029977485537529,
+      "step": 690
+    },
+    {
+      "epoch": 1.08,
+      "learning_rate": 3.546812176909822e-07,
+      "logits/chosen": -1.2295715808868408,
+      "logits/rejected": -1.2652504444122314,
+      "logps/chosen": -297.22796630859375,
+      "logps/rejected": -288.7464294433594,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.021731413900852203,
+      "rewards/margins": 0.02142687328159809,
+      "rewards/rejected": -0.043158285319805145,
+      "step": 700
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 3.518093049971281e-07,
+      "logits/chosen": -1.1974341869354248,
+      "logits/rejected": -1.2550041675567627,
+      "logps/chosen": -307.19482421875,
+      "logps/rejected": -276.5005798339844,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.021447179839015007,
+      "rewards/margins": 0.026276415213942528,
+      "rewards/rejected": -0.04772359877824783,
+      "step": 710
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 3.4893739230327397e-07,
+      "logits/chosen": -1.2148594856262207,
+      "logits/rejected": -1.2141330242156982,
+      "logps/chosen": -309.69671630859375,
+      "logps/rejected": -253.8720245361328,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.015615013428032398,
+      "rewards/margins": 0.021955113857984543,
+      "rewards/rejected": -0.037570130079984665,
+      "step": 720
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 3.4606547960941986e-07,
+      "logits/chosen": -1.1154258251190186,
+      "logits/rejected": -1.2231298685073853,
+      "logps/chosen": -280.8895263671875,
+      "logps/rejected": -236.9070281982422,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.018392009660601616,
+      "rewards/margins": 0.02249971777200699,
+      "rewards/rejected": -0.040891729295253754,
+      "step": 730
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 3.431935669155657e-07,
+      "logits/chosen": -1.1089565753936768,
+      "logits/rejected": -1.2940270900726318,
+      "logps/chosen": -317.8031005859375,
+      "logps/rejected": -251.05783081054688,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.02725188434123993,
+      "rewards/margins": 0.014852823689579964,
+      "rewards/rejected": -0.042104706168174744,
+      "step": 740
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 3.4032165422171165e-07,
+      "logits/chosen": -1.1635140180587769,
+      "logits/rejected": -1.239103078842163,
+      "logps/chosen": -273.4166564941406,
+      "logps/rejected": -257.73419189453125,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.019765909761190414,
+      "rewards/margins": 0.0273622814565897,
+      "rewards/rejected": -0.04712819308042526,
+      "step": 750
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 3.3744974152785755e-07,
+      "logits/chosen": -1.1813712120056152,
+      "logits/rejected": -1.2533613443374634,
+      "logps/chosen": -277.0120849609375,
+      "logps/rejected": -244.0693817138672,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.027658691629767418,
+      "rewards/margins": 0.027155902236700058,
+      "rewards/rejected": -0.05481458827853203,
+      "step": 760
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 3.3457782883400344e-07,
+      "logits/chosen": -1.121671199798584,
+      "logits/rejected": -1.147890329360962,
+      "logps/chosen": -311.65179443359375,
+      "logps/rejected": -263.4295349121094,
+      "loss": 0.68,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.02874746546149254,
+      "rewards/margins": 0.03633704036474228,
+      "rewards/rejected": -0.06508450210094452,
+      "step": 770
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 3.3170591614014934e-07,
+      "logits/chosen": -1.1686742305755615,
+      "logits/rejected": -1.274806261062622,
+      "logps/chosen": -330.0002746582031,
+      "logps/rejected": -265.2279357910156,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.012181037105619907,
+      "rewards/margins": 0.05000365525484085,
+      "rewards/rejected": -0.062184691429138184,
+      "step": 780
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 3.288340034462952e-07,
+      "logits/chosen": -1.1289006471633911,
+      "logits/rejected": -1.2150051593780518,
+      "logps/chosen": -328.3635559082031,
+      "logps/rejected": -280.61029052734375,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.018197020515799522,
+      "rewards/margins": 0.041884154081344604,
+      "rewards/rejected": -0.06008117273449898,
+      "step": 790
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 3.259620907524411e-07,
+      "logits/chosen": -1.0758039951324463,
+      "logits/rejected": -1.1864715814590454,
+      "logps/chosen": -317.11444091796875,
+      "logps/rejected": -275.755615234375,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.023402996361255646,
+      "rewards/margins": 0.04022226482629776,
+      "rewards/rejected": -0.0636252611875534,
+      "step": 800
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 3.23090178058587e-07,
+      "logits/chosen": -1.160382628440857,
+      "logits/rejected": -1.2629868984222412,
+      "logps/chosen": -266.081298828125,
+      "logps/rejected": -271.7577209472656,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.032355837523937225,
+      "rewards/margins": 0.018326448276638985,
+      "rewards/rejected": -0.05068228393793106,
+      "step": 810
+    },
+    {
+      "epoch": 1.27,
+      "learning_rate": 3.202182653647329e-07,
+      "logits/chosen": -1.0827566385269165,
+      "logits/rejected": -1.2318055629730225,
+      "logps/chosen": -285.2849426269531,
+      "logps/rejected": -233.0825958251953,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04408087208867073,
+      "rewards/margins": 0.018497029319405556,
+      "rewards/rejected": -0.06257789582014084,
+      "step": 820
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 3.1734635267087876e-07,
+      "logits/chosen": -1.1579523086547852,
+      "logits/rejected": -1.2240597009658813,
+      "logps/chosen": -298.1856384277344,
+      "logps/rejected": -275.46832275390625,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.02176501601934433,
+      "rewards/margins": 0.053393684327602386,
+      "rewards/rejected": -0.07515870034694672,
+      "step": 830
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 3.1447443997702465e-07,
+      "logits/chosen": -1.1462314128875732,
+      "logits/rejected": -1.2096561193466187,
+      "logps/chosen": -304.68487548828125,
+      "logps/rejected": -271.87030029296875,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.029958650469779968,
+      "rewards/margins": 0.03902842849493027,
+      "rewards/rejected": -0.06898707151412964,
+      "step": 840
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 3.116025272831706e-07,
+      "logits/chosen": -1.1924817562103271,
+      "logits/rejected": -1.1946598291397095,
+      "logps/chosen": -276.5715637207031,
+      "logps/rejected": -265.0469970703125,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.03456965461373329,
+      "rewards/margins": 0.012200916185975075,
+      "rewards/rejected": -0.04677056521177292,
+      "step": 850
+    },
+    {
+      "epoch": 1.33,
+      "learning_rate": 3.087306145893165e-07,
+      "logits/chosen": -1.1429754495620728,
+      "logits/rejected": -1.2813076972961426,
+      "logps/chosen": -265.50518798828125,
+      "logps/rejected": -230.6487579345703,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.02857414446771145,
+      "rewards/margins": 0.04311809688806534,
+      "rewards/rejected": -0.07169224321842194,
+      "step": 860
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 3.058587018954624e-07,
+      "logits/chosen": -1.2023561000823975,
+      "logits/rejected": -1.266048550605774,
+      "logps/chosen": -274.9041442871094,
+      "logps/rejected": -257.80706787109375,
+      "loss": 0.678,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.023446839302778244,
+      "rewards/margins": 0.02842586673796177,
+      "rewards/rejected": -0.05187270790338516,
+      "step": 870
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 3.0298678920160823e-07,
+      "logits/chosen": -1.1376311779022217,
+      "logits/rejected": -1.2214622497558594,
+      "logps/chosen": -261.1297302246094,
+      "logps/rejected": -218.02294921875,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.03140454739332199,
+      "rewards/margins": 0.039807092398405075,
+      "rewards/rejected": -0.07121165096759796,
+      "step": 880
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 3.001148765077541e-07,
+      "logits/chosen": -1.1937434673309326,
+      "logits/rejected": -1.2949789762496948,
+      "logps/chosen": -286.6931457519531,
+      "logps/rejected": -233.8115997314453,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.02188955806195736,
+      "rewards/margins": 0.04148869216442108,
+      "rewards/rejected": -0.06337825208902359,
+      "step": 890
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 2.9724296381390007e-07,
+      "logits/chosen": -1.146175742149353,
+      "logits/rejected": -1.2258495092391968,
+      "logps/chosen": -292.33880615234375,
+      "logps/rejected": -259.52349853515625,
+      "loss": 0.6728,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.029968639835715294,
+      "rewards/margins": 0.03762819617986679,
+      "rewards/rejected": -0.06759683787822723,
+      "step": 900
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 2.9437105112004597e-07,
+      "logits/chosen": -1.175890564918518,
+      "logits/rejected": -1.204742193222046,
+      "logps/chosen": -331.7793884277344,
+      "logps/rejected": -295.1964416503906,
+      "loss": 0.6725,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03465835377573967,
+      "rewards/margins": 0.038827430456876755,
+      "rewards/rejected": -0.07348578423261642,
+      "step": 910
+    },
+    {
+      "epoch": 1.43,
+      "learning_rate": 2.9149913842619186e-07,
+      "logits/chosen": -1.1797144412994385,
+      "logits/rejected": -1.273648738861084,
+      "logps/chosen": -303.84906005859375,
+      "logps/rejected": -226.57861328125,
+      "loss": 0.674,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.018653806298971176,
+      "rewards/margins": 0.048885177820920944,
+      "rewards/rejected": -0.06753899157047272,
+      "step": 920
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 2.886272257323377e-07,
+      "logits/chosen": -1.1142618656158447,
+      "logits/rejected": -1.2289409637451172,
+      "logps/chosen": -284.84564208984375,
+      "logps/rejected": -266.482421875,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03997833654284477,
+      "rewards/margins": 0.0469382144510746,
+      "rewards/rejected": -0.08691655099391937,
+      "step": 930
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 2.857553130384836e-07,
+      "logits/chosen": -1.1320688724517822,
+      "logits/rejected": -1.2028567790985107,
+      "logps/chosen": -295.7919921875,
+      "logps/rejected": -275.61029052734375,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.031079787760972977,
+      "rewards/margins": 0.04319722205400467,
+      "rewards/rejected": -0.07427700608968735,
+      "step": 940
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 2.828834003446295e-07,
+      "logits/chosen": -1.2210772037506104,
+      "logits/rejected": -1.2374837398529053,
+      "logps/chosen": -223.2447052001953,
+      "logps/rejected": -223.93301391601562,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.025652194395661354,
+      "rewards/margins": 0.030472612008452415,
+      "rewards/rejected": -0.05612480640411377,
+      "step": 950
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 2.8001148765077544e-07,
+      "logits/chosen": -1.1035312414169312,
+      "logits/rejected": -1.2297589778900146,
+      "logps/chosen": -272.3734436035156,
+      "logps/rejected": -229.17208862304688,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.02614757977426052,
+      "rewards/margins": 0.05098317191004753,
+      "rewards/rejected": -0.0771307498216629,
+      "step": 960
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 2.7713957495692133e-07,
+      "logits/chosen": -1.1308553218841553,
+      "logits/rejected": -1.1985657215118408,
+      "logps/chosen": -279.56158447265625,
+      "logps/rejected": -252.9062957763672,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.025463193655014038,
+      "rewards/margins": 0.03752460703253746,
+      "rewards/rejected": -0.0629878044128418,
+      "step": 970
+    },
+    {
+      "epoch": 1.52,
+      "learning_rate": 2.742676622630672e-07,
+      "logits/chosen": -1.1320959329605103,
+      "logits/rejected": -1.2268012762069702,
+      "logps/chosen": -269.58233642578125,
+      "logps/rejected": -259.0909423828125,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03594038635492325,
+      "rewards/margins": 0.038814641535282135,
+      "rewards/rejected": -0.07475502789020538,
+      "step": 980
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 2.7139574956921307e-07,
+      "logits/chosen": -1.1909582614898682,
+      "logits/rejected": -1.2770507335662842,
+      "logps/chosen": -306.2381286621094,
+      "logps/rejected": -258.2312316894531,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.013350972905755043,
+      "rewards/margins": 0.0710732489824295,
+      "rewards/rejected": -0.08442423492670059,
+      "step": 990
+    },
+    {
+      "epoch": 1.55,
+      "learning_rate": 2.6852383687535896e-07,
+      "logits/chosen": -1.1517307758331299,
+      "logits/rejected": -1.1953328847885132,
+      "logps/chosen": -309.81707763671875,
+      "logps/rejected": -255.09542846679688,
+      "loss": 0.673,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.02377471514046192,
+      "rewards/margins": 0.05378960445523262,
+      "rewards/rejected": -0.0775643140077591,
+      "step": 1000
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 2.656519241815049e-07,
+      "logits/chosen": -1.172380805015564,
+      "logits/rejected": -1.2242120504379272,
+      "logps/chosen": -289.8918762207031,
+      "logps/rejected": -240.6064453125,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.022976800799369812,
+      "rewards/margins": 0.06258256733417511,
+      "rewards/rejected": -0.08555936068296432,
+      "step": 1010
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 2.6278001148765075e-07,
+      "logits/chosen": -1.209279179573059,
+      "logits/rejected": -1.2857582569122314,
+      "logps/chosen": -306.1195373535156,
+      "logps/rejected": -252.3911590576172,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04376746341586113,
+      "rewards/margins": 0.04451705887913704,
+      "rewards/rejected": -0.08828451484441757,
+      "step": 1020
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 2.5990809879379665e-07,
+      "logits/chosen": -1.119346261024475,
+      "logits/rejected": -1.1804701089859009,
+      "logps/chosen": -279.9603576660156,
+      "logps/rejected": -255.67166137695312,
+      "loss": 0.6713,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.032426051795482635,
+      "rewards/margins": 0.04803738743066788,
+      "rewards/rejected": -0.08046343177556992,
+      "step": 1030
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 2.5703618609994254e-07,
+      "logits/chosen": -1.232366681098938,
+      "logits/rejected": -1.2660514116287231,
+      "logps/chosen": -267.54205322265625,
+      "logps/rejected": -217.8297882080078,
+      "loss": 0.6692,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0329388864338398,
+      "rewards/margins": 0.04563206806778908,
+      "rewards/rejected": -0.07857094705104828,
+      "step": 1040
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 2.5416427340608844e-07,
+      "logits/chosen": -1.1432456970214844,
+      "logits/rejected": -1.2851306200027466,
+      "logps/chosen": -325.90716552734375,
+      "logps/rejected": -264.51043701171875,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.035186149179935455,
+      "rewards/margins": 0.059220410883426666,
+      "rewards/rejected": -0.09440656006336212,
+      "step": 1050
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 2.512923607122344e-07,
+      "logits/chosen": -1.1394827365875244,
+      "logits/rejected": -1.208301305770874,
+      "logps/chosen": -295.6949157714844,
+      "logps/rejected": -240.1582794189453,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.025663334876298904,
+      "rewards/margins": 0.05790363624691963,
+      "rewards/rejected": -0.08356697857379913,
+      "step": 1060
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 2.484204480183802e-07,
+      "logits/chosen": -1.1204135417938232,
+      "logits/rejected": -1.295609712600708,
+      "logps/chosen": -274.9275817871094,
+      "logps/rejected": -255.2529754638672,
+      "loss": 0.6687,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.029915103688836098,
+      "rewards/margins": 0.06398225575685501,
+      "rewards/rejected": -0.09389735758304596,
+      "step": 1070
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 2.455485353245261e-07,
+      "logits/chosen": -1.1964619159698486,
+      "logits/rejected": -1.3005297183990479,
+      "logps/chosen": -285.21209716796875,
+      "logps/rejected": -260.6269226074219,
+      "loss": 0.671,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03515591099858284,
+      "rewards/margins": 0.04965870454907417,
+      "rewards/rejected": -0.08481462299823761,
+      "step": 1080
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 2.42676622630672e-07,
+      "logits/chosen": -1.1853787899017334,
+      "logits/rejected": -1.2281911373138428,
+      "logps/chosen": -270.4370422363281,
+      "logps/rejected": -214.59933471679688,
+      "loss": 0.6685,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03246784955263138,
+      "rewards/margins": 0.046676285564899445,
+      "rewards/rejected": -0.07914413511753082,
+      "step": 1090
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 2.398047099368179e-07,
+      "logits/chosen": -1.2114546298980713,
+      "logits/rejected": -1.253631830215454,
+      "logps/chosen": -294.1374206542969,
+      "logps/rejected": -228.50796508789062,
+      "loss": 0.6681,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.027642633765935898,
+      "rewards/margins": 0.08140935003757477,
+      "rewards/rejected": -0.10905198752880096,
+      "step": 1100
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 2.369327972429638e-07,
+      "logits/chosen": -1.1516766548156738,
+      "logits/rejected": -1.2245798110961914,
+      "logps/chosen": -278.65423583984375,
+      "logps/rejected": -243.95529174804688,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.029988175258040428,
+      "rewards/margins": 0.060140348970890045,
+      "rewards/rejected": -0.09012852609157562,
+      "step": 1110
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 2.340608845491097e-07,
+      "logits/chosen": -1.1279699802398682,
+      "logits/rejected": -1.162719488143921,
+      "logps/chosen": -331.1317443847656,
+      "logps/rejected": -264.7960510253906,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.04450936242938042,
+      "rewards/margins": 0.06046723201870918,
+      "rewards/rejected": -0.1049765944480896,
+      "step": 1120
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 2.311889718552556e-07,
+      "logits/chosen": -1.152203917503357,
+      "logits/rejected": -1.2137216329574585,
+      "logps/chosen": -289.91473388671875,
+      "logps/rejected": -252.63174438476562,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0418432392179966,
+      "rewards/margins": 0.05185099318623543,
+      "rewards/rejected": -0.09369422495365143,
+      "step": 1130
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 2.283170591614015e-07,
+      "logits/chosen": -1.2258096933364868,
+      "logits/rejected": -1.271815299987793,
+      "logps/chosen": -271.7290344238281,
+      "logps/rejected": -253.15481567382812,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.028518235310912132,
+      "rewards/margins": 0.08087646216154099,
+      "rewards/rejected": -0.10939470678567886,
+      "step": 1140
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 2.2544514646754736e-07,
+      "logits/chosen": -1.1422202587127686,
+      "logits/rejected": -1.2273261547088623,
+      "logps/chosen": -291.89166259765625,
+      "logps/rejected": -254.87002563476562,
+      "loss": 0.6662,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04026695340871811,
+      "rewards/margins": 0.055162213742733,
+      "rewards/rejected": -0.09542916715145111,
+      "step": 1150
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 2.2257323377369328e-07,
+      "logits/chosen": -1.1333295106887817,
+      "logits/rejected": -1.2422702312469482,
+      "logps/chosen": -289.364990234375,
+      "logps/rejected": -232.06692504882812,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.021913789212703705,
+      "rewards/margins": 0.06650007516145706,
+      "rewards/rejected": -0.08841386437416077,
+      "step": 1160
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 2.1970132107983917e-07,
+      "logits/chosen": -1.2066811323165894,
+      "logits/rejected": -1.2231658697128296,
+      "logps/chosen": -275.4366149902344,
+      "logps/rejected": -224.98867797851562,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.038331288844347,
+      "rewards/margins": 0.05360294505953789,
+      "rewards/rejected": -0.09193422645330429,
+      "step": 1170
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 2.1682940838598507e-07,
+      "logits/chosen": -1.1569344997406006,
+      "logits/rejected": -1.2456449270248413,
+      "logps/chosen": -274.32159423828125,
+      "logps/rejected": -252.0222625732422,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.052247773855924606,
+      "rewards/margins": 0.022707702592015266,
+      "rewards/rejected": -0.07495548576116562,
+      "step": 1180
+    },
+    {
+      "epoch": 1.84,
+      "learning_rate": 2.1395749569213096e-07,
+      "logits/chosen": -1.123389720916748,
+      "logits/rejected": -1.251558542251587,
+      "logps/chosen": -276.007568359375,
+      "logps/rejected": -235.1134033203125,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04328937083482742,
+      "rewards/margins": 0.03924347832798958,
+      "rewards/rejected": -0.0825328528881073,
+      "step": 1190
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 2.1108558299827683e-07,
+      "logits/chosen": -1.143604040145874,
+      "logits/rejected": -1.2688237428665161,
+      "logps/chosen": -278.1545104980469,
+      "logps/rejected": -237.94912719726562,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.042831793427467346,
+      "rewards/margins": 0.06205050274729729,
+      "rewards/rejected": -0.10488228499889374,
+      "step": 1200
+    },
+    {
+      "epoch": 1.87,
+      "learning_rate": 2.0821367030442275e-07,
+      "logits/chosen": -1.1230822801589966,
+      "logits/rejected": -1.2531689405441284,
+      "logps/chosen": -297.42108154296875,
+      "logps/rejected": -255.71334838867188,
+      "loss": 0.67,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.0464252233505249,
+      "rewards/margins": 0.07465530931949615,
+      "rewards/rejected": -0.12108053267002106,
+      "step": 1210
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 2.0534175761056862e-07,
+      "logits/chosen": -1.1438612937927246,
+      "logits/rejected": -1.1976368427276611,
+      "logps/chosen": -325.1570739746094,
+      "logps/rejected": -293.0627136230469,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.029308268800377846,
+      "rewards/margins": 0.08643002808094025,
+      "rewards/rejected": -0.11573830991983414,
+      "step": 1220
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 2.024698449167145e-07,
+      "logits/chosen": -1.1537196636199951,
+      "logits/rejected": -1.2745457887649536,
+      "logps/chosen": -302.37042236328125,
+      "logps/rejected": -251.1383514404297,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.038567863404750824,
+      "rewards/margins": 0.06020348519086838,
+      "rewards/rejected": -0.0987713485956192,
+      "step": 1230
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 1.9959793222286043e-07,
+      "logits/chosen": -1.1626816987991333,
+      "logits/rejected": -1.2281776666641235,
+      "logps/chosen": -264.9757995605469,
+      "logps/rejected": -235.031982421875,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.04553893953561783,
+      "rewards/margins": 0.0635872557759285,
+      "rewards/rejected": -0.10912618786096573,
+      "step": 1240
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 1.967260195290063e-07,
+      "logits/chosen": -1.1658340692520142,
+      "logits/rejected": -1.210275411605835,
+      "logps/chosen": -296.4925231933594,
+      "logps/rejected": -259.5403747558594,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.04651462659239769,
+      "rewards/margins": 0.060394324362277985,
+      "rewards/rejected": -0.10690895467996597,
+      "step": 1250
+    },
+    {
+      "epoch": 1.95,
+      "learning_rate": 1.9385410683515222e-07,
+      "logits/chosen": -1.1199092864990234,
+      "logits/rejected": -1.1857876777648926,
+      "logps/chosen": -309.8304748535156,
+      "logps/rejected": -252.6132354736328,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.02937604859471321,
+      "rewards/margins": 0.06977292895317078,
+      "rewards/rejected": -0.09914897382259369,
+      "step": 1260
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 1.909821941412981e-07,
+      "logits/chosen": -1.098323106765747,
+      "logits/rejected": -1.2124502658843994,
+      "logps/chosen": -309.689453125,
+      "logps/rejected": -257.48760986328125,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.03487952798604965,
+      "rewards/margins": 0.06294190138578415,
+      "rewards/rejected": -0.0978214293718338,
+      "step": 1270
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 1.8811028144744398e-07,
+      "logits/chosen": -1.1761606931686401,
+      "logits/rejected": -1.239966869354248,
+      "logps/chosen": -341.273681640625,
+      "logps/rejected": -273.9056701660156,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.031055117025971413,
+      "rewards/margins": 0.08567170798778534,
+      "rewards/rejected": -0.1167268380522728,
+      "step": 1280
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 1.8523836875358988e-07,
+      "logits/chosen": -1.2572320699691772,
+      "logits/rejected": -1.2493976354599,
+      "logps/chosen": -280.1720886230469,
+      "logps/rejected": -230.7694854736328,
+      "loss": 0.667,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.05663619190454483,
+      "rewards/margins": 0.0724005326628685,
+      "rewards/rejected": -0.12903672456741333,
+      "step": 1290
+    },
+    {
+      "epoch": 2.0,
+      "eval_logits/chosen": -1.2786041498184204,
+      "eval_logits/rejected": -1.4456814527511597,
+      "eval_logps/chosen": -271.3624572753906,
+      "eval_logps/rejected": -241.35267639160156,
+      "eval_loss": 0.6636343002319336,
+      "eval_rewards/accuracies": 0.6517857313156128,
+      "eval_rewards/chosen": -0.047330476343631744,
+      "eval_rewards/margins": 0.06953465938568115,
+      "eval_rewards/rejected": -0.1168651357293129,
+      "eval_runtime": 335.8327,
+      "eval_samples_per_second": 5.955,
+      "eval_steps_per_second": 0.25,
+      "step": 1291
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 1.8236645605973577e-07,
+      "logits/chosen": -1.2015001773834229,
+      "logits/rejected": -1.2663769721984863,
+      "logps/chosen": -270.6482238769531,
+      "logps/rejected": -263.001708984375,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.04612879827618599,
+      "rewards/margins": 0.05501723289489746,
+      "rewards/rejected": -0.10114602744579315,
+      "step": 1300
+    },
+    {
+      "epoch": 2.03,
+      "learning_rate": 1.7949454336588167e-07,
+      "logits/chosen": -1.1482102870941162,
+      "logits/rejected": -1.2331697940826416,
+      "logps/chosen": -276.347412109375,
+      "logps/rejected": -251.0341796875,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.05418051406741142,
+      "rewards/margins": 0.06352085620164871,
+      "rewards/rejected": -0.11770136654376984,
+      "step": 1310
+    },
+    {
+      "epoch": 2.04,
+      "learning_rate": 1.7662263067202756e-07,
+      "logits/chosen": -1.1770139932632446,
+      "logits/rejected": -1.1769516468048096,
+      "logps/chosen": -285.74200439453125,
+      "logps/rejected": -251.66641235351562,
+      "loss": 0.6635,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.056278862059116364,
+      "rewards/margins": 0.05115005373954773,
+      "rewards/rejected": -0.1074289083480835,
+      "step": 1320
+    },
+    {
+      "epoch": 2.06,
+      "learning_rate": 1.7375071797817346e-07,
+      "logits/chosen": -1.1344884634017944,
+      "logits/rejected": -1.1952565908432007,
+      "logps/chosen": -281.35772705078125,
+      "logps/rejected": -241.61953735351562,
+      "loss": 0.662,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.05323750525712967,
+      "rewards/margins": 0.04273111745715141,
+      "rewards/rejected": -0.09596861898899078,
+      "step": 1330
+    },
+    {
+      "epoch": 2.08,
+      "learning_rate": 1.7087880528431935e-07,
+      "logits/chosen": -1.1021802425384521,
+      "logits/rejected": -1.1934154033660889,
+      "logps/chosen": -306.3484802246094,
+      "logps/rejected": -239.8199005126953,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03922051936388016,
+      "rewards/margins": 0.07538510859012604,
+      "rewards/rejected": -0.1146056279540062,
+      "step": 1340
+    },
+    {
+      "epoch": 2.09,
+      "learning_rate": 1.6800689259046525e-07,
+      "logits/chosen": -1.2181620597839355,
+      "logits/rejected": -1.3131446838378906,
+      "logps/chosen": -274.2059326171875,
+      "logps/rejected": -230.08395385742188,
+      "loss": 0.6647,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.045338355004787445,
+      "rewards/margins": 0.06275372207164764,
+      "rewards/rejected": -0.10809208452701569,
+      "step": 1350
+    },
+    {
+      "epoch": 2.11,
+      "learning_rate": 1.6513497989661112e-07,
+      "logits/chosen": -1.1318702697753906,
+      "logits/rejected": -1.26137113571167,
+      "logps/chosen": -282.0677795410156,
+      "logps/rejected": -236.37948608398438,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.05192691087722778,
+      "rewards/margins": 0.07602426409721375,
+      "rewards/rejected": -0.12795117497444153,
+      "step": 1360
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 1.6226306720275704e-07,
+      "logits/chosen": -1.1841795444488525,
+      "logits/rejected": -1.297102451324463,
+      "logps/chosen": -305.37451171875,
+      "logps/rejected": -253.83737182617188,
+      "loss": 0.6605,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03761546313762665,
+      "rewards/margins": 0.06815005838871002,
+      "rewards/rejected": -0.10576552152633667,
+      "step": 1370
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 1.5939115450890293e-07,
+      "logits/chosen": -1.2069101333618164,
+      "logits/rejected": -1.291855812072754,
+      "logps/chosen": -324.9100341796875,
+      "logps/rejected": -252.5089111328125,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04376627877354622,
+      "rewards/margins": 0.05180872231721878,
+      "rewards/rejected": -0.0955749899148941,
+      "step": 1380
+    },
+    {
+      "epoch": 2.15,
+      "learning_rate": 1.565192418150488e-07,
+      "logits/chosen": -1.158197045326233,
+      "logits/rejected": -1.2244586944580078,
+      "logps/chosen": -276.658935546875,
+      "logps/rejected": -248.20895385742188,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07143920660018921,
+      "rewards/margins": 0.021507441997528076,
+      "rewards/rejected": -0.09294665604829788,
+      "step": 1390
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 1.5364732912119472e-07,
+      "logits/chosen": -1.1375401020050049,
+      "logits/rejected": -1.182159185409546,
+      "logps/chosen": -309.81658935546875,
+      "logps/rejected": -270.9930114746094,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0605141744017601,
+      "rewards/margins": 0.08565731346607208,
+      "rewards/rejected": -0.1461714804172516,
+      "step": 1400
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 1.507754164273406e-07,
+      "logits/chosen": -1.1340091228485107,
+      "logits/rejected": -1.230928897857666,
+      "logps/chosen": -296.4352722167969,
+      "logps/rejected": -244.35397338867188,
+      "loss": 0.659,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04029616340994835,
+      "rewards/margins": 0.09661685675382614,
+      "rewards/rejected": -0.13691303133964539,
+      "step": 1410
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 1.479035037334865e-07,
+      "logits/chosen": -1.2611504793167114,
+      "logits/rejected": -1.2926255464553833,
+      "logps/chosen": -294.1080017089844,
+      "logps/rejected": -224.856201171875,
+      "loss": 0.661,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.060603827238082886,
+      "rewards/margins": 0.07348962128162384,
+      "rewards/rejected": -0.13409346342086792,
+      "step": 1420
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 1.4503159103963238e-07,
+      "logits/chosen": -1.1362766027450562,
+      "logits/rejected": -1.2392619848251343,
+      "logps/chosen": -318.2626037597656,
+      "logps/rejected": -277.73138427734375,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.0653274655342102,
+      "rewards/margins": 0.07404564321041107,
+      "rewards/rejected": -0.13937310874462128,
+      "step": 1430
+    },
+    {
+      "epoch": 2.23,
+      "learning_rate": 1.4215967834577827e-07,
+      "logits/chosen": -1.1209288835525513,
+      "logits/rejected": -1.2438892126083374,
+      "logps/chosen": -298.46087646484375,
+      "logps/rejected": -234.68795776367188,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.04315265268087387,
+      "rewards/margins": 0.07634139806032181,
+      "rewards/rejected": -0.11949405819177628,
+      "step": 1440
+    },
+    {
+      "epoch": 2.25,
+      "learning_rate": 1.392877656519242e-07,
+      "logits/chosen": -1.1044480800628662,
+      "logits/rejected": -1.1969009637832642,
+      "logps/chosen": -299.08575439453125,
+      "logps/rejected": -264.5302429199219,
+      "loss": 0.6613,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.05541745945811272,
+      "rewards/margins": 0.09096468240022659,
+      "rewards/rejected": -0.146382138133049,
+      "step": 1450
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 1.3641585295807006e-07,
+      "logits/chosen": -1.1369094848632812,
+      "logits/rejected": -1.2164510488510132,
+      "logps/chosen": -264.0603332519531,
+      "logps/rejected": -248.4859161376953,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.045277245342731476,
+      "rewards/margins": 0.07062248885631561,
+      "rewards/rejected": -0.11589974164962769,
+      "step": 1460
+    },
+    {
+      "epoch": 2.28,
+      "learning_rate": 1.3354394026421596e-07,
+      "logits/chosen": -1.177728533744812,
+      "logits/rejected": -1.2632405757904053,
+      "logps/chosen": -292.7504577636719,
+      "logps/rejected": -247.25338745117188,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.06021810322999954,
+      "rewards/margins": 0.07726706564426422,
+      "rewards/rejected": -0.13748517632484436,
+      "step": 1470
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 1.3067202757036185e-07,
+      "logits/chosen": -1.0999003648757935,
+      "logits/rejected": -1.2220264673233032,
+      "logps/chosen": -315.4903259277344,
+      "logps/rejected": -256.5108947753906,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.026393622159957886,
+      "rewards/margins": 0.13147220015525818,
+      "rewards/rejected": -0.15786585211753845,
+      "step": 1480
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 1.2780011487650774e-07,
+      "logits/chosen": -1.1594305038452148,
+      "logits/rejected": -1.204489827156067,
+      "logps/chosen": -248.64297485351562,
+      "logps/rejected": -231.457275390625,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06583482772111893,
+      "rewards/margins": 0.06832978874444962,
+      "rewards/rejected": -0.13416460156440735,
+      "step": 1490
+    },
+    {
+      "epoch": 2.32,
+      "learning_rate": 1.2492820218265364e-07,
+      "logits/chosen": -1.2294750213623047,
+      "logits/rejected": -1.2381101846694946,
+      "logps/chosen": -301.78668212890625,
+      "logps/rejected": -262.041259765625,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0509830042719841,
+      "rewards/margins": 0.07165484875440598,
+      "rewards/rejected": -0.12263785302639008,
+      "step": 1500
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 1.2205628948879953e-07,
+      "logits/chosen": -1.187576413154602,
+      "logits/rejected": -1.208719253540039,
+      "logps/chosen": -278.07391357421875,
+      "logps/rejected": -239.7738800048828,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07879756391048431,
+      "rewards/margins": 0.06315238773822784,
+      "rewards/rejected": -0.14194993674755096,
+      "step": 1510
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 1.1918437679494543e-07,
+      "logits/chosen": -1.1406846046447754,
+      "logits/rejected": -1.2449123859405518,
+      "logps/chosen": -280.1681213378906,
+      "logps/rejected": -251.434814453125,
+      "loss": 0.6564,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.051693908870220184,
+      "rewards/margins": 0.08930797874927521,
+      "rewards/rejected": -0.141001895070076,
+      "step": 1520
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 1.1631246410109131e-07,
+      "logits/chosen": -1.1076427698135376,
+      "logits/rejected": -1.200714111328125,
+      "logps/chosen": -302.08673095703125,
+      "logps/rejected": -255.66354370117188,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.046904128044843674,
+      "rewards/margins": 0.09861443936824799,
+      "rewards/rejected": -0.14551857113838196,
+      "step": 1530
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 1.1344055140723722e-07,
+      "logits/chosen": -1.152897596359253,
+      "logits/rejected": -1.2206456661224365,
+      "logps/chosen": -290.43511962890625,
+      "logps/rejected": -253.755615234375,
+      "loss": 0.6623,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.07052378356456757,
+      "rewards/margins": 0.06900990009307861,
+      "rewards/rejected": -0.13953366875648499,
+      "step": 1540
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 1.1056863871338311e-07,
+      "logits/chosen": -1.1606311798095703,
+      "logits/rejected": -1.2234033346176147,
+      "logps/chosen": -275.7051086425781,
+      "logps/rejected": -257.5712585449219,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.06670763343572617,
+      "rewards/margins": 0.07591570168733597,
+      "rewards/rejected": -0.14262333512306213,
+      "step": 1550
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 1.07696726019529e-07,
+      "logits/chosen": -1.0971261262893677,
+      "logits/rejected": -1.2159391641616821,
+      "logps/chosen": -303.5472717285156,
+      "logps/rejected": -262.9560546875,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06790264695882797,
+      "rewards/margins": 0.08240754902362823,
+      "rewards/rejected": -0.1503101885318756,
+      "step": 1560
+    },
+    {
+      "epoch": 2.43,
+      "learning_rate": 1.0482481332567489e-07,
+      "logits/chosen": -1.1438579559326172,
+      "logits/rejected": -1.2413270473480225,
+      "logps/chosen": -302.71905517578125,
+      "logps/rejected": -233.7013702392578,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.049414295703172684,
+      "rewards/margins": 0.07926493883132935,
+      "rewards/rejected": -0.12867923080921173,
+      "step": 1570
+    },
+    {
+      "epoch": 2.45,
+      "learning_rate": 1.0195290063182078e-07,
+      "logits/chosen": -1.1184961795806885,
+      "logits/rejected": -1.191981315612793,
+      "logps/chosen": -271.0045471191406,
+      "logps/rejected": -243.72982788085938,
+      "loss": 0.6574,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0692010149359703,
+      "rewards/margins": 0.06468804180622101,
+      "rewards/rejected": -0.13388904929161072,
+      "step": 1580
+    },
+    {
+      "epoch": 2.46,
+      "learning_rate": 9.908098793796668e-08,
+      "logits/chosen": -1.1387625932693481,
+      "logits/rejected": -1.1771440505981445,
+      "logps/chosen": -298.8290710449219,
+      "logps/rejected": -284.1943054199219,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.049336209893226624,
+      "rewards/margins": 0.08348609507083893,
+      "rewards/rejected": -0.13282229006290436,
+      "step": 1590
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 9.620907524411257e-08,
+      "logits/chosen": -1.1552401781082153,
+      "logits/rejected": -1.2284502983093262,
+      "logps/chosen": -289.09979248046875,
+      "logps/rejected": -264.1502685546875,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.06869687885046005,
+      "rewards/margins": 0.05455329269170761,
+      "rewards/rejected": -0.12325018644332886,
+      "step": 1600
+    },
+    {
+      "epoch": 2.49,
+      "learning_rate": 9.333716255025848e-08,
+      "logits/chosen": -1.2256122827529907,
+      "logits/rejected": -1.3060810565948486,
+      "logps/chosen": -299.1298522949219,
+      "logps/rejected": -260.7080993652344,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.05211590602993965,
+      "rewards/margins": 0.09913495182991028,
+      "rewards/rejected": -0.15125085413455963,
+      "step": 1610
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 9.046524985640436e-08,
+      "logits/chosen": -1.196408987045288,
+      "logits/rejected": -1.234323263168335,
+      "logps/chosen": -276.99462890625,
+      "logps/rejected": -269.7616882324219,
+      "loss": 0.6645,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.06248649209737778,
+      "rewards/margins": 0.0762564167380333,
+      "rewards/rejected": -0.13874289393424988,
+      "step": 1620
+    },
+    {
+      "epoch": 2.53,
+      "learning_rate": 8.759333716255026e-08,
+      "logits/chosen": -1.1722522974014282,
+      "logits/rejected": -1.2352186441421509,
+      "logps/chosen": -317.6283264160156,
+      "logps/rejected": -265.194580078125,
+      "loss": 0.656,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06108713895082474,
+      "rewards/margins": 0.09596574306488037,
+      "rewards/rejected": -0.1570528745651245,
+      "step": 1630
+    },
+    {
+      "epoch": 2.54,
+      "learning_rate": 8.472142446869615e-08,
+      "logits/chosen": -1.253702163696289,
+      "logits/rejected": -1.2648756504058838,
+      "logps/chosen": -311.29071044921875,
+      "logps/rejected": -259.5384826660156,
+      "loss": 0.6568,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.05935510993003845,
+      "rewards/margins": 0.08772717416286469,
+      "rewards/rejected": -0.14708228409290314,
+      "step": 1640
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 8.184951177484204e-08,
+      "logits/chosen": -1.1213136911392212,
+      "logits/rejected": -1.1935571432113647,
+      "logps/chosen": -298.7812194824219,
+      "logps/rejected": -263.63189697265625,
+      "loss": 0.6595,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.04742642492055893,
+      "rewards/margins": 0.12104301154613495,
+      "rewards/rejected": -0.16846942901611328,
+      "step": 1650
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 7.897759908098793e-08,
+      "logits/chosen": -1.1366965770721436,
+      "logits/rejected": -1.2882981300354004,
+      "logps/chosen": -298.13671875,
+      "logps/rejected": -255.32351684570312,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.06480282545089722,
+      "rewards/margins": 0.11208094656467438,
+      "rewards/rejected": -0.1768837720155716,
+      "step": 1660
+    },
+    {
+      "epoch": 2.59,
+      "learning_rate": 7.610568638713383e-08,
+      "logits/chosen": -1.153519630432129,
+      "logits/rejected": -1.1792590618133545,
+      "logps/chosen": -274.3094177246094,
+      "logps/rejected": -277.9222717285156,
+      "loss": 0.6529,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.060344815254211426,
+      "rewards/margins": 0.07477900385856628,
+      "rewards/rejected": -0.1351238191127777,
+      "step": 1670
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 7.323377369327973e-08,
+      "logits/chosen": -1.1138644218444824,
+      "logits/rejected": -1.2077605724334717,
+      "logps/chosen": -284.9149475097656,
+      "logps/rejected": -238.51315307617188,
+      "loss": 0.6584,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.05787507817149162,
+      "rewards/margins": 0.08078931272029877,
+      "rewards/rejected": -0.1386643946170807,
+      "step": 1680
+    },
+    {
+      "epoch": 2.62,
+      "learning_rate": 7.036186099942562e-08,
+      "logits/chosen": -1.1559264659881592,
+      "logits/rejected": -1.2110865116119385,
+      "logps/chosen": -273.95074462890625,
+      "logps/rejected": -252.70968627929688,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.036359649151563644,
+      "rewards/margins": 0.1054670661687851,
+      "rewards/rejected": -0.14182671904563904,
+      "step": 1690
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 6.74899483055715e-08,
+      "logits/chosen": -1.208526849746704,
+      "logits/rejected": -1.26242196559906,
+      "logps/chosen": -297.33843994140625,
+      "logps/rejected": -268.06573486328125,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.046420447528362274,
+      "rewards/margins": 0.08910312503576279,
+      "rewards/rejected": -0.13552358746528625,
+      "step": 1700
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 6.46180356117174e-08,
+      "logits/chosen": -1.1605157852172852,
+      "logits/rejected": -1.1535265445709229,
+      "logps/chosen": -276.1454772949219,
+      "logps/rejected": -264.06512451171875,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.06386712193489075,
+      "rewards/margins": 0.06835543364286423,
+      "rewards/rejected": -0.13222254812717438,
+      "step": 1710
+    },
+    {
+      "epoch": 2.66,
+      "learning_rate": 6.174612291786329e-08,
+      "logits/chosen": -1.1071873903274536,
+      "logits/rejected": -1.2749332189559937,
+      "logps/chosen": -312.7781066894531,
+      "logps/rejected": -263.4851989746094,
+      "loss": 0.654,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07527683675289154,
+      "rewards/margins": 0.07555703818798065,
+      "rewards/rejected": -0.1508338749408722,
+      "step": 1720
+    },
+    {
+      "epoch": 2.68,
+      "learning_rate": 5.887421022400919e-08,
+      "logits/chosen": -1.132735252380371,
+      "logits/rejected": -1.2301145792007446,
+      "logps/chosen": -309.93048095703125,
+      "logps/rejected": -240.8332061767578,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.07498300075531006,
+      "rewards/margins": 0.08411093801259995,
+      "rewards/rejected": -0.1590939462184906,
+      "step": 1730
+    },
+    {
+      "epoch": 2.7,
+      "learning_rate": 5.600229753015508e-08,
+      "logits/chosen": -1.1434319019317627,
+      "logits/rejected": -1.2648731470108032,
+      "logps/chosen": -310.2062683105469,
+      "logps/rejected": -241.0065155029297,
+      "loss": 0.6521,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.058048129081726074,
+      "rewards/margins": 0.09094297885894775,
+      "rewards/rejected": -0.14899109303951263,
+      "step": 1740
+    },
+    {
+      "epoch": 2.71,
+      "learning_rate": 5.313038483630097e-08,
+      "logits/chosen": -1.1104892492294312,
+      "logits/rejected": -1.2646420001983643,
+      "logps/chosen": -319.0297546386719,
+      "logps/rejected": -264.7847595214844,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.03740755841135979,
+      "rewards/margins": 0.09465880692005157,
+      "rewards/rejected": -0.13206636905670166,
+      "step": 1750
+    },
+    {
+      "epoch": 2.73,
+      "learning_rate": 5.025847214244687e-08,
+      "logits/chosen": -1.1556150913238525,
+      "logits/rejected": -1.1965545415878296,
+      "logps/chosen": -302.27960205078125,
+      "logps/rejected": -237.3252716064453,
+      "loss": 0.6585,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.0522063672542572,
+      "rewards/margins": 0.08026628196239471,
+      "rewards/rejected": -0.13247264921665192,
+      "step": 1760
+    },
+    {
+      "epoch": 2.74,
+      "learning_rate": 4.738655944859276e-08,
+      "logits/chosen": -1.1446633338928223,
+      "logits/rejected": -1.2452003955841064,
+      "logps/chosen": -330.00531005859375,
+      "logps/rejected": -293.3304138183594,
+      "loss": 0.6612,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07666303217411041,
+      "rewards/margins": 0.08382611721754074,
+      "rewards/rejected": -0.16048914194107056,
+      "step": 1770
+    },
+    {
+      "epoch": 2.76,
+      "learning_rate": 4.4514646754738654e-08,
+      "logits/chosen": -1.1485297679901123,
+      "logits/rejected": -1.2428683042526245,
+      "logps/chosen": -301.80657958984375,
+      "logps/rejected": -243.60043334960938,
+      "loss": 0.6477,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.060366760939359665,
+      "rewards/margins": 0.09364822506904602,
+      "rewards/rejected": -0.15401498973369598,
+      "step": 1780
+    },
+    {
+      "epoch": 2.77,
+      "learning_rate": 4.164273406088455e-08,
+      "logits/chosen": -1.1510671377182007,
+      "logits/rejected": -1.2577351331710815,
+      "logps/chosen": -299.68896484375,
+      "logps/rejected": -282.103515625,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.05730317905545235,
+      "rewards/margins": 0.09459875524044037,
+      "rewards/rejected": -0.1519019454717636,
+      "step": 1790
+    },
+    {
+      "epoch": 2.79,
+      "learning_rate": 3.877082136703044e-08,
+      "logits/chosen": -1.06912100315094,
+      "logits/rejected": -1.2072498798370361,
+      "logps/chosen": -320.6122131347656,
+      "logps/rejected": -260.58599853515625,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.0490909144282341,
+      "rewards/margins": 0.12078486382961273,
+      "rewards/rejected": -0.16987577080726624,
+      "step": 1800
+    },
+    {
+      "epoch": 2.8,
+      "learning_rate": 3.589890867317633e-08,
+      "logits/chosen": -1.0877912044525146,
+      "logits/rejected": -1.1389758586883545,
+      "logps/chosen": -312.76190185546875,
+      "logps/rejected": -258.6143798828125,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.05302386358380318,
+      "rewards/margins": 0.0961315929889679,
+      "rewards/rejected": -0.14915546774864197,
+      "step": 1810
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 3.302699597932223e-08,
+      "logits/chosen": -1.1518110036849976,
+      "logits/rejected": -1.1895443201065063,
+      "logps/chosen": -286.28955078125,
+      "logps/rejected": -237.89675903320312,
+      "loss": 0.6577,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.03918531537055969,
+      "rewards/margins": 0.10102204233407974,
+      "rewards/rejected": -0.14020733535289764,
+      "step": 1820
+    },
+    {
+      "epoch": 2.84,
+      "learning_rate": 3.015508328546812e-08,
+      "logits/chosen": -1.161084771156311,
+      "logits/rejected": -1.2788031101226807,
+      "logps/chosen": -280.8258972167969,
+      "logps/rejected": -248.14028930664062,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.06762611865997314,
+      "rewards/margins": 0.08084806054830551,
+      "rewards/rejected": -0.14847418665885925,
+      "step": 1830
+    },
+    {
+      "epoch": 2.85,
+      "learning_rate": 2.728317059161401e-08,
+      "logits/chosen": -1.144046664237976,
+      "logits/rejected": -1.2353025674819946,
+      "logps/chosen": -296.1619567871094,
+      "logps/rejected": -254.094970703125,
+      "loss": 0.6511,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06453101336956024,
+      "rewards/margins": 0.08718563616275787,
+      "rewards/rejected": -0.1517166644334793,
+      "step": 1840
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 2.4411257897759906e-08,
+      "logits/chosen": -1.0756458044052124,
+      "logits/rejected": -1.1395611763000488,
+      "logps/chosen": -272.8941955566406,
+      "logps/rejected": -248.87109375,
+      "loss": 0.6567,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06941407173871994,
+      "rewards/margins": 0.057349514216184616,
+      "rewards/rejected": -0.12676359713077545,
+      "step": 1850
+    },
+    {
+      "epoch": 2.88,
+      "learning_rate": 2.1539345203905797e-08,
+      "logits/chosen": -1.2316334247589111,
+      "logits/rejected": -1.2592030763626099,
+      "logps/chosen": -285.9391784667969,
+      "logps/rejected": -258.1458740234375,
+      "loss": 0.6625,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0662040263414383,
+      "rewards/margins": 0.08364922553300858,
+      "rewards/rejected": -0.14985325932502747,
+      "step": 1860
+    },
+    {
+      "epoch": 2.9,
+      "learning_rate": 1.8667432510051692e-08,
+      "logits/chosen": -1.1365151405334473,
+      "logits/rejected": -1.273109793663025,
+      "logps/chosen": -293.24676513671875,
+      "logps/rejected": -247.6940460205078,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.07556969672441483,
+      "rewards/margins": 0.10609056055545807,
+      "rewards/rejected": -0.1816602647304535,
+      "step": 1870
+    },
+    {
+      "epoch": 2.91,
+      "learning_rate": 1.5795519816197586e-08,
+      "logits/chosen": -1.20346999168396,
+      "logits/rejected": -1.225865364074707,
+      "logps/chosen": -296.02911376953125,
+      "logps/rejected": -248.4931640625,
+      "loss": 0.6531,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04528854414820671,
+      "rewards/margins": 0.09233234822750092,
+      "rewards/rejected": -0.13762088119983673,
+      "step": 1880
+    },
+    {
+      "epoch": 2.93,
+      "learning_rate": 1.2923607122343481e-08,
+      "logits/chosen": -1.1362476348876953,
+      "logits/rejected": -1.1712617874145508,
+      "logps/chosen": -327.4137268066406,
+      "logps/rejected": -262.9008483886719,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06112866848707199,
+      "rewards/margins": 0.09199599176645279,
+      "rewards/rejected": -0.15312466025352478,
+      "step": 1890
+    },
+    {
+      "epoch": 2.94,
+      "learning_rate": 1.0051694428489374e-08,
+      "logits/chosen": -1.1542044878005981,
+      "logits/rejected": -1.2174440622329712,
+      "logps/chosen": -274.6893615722656,
+      "logps/rejected": -254.463623046875,
+      "loss": 0.6558,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.060687076300382614,
+      "rewards/margins": 0.0842573344707489,
+      "rewards/rejected": -0.14494439959526062,
+      "step": 1900
+    },
+    {
+      "epoch": 2.96,
+      "learning_rate": 7.179781734635267e-09,
+      "logits/chosen": -1.108367919921875,
+      "logits/rejected": -1.16150963306427,
+      "logps/chosen": -288.6197509765625,
+      "logps/rejected": -249.8366241455078,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.060938142240047455,
+      "rewards/margins": 0.08311086148023605,
+      "rewards/rejected": -0.1440490037202835,
+      "step": 1910
+    },
+    {
+      "epoch": 2.97,
+      "learning_rate": 4.30786904078116e-09,
+      "logits/chosen": -1.0845935344696045,
+      "logits/rejected": -1.1811341047286987,
+      "logps/chosen": -297.39599609375,
+      "logps/rejected": -259.75,
+      "loss": 0.6557,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.04327515885233879,
+      "rewards/margins": 0.0852704644203186,
+      "rewards/rejected": -0.1285456269979477,
+      "step": 1920
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 1.4359563469270533e-09,
+      "logits/chosen": -1.2177361249923706,
+      "logits/rejected": -1.2058055400848389,
+      "logps/chosen": -263.7599182128906,
+      "logps/rejected": -247.1952667236328,
+      "loss": 0.6518,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.06555221229791641,
+      "rewards/margins": 0.0862443745136261,
+      "rewards/rejected": -0.15179657936096191,
+      "step": 1930
+    },
+    {
+      "epoch": 3.0,
+      "eval_logits/chosen": -1.2746007442474365,
+      "eval_logits/rejected": -1.4412107467651367,
+      "eval_logps/chosen": -271.52374267578125,
+      "eval_logps/rejected": -241.7964630126953,
+      "eval_loss": 0.6550792455673218,
+      "eval_rewards/accuracies": 0.6904761791229248,
+      "eval_rewards/chosen": -0.0634559839963913,
+      "eval_rewards/margins": 0.09778770059347153,
+      "eval_rewards/rejected": -0.16124369204044342,
+      "eval_runtime": 340.5371,
+      "eval_samples_per_second": 5.873,
+      "eval_steps_per_second": 0.247,
+      "step": 1935
+    },
+    {
+      "epoch": 3.0,
+      "step": 1935,
+      "total_flos": 0.0,
+      "train_loss": 0.6738453804676539,
+      "train_runtime": 49317.7491,
+      "train_samples_per_second": 3.769,
+      "train_steps_per_second": 0.039
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1935,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:24524aac373f1c2b4d3963d3d935ed08a5a179ba07bd5079fecbbcf988d6198c
+size 4728