Duplicate from alignment-handbook/zephyr-7b-dpo-qlora

Co-authored-by: Edward Beeching <edbeeching@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,87 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- alignment-handbook
+- generated_from_trainer
+- trl
+- dpo
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
+base_model: mistralai/Mistral-7B-v0.1
+model-index:
+- name: zephyr-7b-dpo-qlora
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-qlora
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-qlora](https://huggingface.co/alignment-handbook/zephyr-7b-sft-qlora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5473
+- Rewards/chosen: -0.8609
+- Rewards/rejected: -1.5251
+- Rewards/accuracies: 0.7422
+- Rewards/margins: 0.6641
+- Logps/rejected: -404.3018
+- Logps/chosen: -336.2481
+- Logits/rejected: 0.0706
+- Logits/chosen: -0.1471
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6812        | 0.1   | 100  | 0.6787          | 0.0452         | 0.0120           | 0.6992             | 0.0332          | -250.5929      | -245.6322    | -2.1942         | -2.2517       |
+| 0.6066        | 0.21  | 200  | 0.6151          | -0.2303        | -0.5020          | 0.6992             | 0.2717          | -301.9975      | -273.1855    | -1.9906         | -2.0610       |
+| 0.5711        | 0.31  | 300  | 0.5927          | -0.4441        | -0.8513          | 0.7188             | 0.4072          | -336.9228      | -294.5666    | -1.9417         | -2.0223       |
+| 0.557         | 0.42  | 400  | 0.5817          | -0.5958        | -1.0732          | 0.7227             | 0.4773          | -359.1117      | -309.7378    | -1.7434         | -1.8364       |
+| 0.5703        | 0.52  | 500  | 0.5679          | -0.7215        | -1.2405          | 0.7266             | 0.5189          | -375.8402      | -322.3068    | -0.8467         | -0.9967       |
+| 0.5498        | 0.63  | 600  | 0.5582          | -0.7003        | -1.2848          | 0.7578             | 0.5845          | -380.2699      | -320.1794    | -0.2510         | -0.4463       |
+| 0.5279        | 0.73  | 700  | 0.5490          | -0.8400        | -1.4901          | 0.75               | 0.6501          | -400.8082      | -334.1553    | 0.0145          | -0.1988       |
+| 0.5264        | 0.84  | 800  | 0.5475          | -0.8613        | -1.5228          | 0.7461             | 0.6615          | -404.0751      | -336.2833    | 0.0604          | -0.1549       |
+| 0.5639        | 0.94  | 900  | 0.5475          | -0.8628        | -1.5267          | 0.7422             | 0.6639          | -404.4688      | -336.4348    | 0.0704          | -0.1466       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.0

adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "v_proj",
+    "down_proj",
+    "gate_proj",
+    "q_proj",
+    "o_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df088d529a99139afd8aba68b8c1e31a72348a5aab3f67350d249cc19d8abd38
+size 83945744

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -0.14711737632751465,
+    "eval_logits/rejected": 0.07057683169841766,
+    "eval_logps/chosen": -336.2481384277344,
+    "eval_logps/rejected": -404.3017578125,
+    "eval_loss": 0.5472621321678162,
+    "eval_rewards/accuracies": 0.7421875,
+    "eval_rewards/chosen": -0.8609448075294495,
+    "eval_rewards/margins": 0.6641340851783752,
+    "eval_rewards/rejected": -1.5250788927078247,
+    "eval_runtime": 99.6217,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 20.076,
+    "eval_steps_per_second": 0.321,
+    "train_loss": 0.583915277301329,
+    "train_runtime": 6210.8046,
+    "train_samples": 61135,
+    "train_samples_per_second": 9.843,
+    "train_steps_per_second": 0.154
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -0.14711737632751465,
+    "eval_logits/rejected": 0.07057683169841766,
+    "eval_logps/chosen": -336.2481384277344,
+    "eval_logps/rejected": -404.3017578125,
+    "eval_loss": 0.5472621321678162,
+    "eval_rewards/accuracies": 0.7421875,
+    "eval_rewards/chosen": -0.8609448075294495,
+    "eval_rewards/margins": 0.6641340851783752,
+    "eval_rewards/rejected": -1.5250788927078247,
+    "eval_runtime": 99.6217,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 20.076,
+    "eval_steps_per_second": 0.321
+}

runs/Jan09_05-07-46_ip-26-0-175-170/events.out.tfevents.1704777003.ip-26-0-175-170.1799139.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07ae93d49c31ce3dd08cfe99bd93ac32a5d111827d3113468a1ef52d46c8e90c
+size 140544

runs/Jan09_05-07-46_ip-26-0-175-170/events.out.tfevents.1704784713.ip-26-0-175-170.1799139.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea4e75f57b9d81409e2cae26268f15bb6b03b1bfe6164baadf83b6b0a10d5156
+size 828

runs/Jan09_21-38-54_ip-26-0-163-134/events.out.tfevents.1704836456.ip-26-0-163-134.3006272.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9aded02ad5abdc9d150dca93aa11939c432d7679fc6f7d00ddbd37b60caf962
+size 72280

runs/Jan09_21-38-54_ip-26-0-163-134/events.out.tfevents.1704842766.ip-26-0-163-134.3006272.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec72fa83fca2cfd2971a75f79691c91843a18eef743837ac959b4d655fbcba78
+size 828

runs/Nov10_12-20-56_ip-26-0-152-211/events.out.tfevents.1699619038.ip-26-0-152-211.859454.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c586c809d7ec4c3f124fbde2bab0946be97638a3d620e49dd6dd6f9e1209bb5d
+size 191268

runs/Nov10_12-20-56_ip-26-0-152-211/events.out.tfevents.1699627184.ip-26-0-152-211.859454.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88fec9f9c49e10efa8c9e63c0c5ca51aecc9b402f8b7573a96c19f94074f65cf
+size 828

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.583915277301329,
+    "train_runtime": 6210.8046,
+    "train_samples": 61135,
+    "train_samples_per_second": 9.843,
+    "train_steps_per_second": 0.154
+}

trainer_state.json

@@ -0,0 +1,1518 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9994767137624281,
+  "eval_steps": 100,
+  "global_step": 955,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.4445395469665527,
+      "logits/rejected": -2.388306140899658,
+      "logps/chosen": -221.89984130859375,
+      "logps/rejected": -197.11672973632812,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.208333333333334e-07,
+      "logits/chosen": -2.385066509246826,
+      "logits/rejected": -2.237809181213379,
+      "logps/chosen": -315.8031005859375,
+      "logps/rejected": -248.2730255126953,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 8.269523277704138e-06,
+      "rewards/margins": -0.00012648198753595352,
+      "rewards/rejected": 0.00013475156447384506,
+      "step": 10
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-06,
+      "logits/chosen": -2.166614294052124,
+      "logits/rejected": -2.138037919998169,
+      "logps/chosen": -230.2421112060547,
+      "logps/rejected": -215.0667724609375,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00011211812670808285,
+      "rewards/margins": -0.00017659642617218196,
+      "rewards/rejected": 6.44782921881415e-05,
+      "step": 20
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5625e-06,
+      "logits/chosen": -2.359757900238037,
+      "logits/rejected": -2.239750623703003,
+      "logps/chosen": -287.16546630859375,
+      "logps/rejected": -250.483154296875,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.0006519377348013222,
+      "rewards/margins": -0.000464016105979681,
+      "rewards/rejected": 0.0011159538989886642,
+      "step": 30
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333334e-06,
+      "logits/chosen": -2.332831859588623,
+      "logits/rejected": -2.2222561836242676,
+      "logps/chosen": -268.5550231933594,
+      "logps/rejected": -245.136474609375,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0009799805702641606,
+      "rewards/margins": 0.00043143361108377576,
+      "rewards/rejected": 0.0005485471338033676,
+      "step": 40
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.604166666666667e-06,
+      "logits/chosen": -2.3112823963165283,
+      "logits/rejected": -2.315413236618042,
+      "logps/chosen": -252.92160034179688,
+      "logps/rejected": -255.1580352783203,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0017378001939505339,
+      "rewards/margins": 0.000829376163892448,
+      "rewards/rejected": 0.0009084242628887296,
+      "step": 50
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.125e-06,
+      "logits/chosen": -2.2834067344665527,
+      "logits/rejected": -2.2284858226776123,
+      "logps/chosen": -261.4151306152344,
+      "logps/rejected": -252.23080444335938,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0034083526115864515,
+      "rewards/margins": 0.0015303167747333646,
+      "rewards/rejected": 0.0018780359532684088,
+      "step": 60
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6458333333333333e-06,
+      "logits/chosen": -2.319131374359131,
+      "logits/rejected": -2.1937270164489746,
+      "logps/chosen": -305.9224548339844,
+      "logps/rejected": -275.6637268066406,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.010511571541428566,
+      "rewards/margins": 0.006470891647040844,
+      "rewards/rejected": 0.004040678963065147,
+      "step": 70
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.166666666666667e-06,
+      "logits/chosen": -2.155569553375244,
+      "logits/rejected": -1.9963390827178955,
+      "logps/chosen": -284.39007568359375,
+      "logps/rejected": -246.76455688476562,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.014133408665657043,
+      "rewards/margins": 0.012079447507858276,
+      "rewards/rejected": 0.002053960459306836,
+      "step": 80
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.6875000000000004e-06,
+      "logits/chosen": -2.2905330657958984,
+      "logits/rejected": -2.254070520401001,
+      "logps/chosen": -289.9971008300781,
+      "logps/rejected": -273.9203796386719,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.032837510108947754,
+      "rewards/margins": 0.019895007833838463,
+      "rewards/rejected": 0.012942495755851269,
+      "step": 90
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9997324926814375e-06,
+      "logits/chosen": -2.243922710418701,
+      "logits/rejected": -2.2560582160949707,
+      "logps/chosen": -273.7684326171875,
+      "logps/rejected": -287.9248962402344,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03909907117486,
+      "rewards/margins": 0.024433141574263573,
+      "rewards/rejected": 0.014665926806628704,
+      "step": 100
+    },
+    {
+      "epoch": 0.1,
+      "eval_logits/chosen": -2.251671075820923,
+      "eval_logits/rejected": -2.194241762161255,
+      "eval_logps/chosen": -245.63218688964844,
+      "eval_logps/rejected": -250.59286499023438,
+      "eval_loss": 0.6787430644035339,
+      "eval_rewards/accuracies": 0.69921875,
+      "eval_rewards/chosen": 0.04521488770842552,
+      "eval_rewards/margins": 0.033204685896635056,
+      "eval_rewards/rejected": 0.012010199017822742,
+      "eval_runtime": 100.4664,
+      "eval_samples_per_second": 19.907,
+      "eval_steps_per_second": 0.319,
+      "step": 100
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.996723692767927e-06,
+      "logits/chosen": -2.2775979042053223,
+      "logits/rejected": -2.168391704559326,
+      "logps/chosen": -244.2776641845703,
+      "logps/rejected": -219.0230712890625,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.03759222850203514,
+      "rewards/margins": 0.03267201781272888,
+      "rewards/rejected": 0.004920212086290121,
+      "step": 110
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9903757462135984e-06,
+      "logits/chosen": -2.20393705368042,
+      "logits/rejected": -2.0801148414611816,
+      "logps/chosen": -254.13204956054688,
+      "logps/rejected": -222.89602661132812,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03248171508312225,
+      "rewards/margins": 0.04965170845389366,
+      "rewards/rejected": -0.017169995233416557,
+      "step": 120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980697142834315e-06,
+      "logits/chosen": -2.3278605937957764,
+      "logits/rejected": -2.1921324729919434,
+      "logps/chosen": -306.6998596191406,
+      "logps/rejected": -257.21942138671875,
+      "loss": 0.661,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0004948576679453254,
+      "rewards/margins": 0.06153837591409683,
+      "rewards/rejected": -0.062033236026763916,
+      "step": 130
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967700826904229e-06,
+      "logits/chosen": -2.1966867446899414,
+      "logits/rejected": -2.082227945327759,
+      "logps/chosen": -234.10623168945312,
+      "logps/rejected": -225.2598114013672,
+      "loss": 0.649,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.027638202533125877,
+      "rewards/margins": 0.09491725265979767,
+      "rewards/rejected": -0.1225554570555687,
+      "step": 140
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.951404179843963e-06,
+      "logits/chosen": -2.0880074501037598,
+      "logits/rejected": -2.0883960723876953,
+      "logps/chosen": -279.95965576171875,
+      "logps/rejected": -290.29327392578125,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.09183233976364136,
+      "rewards/margins": 0.10491160303354263,
+      "rewards/rejected": -0.1967439353466034,
+      "step": 150
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931828996974498e-06,
+      "logits/chosen": -2.1437039375305176,
+      "logits/rejected": -2.035672187805176,
+      "logps/chosen": -293.6642761230469,
+      "logps/rejected": -308.59246826171875,
+      "loss": 0.6161,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.10774645954370499,
+      "rewards/margins": 0.21509592235088348,
+      "rewards/rejected": -0.32284238934516907,
+      "step": 160
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.909001458367867e-06,
+      "logits/chosen": -2.1584107875823975,
+      "logits/rejected": -1.9964697360992432,
+      "logps/chosen": -278.0213928222656,
+      "logps/rejected": -249.1472930908203,
+      "loss": 0.6207,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1264086812734604,
+      "rewards/margins": 0.18005268275737762,
+      "rewards/rejected": -0.306461364030838,
+      "step": 170
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882952093833628e-06,
+      "logits/chosen": -2.049844741821289,
+      "logits/rejected": -2.0112996101379395,
+      "logps/chosen": -238.4535369873047,
+      "logps/rejected": -298.28375244140625,
+      "loss": 0.6092,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.26498228311538696,
+      "rewards/margins": 0.2150886058807373,
+      "rewards/rejected": -0.4800708293914795,
+      "step": 180
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.853715742087947e-06,
+      "logits/chosen": -2.2220816612243652,
+      "logits/rejected": -2.0675864219665527,
+      "logps/chosen": -294.275390625,
+      "logps/rejected": -279.9860534667969,
+      "loss": 0.6071,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.271901398897171,
+      "rewards/margins": 0.23620745539665222,
+      "rewards/rejected": -0.5081088542938232,
+      "step": 190
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.821331504159906e-06,
+      "logits/chosen": -2.120783567428589,
+      "logits/rejected": -2.0360219478607178,
+      "logps/chosen": -284.9825134277344,
+      "logps/rejected": -325.1084899902344,
+      "loss": 0.6066,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.18728381395339966,
+      "rewards/margins": 0.21014633774757385,
+      "rewards/rejected": -0.3974301517009735,
+      "step": 200
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -2.0610172748565674,
+      "eval_logits/rejected": -1.9906072616577148,
+      "eval_logps/chosen": -273.18548583984375,
+      "eval_logps/rejected": -301.99749755859375,
+      "eval_loss": 0.6150946021080017,
+      "eval_rewards/accuracies": 0.69921875,
+      "eval_rewards/chosen": -0.23031830787658691,
+      "eval_rewards/margins": 0.2717178463935852,
+      "eval_rewards/rejected": -0.5020360946655273,
+      "eval_runtime": 100.7938,
+      "eval_samples_per_second": 19.842,
+      "eval_steps_per_second": 0.317,
+      "step": 200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.7858426910973435e-06,
+      "logits/chosen": -2.1503536701202393,
+      "logits/rejected": -1.9159533977508545,
+      "logps/chosen": -337.6061706542969,
+      "logps/rejected": -290.7955322265625,
+      "loss": 0.6126,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.21691349148750305,
+      "rewards/margins": 0.3816342353820801,
+      "rewards/rejected": -0.5985476970672607,
+      "step": 210
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.747296766042161e-06,
+      "logits/chosen": -2.0444750785827637,
+      "logits/rejected": -1.9728294610977173,
+      "logps/chosen": -281.8607177734375,
+      "logps/rejected": -295.66534423828125,
+      "loss": 0.6085,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.19859151542186737,
+      "rewards/margins": 0.28593915700912476,
+      "rewards/rejected": -0.4845306873321533,
+      "step": 220
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.705745280752586e-06,
+      "logits/chosen": -2.0718815326690674,
+      "logits/rejected": -1.9338264465332031,
+      "logps/chosen": -306.795166015625,
+      "logps/rejected": -283.50030517578125,
+      "loss": 0.6077,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2848009765148163,
+      "rewards/margins": 0.24246558547019958,
+      "rewards/rejected": -0.5272665619850159,
+      "step": 230
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.661243806657256e-06,
+      "logits/chosen": -1.994541883468628,
+      "logits/rejected": -1.8617355823516846,
+      "logps/chosen": -278.3886413574219,
+      "logps/rejected": -256.62603759765625,
+      "loss": 0.6076,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.27843061089515686,
+      "rewards/margins": 0.25517329573631287,
+      "rewards/rejected": -0.5336039066314697,
+      "step": 240
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.613851860533367e-06,
+      "logits/chosen": -1.980902910232544,
+      "logits/rejected": -1.8700807094573975,
+      "logps/chosen": -309.05877685546875,
+      "logps/rejected": -351.293212890625,
+      "loss": 0.6002,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.37613645195961,
+      "rewards/margins": 0.3849337697029114,
+      "rewards/rejected": -0.761070191860199,
+      "step": 250
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.563632824908252e-06,
+      "logits/chosen": -1.9446996450424194,
+      "logits/rejected": -1.862592339515686,
+      "logps/chosen": -309.58544921875,
+      "logps/rejected": -325.975341796875,
+      "loss": 0.5955,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4704383909702301,
+      "rewards/margins": 0.2729704976081848,
+      "rewards/rejected": -0.7434087991714478,
+      "step": 260
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.510653863290871e-06,
+      "logits/chosen": -2.056185483932495,
+      "logits/rejected": -1.9320430755615234,
+      "logps/chosen": -311.8960876464844,
+      "logps/rejected": -307.3726501464844,
+      "loss": 0.5843,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.35895800590515137,
+      "rewards/margins": 0.3666152358055115,
+      "rewards/rejected": -0.7255732417106628,
+      "step": 270
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.454985830346574e-06,
+      "logits/chosen": -2.163935422897339,
+      "logits/rejected": -2.0811800956726074,
+      "logps/chosen": -313.5486145019531,
+      "logps/rejected": -329.7764587402344,
+      "loss": 0.595,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3059764504432678,
+      "rewards/margins": 0.3349177837371826,
+      "rewards/rejected": -0.6408942937850952,
+      "step": 280
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.396703177135262e-06,
+      "logits/chosen": -2.090761423110962,
+      "logits/rejected": -2.0192935466766357,
+      "logps/chosen": -313.79547119140625,
+      "logps/rejected": -292.3644104003906,
+      "loss": 0.5862,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.42916345596313477,
+      "rewards/margins": 0.19854620099067688,
+      "rewards/rejected": -0.6277095675468445,
+      "step": 290
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.335883851539693e-06,
+      "logits/chosen": -2.0020194053649902,
+      "logits/rejected": -1.8026950359344482,
+      "logps/chosen": -264.9991455078125,
+      "logps/rejected": -290.67193603515625,
+      "loss": 0.5711,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.41096872091293335,
+      "rewards/margins": 0.36055171489715576,
+      "rewards/rejected": -0.7715204358100891,
+      "step": 300
+    },
+    {
+      "epoch": 0.31,
+      "eval_logits/chosen": -2.0222675800323486,
+      "eval_logits/rejected": -1.9416760206222534,
+      "eval_logps/chosen": -294.5666198730469,
+      "eval_logps/rejected": -336.92279052734375,
+      "eval_loss": 0.5926566123962402,
+      "eval_rewards/accuracies": 0.71875,
+      "eval_rewards/chosen": -0.4441298246383667,
+      "eval_rewards/margins": 0.40715909004211426,
+      "eval_rewards/rejected": -0.851288914680481,
+      "eval_runtime": 100.4732,
+      "eval_samples_per_second": 19.906,
+      "eval_steps_per_second": 0.318,
+      "step": 300
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.2726091940171055e-06,
+      "logits/chosen": -1.9914287328720093,
+      "logits/rejected": -1.8461036682128906,
+      "logps/chosen": -303.94287109375,
+      "logps/rejected": -359.288330078125,
+      "loss": 0.5712,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4550764560699463,
+      "rewards/margins": 0.5030550956726074,
+      "rewards/rejected": -0.9581316113471985,
+      "step": 310
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.206963828813555e-06,
+      "logits/chosen": -1.9920743703842163,
+      "logits/rejected": -1.8297895193099976,
+      "logps/chosen": -330.64007568359375,
+      "logps/rejected": -369.685302734375,
+      "loss": 0.5752,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5053039789199829,
+      "rewards/margins": 0.44878053665161133,
+      "rewards/rejected": -0.9540846943855286,
+      "step": 320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.139035550786495e-06,
+      "logits/chosen": -1.977081298828125,
+      "logits/rejected": -1.7652740478515625,
+      "logps/chosen": -318.32421875,
+      "logps/rejected": -294.5540466308594,
+      "loss": 0.6065,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5532656311988831,
+      "rewards/margins": 0.37579816579818726,
+      "rewards/rejected": -0.9290636777877808,
+      "step": 330
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.068915207986931e-06,
+      "logits/chosen": -1.8924455642700195,
+      "logits/rejected": -1.7873607873916626,
+      "logps/chosen": -267.74505615234375,
+      "logps/rejected": -279.3656921386719,
+      "loss": 0.6171,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5295882225036621,
+      "rewards/margins": 0.28365662693977356,
+      "rewards/rejected": -0.8132449388504028,
+      "step": 340
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.996696580158211e-06,
+      "logits/chosen": -1.8571460247039795,
+      "logits/rejected": -1.9016132354736328,
+      "logps/chosen": -282.2213134765625,
+      "logps/rejected": -360.13616943359375,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5124384164810181,
+      "rewards/margins": 0.5215515494346619,
+      "rewards/rejected": -1.0339899063110352,
+      "step": 350
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.922476253313921e-06,
+      "logits/chosen": -2.048137664794922,
+      "logits/rejected": -1.8757222890853882,
+      "logps/chosen": -334.61102294921875,
+      "logps/rejected": -348.38677978515625,
+      "loss": 0.577,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5598105192184448,
+      "rewards/margins": 0.36239486932754517,
+      "rewards/rejected": -0.9222054481506348,
+      "step": 360
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.846353490562664e-06,
+      "logits/chosen": -1.8339828252792358,
+      "logits/rejected": -1.7505193948745728,
+      "logps/chosen": -286.8439025878906,
+      "logps/rejected": -329.70751953125,
+      "loss": 0.5692,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.46078333258628845,
+      "rewards/margins": 0.2924962639808655,
+      "rewards/rejected": -0.7532795667648315,
+      "step": 370
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.768430099352445e-06,
+      "logits/chosen": -1.9885179996490479,
+      "logits/rejected": -1.9461723566055298,
+      "logps/chosen": -340.37432861328125,
+      "logps/rejected": -348.7174377441406,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5099226236343384,
+      "rewards/margins": 0.3366420865058899,
+      "rewards/rejected": -0.846564769744873,
+      "step": 380
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6888102953122307e-06,
+      "logits/chosen": -1.7645511627197266,
+      "logits/rejected": -1.5222413539886475,
+      "logps/chosen": -293.2809753417969,
+      "logps/rejected": -302.76483154296875,
+      "loss": 0.586,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.5093228220939636,
+      "rewards/margins": 0.5271816253662109,
+      "rewards/rejected": -1.0365045070648193,
+      "step": 390
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.607600562872785e-06,
+      "logits/chosen": -2.1015868186950684,
+      "logits/rejected": -1.896120309829712,
+      "logps/chosen": -386.51507568359375,
+      "logps/rejected": -355.5737609863281,
+      "loss": 0.557,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6686216592788696,
+      "rewards/margins": 0.3796016275882721,
+      "rewards/rejected": -1.0482232570648193,
+      "step": 400
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": -1.8364356756210327,
+      "eval_logits/rejected": -1.743373155593872,
+      "eval_logps/chosen": -309.73779296875,
+      "eval_logps/rejected": -359.1117248535156,
+      "eval_loss": 0.5816810727119446,
+      "eval_rewards/accuracies": 0.72265625,
+      "eval_rewards/chosen": -0.5958413481712341,
+      "eval_rewards/margins": 0.47733715176582336,
+      "eval_rewards/rejected": -1.0731785297393799,
+      "eval_runtime": 100.2581,
+      "eval_samples_per_second": 19.949,
+      "eval_steps_per_second": 0.319,
+      "step": 400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5249095128531863e-06,
+      "logits/chosen": -2.0195209980010986,
+      "logits/rejected": -1.954064130783081,
+      "logps/chosen": -382.4500732421875,
+      "logps/rejected": -396.9842834472656,
+      "loss": 0.5825,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.507409393787384,
+      "rewards/margins": 0.45680397748947144,
+      "rewards/rejected": -0.964213490486145,
+      "step": 410
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4408477372034743e-06,
+      "logits/chosen": -1.6744167804718018,
+      "logits/rejected": -1.5238146781921387,
+      "logps/chosen": -277.4457702636719,
+      "logps/rejected": -303.87213134765625,
+      "loss": 0.5778,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5154739022254944,
+      "rewards/margins": 0.3824000358581543,
+      "rewards/rejected": -0.8978738784790039,
+      "step": 420
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.355527661097728e-06,
+      "logits/chosen": -1.8267688751220703,
+      "logits/rejected": -1.8169567584991455,
+      "logps/chosen": -271.2919616699219,
+      "logps/rejected": -293.12933349609375,
+      "loss": 0.5999,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4637935757637024,
+      "rewards/margins": 0.3348856568336487,
+      "rewards/rejected": -0.7986792325973511,
+      "step": 430
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.269063392575352e-06,
+      "logits/chosen": -1.7343038320541382,
+      "logits/rejected": -1.7039591073989868,
+      "logps/chosen": -298.5570983886719,
+      "logps/rejected": -324.7281799316406,
+      "loss": 0.5865,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.56922447681427,
+      "rewards/margins": 0.3157515227794647,
+      "rewards/rejected": -0.8849760293960571,
+      "step": 440
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.181570569931697e-06,
+      "logits/chosen": -1.3839043378829956,
+      "logits/rejected": -1.3726271390914917,
+      "logps/chosen": -274.1092529296875,
+      "logps/rejected": -318.86492919921875,
+      "loss": 0.578,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5215469598770142,
+      "rewards/margins": 0.42288708686828613,
+      "rewards/rejected": -0.9444340467453003,
+      "step": 450
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.09316620706208e-06,
+      "logits/chosen": -1.7264198064804077,
+      "logits/rejected": -1.616563081741333,
+      "logps/chosen": -324.797119140625,
+      "logps/rejected": -361.42303466796875,
+      "loss": 0.585,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5468829274177551,
+      "rewards/margins": 0.3461124002933502,
+      "rewards/rejected": -0.8929953575134277,
+      "step": 460
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0039685369660785e-06,
+      "logits/chosen": -1.6359647512435913,
+      "logits/rejected": -1.3307011127471924,
+      "logps/chosen": -346.19561767578125,
+      "logps/rejected": -339.08978271484375,
+      "loss": 0.5644,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6865732669830322,
+      "rewards/margins": 0.44049325585365295,
+      "rewards/rejected": -1.1270664930343628,
+      "step": 470
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.91409685362137e-06,
+      "logits/chosen": -1.1181867122650146,
+      "logits/rejected": -0.9776461720466614,
+      "logps/chosen": -293.79425048828125,
+      "logps/rejected": -316.5558166503906,
+      "loss": 0.6066,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.7594456076622009,
+      "rewards/margins": 0.25457096099853516,
+      "rewards/rejected": -1.0140166282653809,
+      "step": 480
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8236713524386085e-06,
+      "logits/chosen": -1.3658815622329712,
+      "logits/rejected": -1.1311851739883423,
+      "logps/chosen": -310.53594970703125,
+      "logps/rejected": -355.07257080078125,
+      "loss": 0.5688,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6269619464874268,
+      "rewards/margins": 0.4374374449253082,
+      "rewards/rejected": -1.0643993616104126,
+      "step": 490
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7328129695107205e-06,
+      "logits/chosen": -1.2549781799316406,
+      "logits/rejected": -0.9512443542480469,
+      "logps/chosen": -343.2566833496094,
+      "logps/rejected": -383.65936279296875,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.7175710201263428,
+      "rewards/margins": 0.569162905216217,
+      "rewards/rejected": -1.286733865737915,
+      "step": 500
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": -0.9966566562652588,
+      "eval_logits/rejected": -0.8466635942459106,
+      "eval_logps/chosen": -322.3067932128906,
+      "eval_logps/rejected": -375.8402099609375,
+      "eval_loss": 0.5679102540016174,
+      "eval_rewards/accuracies": 0.7265625,
+      "eval_rewards/chosen": -0.7215311527252197,
+      "eval_rewards/margins": 0.5189324617385864,
+      "eval_rewards/rejected": -1.2404634952545166,
+      "eval_runtime": 100.2462,
+      "eval_samples_per_second": 19.951,
+      "eval_steps_per_second": 0.319,
+      "step": 500
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.641643219871597e-06,
+      "logits/chosen": -1.133755087852478,
+      "logits/rejected": -0.6763060092926025,
+      "logps/chosen": -343.71722412109375,
+      "logps/rejected": -342.73590087890625,
+      "loss": 0.5469,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.820611834526062,
+      "rewards/margins": 0.41005903482437134,
+      "rewards/rejected": -1.2306709289550781,
+      "step": 510
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5502840349805074e-06,
+      "logits/chosen": -1.0033422708511353,
+      "logits/rejected": -0.8227261304855347,
+      "logps/chosen": -342.0159606933594,
+      "logps/rejected": -367.8998718261719,
+      "loss": 0.5467,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.6553717851638794,
+      "rewards/margins": 0.48232603073120117,
+      "rewards/rejected": -1.1376978158950806,
+      "step": 520
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4588575996495797e-06,
+      "logits/chosen": -1.015627145767212,
+      "logits/rejected": -0.7538983225822449,
+      "logps/chosen": -377.20703125,
+      "logps/rejected": -394.6551513671875,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8781946301460266,
+      "rewards/margins": 0.4648224413394928,
+      "rewards/rejected": -1.3430172204971313,
+      "step": 530
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.367486188632446e-06,
+      "logits/chosen": -1.0752605199813843,
+      "logits/rejected": -0.7782305479049683,
+      "logps/chosen": -355.3316955566406,
+      "logps/rejected": -417.9510803222656,
+      "loss": 0.5496,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.738643229007721,
+      "rewards/margins": 0.645391583442688,
+      "rewards/rejected": -1.3840347528457642,
+      "step": 540
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.276292003092593e-06,
+      "logits/chosen": -0.9742870330810547,
+      "logits/rejected": -0.5593458414077759,
+      "logps/chosen": -347.0143127441406,
+      "logps/rejected": -346.98065185546875,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7598128914833069,
+      "rewards/margins": 0.5269237756729126,
+      "rewards/rejected": -1.2867367267608643,
+      "step": 550
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1853970071701415e-06,
+      "logits/chosen": -0.7387961149215698,
+      "logits/rejected": -0.4743451476097107,
+      "logps/chosen": -338.78662109375,
+      "logps/rejected": -341.47320556640625,
+      "loss": 0.554,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.8780034780502319,
+      "rewards/margins": 0.41334277391433716,
+      "rewards/rejected": -1.2913460731506348,
+      "step": 560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0949227648656194e-06,
+      "logits/chosen": -0.6523474454879761,
+      "logits/rejected": -0.389020174741745,
+      "logps/chosen": -314.07220458984375,
+      "logps/rejected": -362.8746643066406,
+      "loss": 0.5822,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8458138704299927,
+      "rewards/margins": 0.4794779419898987,
+      "rewards/rejected": -1.3252918720245361,
+      "step": 570
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.00499027745888e-06,
+      "logits/chosen": -0.516838788986206,
+      "logits/rejected": -0.14539854228496552,
+      "logps/chosen": -334.552001953125,
+      "logps/rejected": -365.7865295410156,
+      "loss": 0.5619,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.7920180559158325,
+      "rewards/margins": 0.6189525723457336,
+      "rewards/rejected": -1.410970687866211,
+      "step": 580
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.915719821680624e-06,
+      "logits/chosen": -0.4588368833065033,
+      "logits/rejected": -0.17827937006950378,
+      "logps/chosen": -302.94464111328125,
+      "logps/rejected": -334.8900146484375,
+      "loss": 0.5691,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7475640773773193,
+      "rewards/margins": 0.3807729184627533,
+      "rewards/rejected": -1.1283371448516846,
+      "step": 590
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8272307888529276e-06,
+      "logits/chosen": -0.7035941481590271,
+      "logits/rejected": -0.5019673109054565,
+      "logps/chosen": -361.04827880859375,
+      "logps/rejected": -436.74560546875,
+      "loss": 0.5498,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.5611416101455688,
+      "rewards/margins": 0.6350690126419067,
+      "rewards/rejected": -1.1962106227874756,
+      "step": 600
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -0.44625866413116455,
+      "eval_logits/rejected": -0.25102561712265015,
+      "eval_logps/chosen": -320.17938232421875,
+      "eval_logps/rejected": -380.2698974609375,
+      "eval_loss": 0.5582141280174255,
+      "eval_rewards/accuracies": 0.7578125,
+      "eval_rewards/chosen": -0.7002571821212769,
+      "eval_rewards/margins": 0.5845023393630981,
+      "eval_rewards/rejected": -1.284759521484375,
+      "eval_runtime": 100.0013,
+      "eval_samples_per_second": 20.0,
+      "eval_steps_per_second": 0.32,
+      "step": 600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.739641525213929e-06,
+      "logits/chosen": -0.5911251306533813,
+      "logits/rejected": -0.18869177997112274,
+      "logps/chosen": -325.91033935546875,
+      "logps/rejected": -326.54095458984375,
+      "loss": 0.5425,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7186974287033081,
+      "rewards/margins": 0.4664137363433838,
+      "rewards/rejected": -1.185111165046692,
+      "step": 610
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6530691736402317e-06,
+      "logits/chosen": -0.8488653898239136,
+      "logits/rejected": -0.5028330087661743,
+      "logps/chosen": -356.1060485839844,
+      "logps/rejected": -355.0657958984375,
+      "loss": 0.5542,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8343976140022278,
+      "rewards/margins": 0.4692300260066986,
+      "rewards/rejected": -1.303627610206604,
+      "step": 620
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5676295169786864e-06,
+      "logits/chosen": -0.2444450557231903,
+      "logits/rejected": 0.13049665093421936,
+      "logps/chosen": -350.13702392578125,
+      "logps/rejected": -413.64398193359375,
+      "loss": 0.5379,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8719974756240845,
+      "rewards/margins": 0.7344092130661011,
+      "rewards/rejected": -1.606406807899475,
+      "step": 630
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4834368231970922e-06,
+      "logits/chosen": -0.38939914107322693,
+      "logits/rejected": 0.05538179352879524,
+      "logps/chosen": -322.5318298339844,
+      "logps/rejected": -368.46343994140625,
+      "loss": 0.5401,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7830222845077515,
+      "rewards/margins": 0.7185503244400024,
+      "rewards/rejected": -1.501572847366333,
+      "step": 640
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4006036925609245e-06,
+      "logits/chosen": -0.8077508807182312,
+      "logits/rejected": -0.2668865919113159,
+      "logps/chosen": -380.40252685546875,
+      "logps/rejected": -398.7056579589844,
+      "loss": 0.5383,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8857347369194031,
+      "rewards/margins": 0.5326262712478638,
+      "rewards/rejected": -1.4183608293533325,
+      "step": 650
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3192409070404582e-06,
+      "logits/chosen": -0.5546111464500427,
+      "logits/rejected": -0.17088347673416138,
+      "logps/chosen": -398.8465881347656,
+      "logps/rejected": -376.5204162597656,
+      "loss": 0.554,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9672527313232422,
+      "rewards/margins": 0.5284374952316284,
+      "rewards/rejected": -1.4956903457641602,
+      "step": 660
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2394572821496953e-06,
+      "logits/chosen": -0.39746540784835815,
+      "logits/rejected": -0.14205250144004822,
+      "logps/chosen": -365.32281494140625,
+      "logps/rejected": -412.72650146484375,
+      "loss": 0.5385,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.8937069177627563,
+      "rewards/margins": 0.5911084413528442,
+      "rewards/rejected": -1.4848153591156006,
+      "step": 670
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1613595214152713e-06,
+      "logits/chosen": -0.23002564907073975,
+      "logits/rejected": -0.18054810166358948,
+      "logps/chosen": -314.38275146484375,
+      "logps/rejected": -324.4413146972656,
+      "loss": 0.5657,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.8053790330886841,
+      "rewards/margins": 0.24868443608283997,
+      "rewards/rejected": -1.054063320159912,
+      "step": 680
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0850520736699362e-06,
+      "logits/chosen": -0.35535928606987,
+      "logits/rejected": 0.08797712624073029,
+      "logps/chosen": -279.0433349609375,
+      "logps/rejected": -323.66754150390625,
+      "loss": 0.5362,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.672313928604126,
+      "rewards/margins": 0.6532977223396301,
+      "rewards/rejected": -1.3256117105484009,
+      "step": 690
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0106369933615043e-06,
+      "logits/chosen": -0.46180295944213867,
+      "logits/rejected": 0.07596547156572342,
+      "logps/chosen": -314.94793701171875,
+      "logps/rejected": -355.2444152832031,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8018286824226379,
+      "rewards/margins": 0.6314573884010315,
+      "rewards/rejected": -1.4332860708236694,
+      "step": 700
+    },
+    {
+      "epoch": 0.73,
+      "eval_logits/chosen": -0.198812335729599,
+      "eval_logits/rejected": 0.014529339037835598,
+      "eval_logps/chosen": -334.1552734375,
+      "eval_logps/rejected": -400.80816650390625,
+      "eval_loss": 0.5489765405654907,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -0.8400164246559143,
+      "eval_rewards/margins": 0.650126576423645,
+      "eval_rewards/rejected": -1.4901429414749146,
+      "eval_runtime": 99.961,
+      "eval_samples_per_second": 20.008,
+      "eval_steps_per_second": 0.32,
+      "step": 700
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.382138040640714e-07,
+      "logits/chosen": -0.22463122010231018,
+      "logits/rejected": 0.20351561903953552,
+      "logps/chosen": -352.7989196777344,
+      "logps/rejected": -359.4533386230469,
+      "loss": 0.546,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9452872276306152,
+      "rewards/margins": 0.49813446402549744,
+      "rewards/rejected": -1.443421721458435,
+      "step": 710
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.678793653740633e-07,
+      "logits/chosen": -0.256120890378952,
+      "logits/rejected": 0.14627498388290405,
+      "logps/chosen": -387.221435546875,
+      "logps/rejected": -435.5755310058594,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.9933661222457886,
+      "rewards/margins": 0.663570761680603,
+      "rewards/rejected": -1.6569368839263916,
+      "step": 720
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.997277433690984e-07,
+      "logits/chosen": -0.48140692710876465,
+      "logits/rejected": 0.015109086409211159,
+      "logps/chosen": -395.16815185546875,
+      "logps/rejected": -368.5047302246094,
+      "loss": 0.5629,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8996418714523315,
+      "rewards/margins": 0.48376113176345825,
+      "rewards/rejected": -1.3834030628204346,
+      "step": 730
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.338500848029603e-07,
+      "logits/chosen": -0.3346256911754608,
+      "logits/rejected": 0.1683504283428192,
+      "logps/chosen": -410.95672607421875,
+      "logps/rejected": -427.506591796875,
+      "loss": 0.5523,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.8928782343864441,
+      "rewards/margins": 0.6983044147491455,
+      "rewards/rejected": -1.5911829471588135,
+      "step": 740
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.70334495204884e-07,
+      "logits/chosen": -0.5112616419792175,
+      "logits/rejected": -0.16424962878227234,
+      "logps/chosen": -346.962890625,
+      "logps/rejected": -423.4779357910156,
+      "loss": 0.5376,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8174928426742554,
+      "rewards/margins": 0.6463109850883484,
+      "rewards/rejected": -1.4638036489486694,
+      "step": 750
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.092659210462232e-07,
+      "logits/chosen": -0.33304744958877563,
+      "logits/rejected": -0.1934703141450882,
+      "logps/chosen": -360.96124267578125,
+      "logps/rejected": -388.1688537597656,
+      "loss": 0.5219,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8719509840011597,
+      "rewards/margins": 0.5291545987129211,
+      "rewards/rejected": -1.4011056423187256,
+      "step": 760
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.507260361320738e-07,
+      "logits/chosen": -0.3076072931289673,
+      "logits/rejected": -0.03598857671022415,
+      "logps/chosen": -417.52679443359375,
+      "logps/rejected": -437.1398010253906,
+      "loss": 0.5557,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.8676769137382507,
+      "rewards/margins": 0.46329760551452637,
+      "rewards/rejected": -1.3309745788574219,
+      "step": 770
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.947931323697983e-07,
+      "logits/chosen": 0.10995174944400787,
+      "logits/rejected": 0.25233790278434753,
+      "logps/chosen": -298.9948425292969,
+      "logps/rejected": -337.920166015625,
+      "loss": 0.5422,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9167734980583191,
+      "rewards/margins": 0.42617493867874146,
+      "rewards/rejected": -1.3429481983184814,
+      "step": 780
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.4154201506053985e-07,
+      "logits/chosen": -0.15819688141345978,
+      "logits/rejected": -0.1625043898820877,
+      "logps/chosen": -366.3085632324219,
+      "logps/rejected": -413.33343505859375,
+      "loss": 0.5391,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.967660129070282,
+      "rewards/margins": 0.4664042592048645,
+      "rewards/rejected": -1.434064507484436,
+      "step": 790
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.910439028537638e-07,
+      "logits/chosen": -0.2914949953556061,
+      "logits/rejected": 0.14831864833831787,
+      "logps/chosen": -412.54827880859375,
+      "logps/rejected": -403.07647705078125,
+      "loss": 0.5264,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9057921171188354,
+      "rewards/margins": 0.6066713333129883,
+      "rewards/rejected": -1.5124633312225342,
+      "step": 800
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -0.15486477315425873,
+      "eval_logits/rejected": 0.06035802513360977,
+      "eval_logps/chosen": -336.28326416015625,
+      "eval_logps/rejected": -404.0750732421875,
+      "eval_loss": 0.5475199818611145,
+      "eval_rewards/accuracies": 0.74609375,
+      "eval_rewards/chosen": -0.8612961769104004,
+      "eval_rewards/margins": 0.6615157127380371,
+      "eval_rewards/rejected": -1.5228118896484375,
+      "eval_runtime": 100.0077,
+      "eval_samples_per_second": 19.998,
+      "eval_steps_per_second": 0.32,
+      "step": 800
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.4336633249862084e-07,
+      "logits/chosen": -0.3513889014720917,
+      "logits/rejected": 0.16097551584243774,
+      "logps/chosen": -367.8076477050781,
+      "logps/rejected": -368.91473388671875,
+      "loss": 0.5401,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0032860040664673,
+      "rewards/margins": 0.39093494415283203,
+      "rewards/rejected": -1.3942210674285889,
+      "step": 810
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.98573068519539e-07,
+      "logits/chosen": -0.27493131160736084,
+      "logits/rejected": 0.1548646241426468,
+      "logps/chosen": -365.89007568359375,
+      "logps/rejected": -377.8901672363281,
+      "loss": 0.5636,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.8509780764579773,
+      "rewards/margins": 0.6635745167732239,
+      "rewards/rejected": -1.5145527124404907,
+      "step": 820
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5672401793681854e-07,
+      "logits/chosen": -0.1434798389673233,
+      "logits/rejected": -0.3032146990299225,
+      "logps/chosen": -335.52349853515625,
+      "logps/rejected": -418.47979736328125,
+      "loss": 0.5174,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9104019999504089,
+      "rewards/margins": 0.5144548416137695,
+      "rewards/rejected": -1.4248569011688232,
+      "step": 830
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.178751501463036e-07,
+      "logits/chosen": -0.28891128301620483,
+      "logits/rejected": -0.015888934955000877,
+      "logps/chosen": -357.6486511230469,
+      "logps/rejected": -387.8254089355469,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.9722326993942261,
+      "rewards/margins": 0.40555062890052795,
+      "rewards/rejected": -1.3777834177017212,
+      "step": 840
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.820784220652766e-07,
+      "logits/chosen": -0.457050621509552,
+      "logits/rejected": -0.08179013431072235,
+      "logps/chosen": -365.02935791015625,
+      "logps/rejected": -377.90484619140625,
+      "loss": 0.5464,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.8087032437324524,
+      "rewards/margins": 0.6943231821060181,
+      "rewards/rejected": -1.5030266046524048,
+      "step": 850
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4938170864468636e-07,
+      "logits/chosen": -0.2849724292755127,
+      "logits/rejected": -0.0881032794713974,
+      "logps/chosen": -353.9926452636719,
+      "logps/rejected": -402.39886474609375,
+      "loss": 0.551,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8892512321472168,
+      "rewards/margins": 0.6870072484016418,
+      "rewards/rejected": -1.5762584209442139,
+      "step": 860
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1982873884064466e-07,
+      "logits/chosen": -0.2418334186077118,
+      "logits/rejected": -0.05195746570825577,
+      "logps/chosen": -314.6041564941406,
+      "logps/rejected": -390.5324401855469,
+      "loss": 0.5528,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.6465216279029846,
+      "rewards/margins": 0.8042505383491516,
+      "rewards/rejected": -1.4507721662521362,
+      "step": 870
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.345903713082305e-08,
+      "logits/chosen": -0.3975564241409302,
+      "logits/rejected": -0.0711708664894104,
+      "logps/chosen": -384.4748229980469,
+      "logps/rejected": -408.9742126464844,
+      "loss": 0.5344,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9061468243598938,
+      "rewards/margins": 0.4584124684333801,
+      "rewards/rejected": -1.3645591735839844,
+      "step": 880
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.030787065396866e-08,
+      "logits/chosen": -0.34069642424583435,
+      "logits/rejected": -0.19395461678504944,
+      "logps/chosen": -320.58380126953125,
+      "logps/rejected": -402.9022216796875,
+      "loss": 0.5508,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.795324981212616,
+      "rewards/margins": 0.6657498478889465,
+      "rewards/rejected": -1.461074948310852,
+      "step": 890
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.0406202043228604e-08,
+      "logits/chosen": -0.0745534598827362,
+      "logits/rejected": 0.35236138105392456,
+      "logps/chosen": -382.5845642089844,
+      "logps/rejected": -400.34735107421875,
+      "loss": 0.5639,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.9554711580276489,
+      "rewards/margins": 0.41256728768348694,
+      "rewards/rejected": -1.3680384159088135,
+      "step": 900
+    },
+    {
+      "epoch": 0.94,
+      "eval_logits/chosen": -0.14664390683174133,
+      "eval_logits/rejected": 0.07037673145532608,
+      "eval_logps/chosen": -336.4347839355469,
+      "eval_logps/rejected": -404.4688415527344,
+      "eval_loss": 0.5475045442581177,
+      "eval_rewards/accuracies": 0.7421875,
+      "eval_rewards/chosen": -0.8628111481666565,
+      "eval_rewards/margins": 0.6639385223388672,
+      "eval_rewards/rejected": -1.5267497301101685,
+      "eval_runtime": 99.9101,
+      "eval_samples_per_second": 20.018,
+      "eval_steps_per_second": 0.32,
+      "step": 900
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.378064801637687e-08,
+      "logits/chosen": -0.18566574156284332,
+      "logits/rejected": 0.08027581125497818,
+      "logps/chosen": -338.36224365234375,
+      "logps/rejected": -387.63946533203125,
+      "loss": 0.5317,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8497447967529297,
+      "rewards/margins": 0.5736583471298218,
+      "rewards/rejected": -1.423403263092041,
+      "step": 910
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0453443778310766e-08,
+      "logits/chosen": -0.29515573382377625,
+      "logits/rejected": 0.24348752200603485,
+      "logps/chosen": -364.8608703613281,
+      "logps/rejected": -390.37200927734375,
+      "loss": 0.5321,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.9079607725143433,
+      "rewards/margins": 0.5625349283218384,
+      "rewards/rejected": -1.470495581626892,
+      "step": 920
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0442413283435759e-08,
+      "logits/chosen": -0.34043288230895996,
+      "logits/rejected": -0.010392585769295692,
+      "logps/chosen": -288.91424560546875,
+      "logps/rejected": -375.74945068359375,
+      "loss": 0.51,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.7094988226890564,
+      "rewards/margins": 0.8463606834411621,
+      "rewards/rejected": -1.5558595657348633,
+      "step": 930
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.760945397705828e-09,
+      "logits/chosen": -0.4917779862880707,
+      "logits/rejected": -0.014173048548400402,
+      "logps/chosen": -391.59149169921875,
+      "logps/rejected": -432.360107421875,
+      "loss": 0.5395,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.9179250597953796,
+      "rewards/margins": 0.5526332855224609,
+      "rewards/rejected": -1.4705584049224854,
+      "step": 940
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.1797599220405605e-10,
+      "logits/chosen": -0.1975761204957962,
+      "logits/rejected": 0.10586023330688477,
+      "logps/chosen": -334.07745361328125,
+      "logps/rejected": -401.03680419921875,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.9642227292060852,
+      "rewards/margins": 0.754417359828949,
+      "rewards/rejected": -1.7186399698257446,
+      "step": 950
+    },
+    {
+      "epoch": 1.0,
+      "step": 955,
+      "total_flos": 0.0,
+      "train_loss": 0.583915277301329,
+      "train_runtime": 6210.8046,
+      "train_samples_per_second": 9.843,
+      "train_steps_per_second": 0.154
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 955,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:237a8724e74a1bf1aeb400498b58df3e4be6c3bd6a8d522767c8c3ac48b8709a
+size 4856