Model save

README.md

@@ -0,0 +1,73 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- generated_from_trainer
+model-index:
+- name: sambar-7b-dpo-lora
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# sambar-7b-dpo-lora
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5747
+- Rewards/chosen: -0.0141
+- Rewards/rejected: -0.4147
+- Rewards/accuracies: 0.7060
+- Rewards/margins: 0.4006
+- Logps/rejected: -221.3069
+- Logps/chosen: -263.0773
+- Logits/rejected: -2.1478
+- Logits/chosen: -2.2594
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 32
+- total_train_batch_size: 256
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6213        | 1.0   | 242  | 0.6182          | 0.0426         | -0.1569          | 0.6860             | 0.1995          | -218.7293      | -262.5110    | -2.1605         | -2.2727       |
+| 0.5903        | 2.0   | 484  | 0.5826          | 0.0046         | -0.3500          | 0.6940             | 0.3546          | -220.6603      | -262.8906    | -2.1517         | -2.2634       |
+| 0.5743        | 3.0   | 726  | 0.5747          | -0.0141        | -0.4147          | 0.7060             | 0.4006          | -221.3069      | -263.0773    | -2.1478         | -2.2594       |
+
+
+### Framework versions
+
+- Transformers 4.35.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.14.1

adapter_config.json

@@ -0,0 +1,25 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "o_proj",
+    "v_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8213aa18221d9621f8acf465fae81941547afc8f863ba01eb0c3b90f44c7a05
+size 218138576

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -2.259361505508423,
+    "eval_logits/rejected": -2.14780330657959,
+    "eval_logps/chosen": -263.0772705078125,
+    "eval_logps/rejected": -221.30685424804688,
+    "eval_loss": 0.5746620893478394,
+    "eval_rewards/accuracies": 0.7059999704360962,
+    "eval_rewards/chosen": -0.014065464027225971,
+    "eval_rewards/margins": 0.4006173312664032,
+    "eval_rewards/rejected": -0.41468286514282227,
+    "eval_runtime": 237.4099,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 8.424,
+    "eval_steps_per_second": 0.527,
+    "train_loss": 0.6110922341996973,
+    "train_runtime": 33040.6759,
+    "train_samples": 61966,
+    "train_samples_per_second": 5.626,
+    "train_steps_per_second": 0.022
+}

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -2.259361505508423,
+    "eval_logits/rejected": -2.14780330657959,
+    "eval_logps/chosen": -263.0772705078125,
+    "eval_logps/rejected": -221.30685424804688,
+    "eval_loss": 0.5746620893478394,
+    "eval_rewards/accuracies": 0.7059999704360962,
+    "eval_rewards/chosen": -0.014065464027225971,
+    "eval_rewards/margins": 0.4006173312664032,
+    "eval_rewards/rejected": -0.41468286514282227,
+    "eval_runtime": 237.4099,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 8.424,
+    "eval_steps_per_second": 0.527
+}

runs/Jan02_22-07-01_node-0/events.out.tfevents.1704262093.node-0.91323.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d52f905ed0ff120b4f4826276a1830bd910e302635f52a86a19a3d94e4e483a
+size 53035

runs/Jan02_22-07-01_node-0/events.out.tfevents.1704295373.node-0.91323.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74a2baa2923367618c442e4eaf1575001facd4f31cdff37bdef496979b24fd87
+size 828

runs/Jan03_09-50-50_node-0/events.out.tfevents.1704304326.node-0.177510.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fe5aff3b80f7b55c3e54be8fa5588a36a71264ef0c1ce8ad9090daa2c016ea1
+size 53035

runs/Jan03_09-50-50_node-0/events.out.tfevents.1704337604.node-0.177510.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c90567cbfcdfa37c1ab8a67d8a7f63c072d58f918c1efedce700e1f1ed7cb62e
+size 828

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,41 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.6110922341996973,
+    "train_runtime": 33040.6759,
+    "train_samples": 61966,
+    "train_samples_per_second": 5.626,
+    "train_steps_per_second": 0.022
+}

trainer_state.json

@@ -0,0 +1,1098 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9992254066615027,
+  "eval_steps": 100,
+  "global_step": 726,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.84931506849315e-09,
+      "logits/chosen": -2.445258378982544,
+      "logits/rejected": -2.482508659362793,
+      "logps/chosen": -270.6954040527344,
+      "logps/rejected": -206.76272583007812,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 6.84931506849315e-08,
+      "logits/chosen": -2.507073163986206,
+      "logits/rejected": -2.4275779724121094,
+      "logps/chosen": -291.61688232421875,
+      "logps/rejected": -224.1806182861328,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.4322916567325592,
+      "rewards/chosen": 0.0008433780749328434,
+      "rewards/margins": -0.0009758697124198079,
+      "rewards/rejected": 0.0018192478455603123,
+      "step": 10
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.36986301369863e-07,
+      "logits/chosen": -2.517071008682251,
+      "logits/rejected": -2.4758286476135254,
+      "logps/chosen": -276.9480285644531,
+      "logps/rejected": -214.7642822265625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.49687498807907104,
+      "rewards/chosen": -0.00105036492459476,
+      "rewards/margins": 0.001187985180877149,
+      "rewards/rejected": -0.0022383497562259436,
+      "step": 20
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 2.054794520547945e-07,
+      "logits/chosen": -2.481997013092041,
+      "logits/rejected": -2.4269680976867676,
+      "logps/chosen": -251.0941162109375,
+      "logps/rejected": -205.1868438720703,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.4781250059604645,
+      "rewards/chosen": 0.0005288544343784451,
+      "rewards/margins": 0.0002331261057406664,
+      "rewards/rejected": 0.0002957289107143879,
+      "step": 30
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 2.73972602739726e-07,
+      "logits/chosen": -2.548166036605835,
+      "logits/rejected": -2.4914188385009766,
+      "logps/chosen": -281.9214782714844,
+      "logps/rejected": -214.442626953125,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.520312488079071,
+      "rewards/chosen": 0.002253689104691148,
+      "rewards/margins": 0.003704611212015152,
+      "rewards/rejected": -0.0014509217580780387,
+      "step": 40
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 3.424657534246575e-07,
+      "logits/chosen": -2.5232255458831787,
+      "logits/rejected": -2.47584867477417,
+      "logps/chosen": -265.45989990234375,
+      "logps/rejected": -221.6902313232422,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5218750238418579,
+      "rewards/chosen": -0.0004125732812099159,
+      "rewards/margins": 0.001857149414718151,
+      "rewards/rejected": -0.0022697225213050842,
+      "step": 50
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.10958904109589e-07,
+      "logits/chosen": -2.503446578979492,
+      "logits/rejected": -2.4641902446746826,
+      "logps/chosen": -265.10125732421875,
+      "logps/rejected": -212.9536590576172,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.5765625238418579,
+      "rewards/chosen": 0.0057233949191868305,
+      "rewards/margins": 0.011815843172371387,
+      "rewards/rejected": -0.006092446856200695,
+      "step": 60
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.794520547945205e-07,
+      "logits/chosen": -2.4819159507751465,
+      "logits/rejected": -2.4763991832733154,
+      "logps/chosen": -252.2348175048828,
+      "logps/rejected": -212.4091339111328,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.582812488079071,
+      "rewards/chosen": 0.007895523682236671,
+      "rewards/margins": 0.018240801990032196,
+      "rewards/rejected": -0.01034527737647295,
+      "step": 70
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.946401225114854e-07,
+      "logits/chosen": -2.5163493156433105,
+      "logits/rejected": -2.4587182998657227,
+      "logps/chosen": -263.52581787109375,
+      "logps/rejected": -216.7571258544922,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.01336698792874813,
+      "rewards/margins": 0.022466376423835754,
+      "rewards/rejected": -0.009099386632442474,
+      "step": 80
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.869831546707504e-07,
+      "logits/chosen": -2.559368848800659,
+      "logits/rejected": -2.5,
+      "logps/chosen": -269.92730712890625,
+      "logps/rejected": -222.235107421875,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": 0.019632169976830482,
+      "rewards/margins": 0.03590407967567444,
+      "rewards/rejected": -0.016271911561489105,
+      "step": 90
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 4.793261868300153e-07,
+      "logits/chosen": -2.5028889179229736,
+      "logits/rejected": -2.4726669788360596,
+      "logps/chosen": -272.01385498046875,
+      "logps/rejected": -225.379150390625,
+      "loss": 0.6748,
+      "rewards/accuracies": 0.651562511920929,
+      "rewards/chosen": 0.024208087474107742,
+      "rewards/margins": 0.0442068949341774,
+      "rewards/rejected": -0.019998803734779358,
+      "step": 100
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.7166921898928023e-07,
+      "logits/chosen": -2.539785623550415,
+      "logits/rejected": -2.4715189933776855,
+      "logps/chosen": -250.6864776611328,
+      "logps/rejected": -220.31857299804688,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.651562511920929,
+      "rewards/chosen": 0.018719878047704697,
+      "rewards/margins": 0.04420315474271774,
+      "rewards/rejected": -0.025483276695013046,
+      "step": 110
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 4.640122511485451e-07,
+      "logits/chosen": -2.5032382011413574,
+      "logits/rejected": -2.4787399768829346,
+      "logps/chosen": -255.07101440429688,
+      "logps/rejected": -217.56332397460938,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.651562511920929,
+      "rewards/chosen": 0.02690967358648777,
+      "rewards/margins": 0.0577348992228508,
+      "rewards/rejected": -0.030825233086943626,
+      "step": 120
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.563552833078101e-07,
+      "logits/chosen": -2.5442593097686768,
+      "logits/rejected": -2.4793853759765625,
+      "logps/chosen": -264.15234375,
+      "logps/rejected": -223.4934844970703,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": 0.029823919758200645,
+      "rewards/margins": 0.07180732488632202,
+      "rewards/rejected": -0.041983410716056824,
+      "step": 130
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.4869831546707505e-07,
+      "logits/chosen": -2.5460541248321533,
+      "logits/rejected": -2.4870338439941406,
+      "logps/chosen": -268.638427734375,
+      "logps/rejected": -227.9025421142578,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.6734374761581421,
+      "rewards/chosen": 0.041604138910770416,
+      "rewards/margins": 0.09325676411390305,
+      "rewards/rejected": -0.05165262892842293,
+      "step": 140
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 4.4104134762633994e-07,
+      "logits/chosen": -2.5670578479766846,
+      "logits/rejected": -2.5104963779449463,
+      "logps/chosen": -264.26611328125,
+      "logps/rejected": -224.1782684326172,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.692187488079071,
+      "rewards/chosen": 0.03908708691596985,
+      "rewards/margins": 0.09105747193098068,
+      "rewards/rejected": -0.05197037383913994,
+      "step": 150
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 4.333843797856049e-07,
+      "logits/chosen": -2.5320677757263184,
+      "logits/rejected": -2.4734749794006348,
+      "logps/chosen": -277.51861572265625,
+      "logps/rejected": -236.5709991455078,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.04750001057982445,
+      "rewards/margins": 0.1084158793091774,
+      "rewards/rejected": -0.06091586500406265,
+      "step": 160
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.257274119448698e-07,
+      "logits/chosen": -2.565639019012451,
+      "logits/rejected": -2.530691146850586,
+      "logps/chosen": -268.9267578125,
+      "logps/rejected": -236.61929321289062,
+      "loss": 0.6453,
+      "rewards/accuracies": 0.6703125238418579,
+      "rewards/chosen": 0.03782298043370247,
+      "rewards/margins": 0.10148061811923981,
+      "rewards/rejected": -0.06365764141082764,
+      "step": 170
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 4.180704441041347e-07,
+      "logits/chosen": -2.522090196609497,
+      "logits/rejected": -2.470818519592285,
+      "logps/chosen": -254.6784210205078,
+      "logps/rejected": -223.4536590576172,
+      "loss": 0.6455,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.03428062051534653,
+      "rewards/margins": 0.1071067601442337,
+      "rewards/rejected": -0.07282613217830658,
+      "step": 180
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.1041347626339966e-07,
+      "logits/chosen": -2.5339770317077637,
+      "logits/rejected": -2.485583543777466,
+      "logps/chosen": -259.6706237792969,
+      "logps/rejected": -210.58071899414062,
+      "loss": 0.6367,
+      "rewards/accuracies": 0.6703125238418579,
+      "rewards/chosen": 0.04104261472821236,
+      "rewards/margins": 0.13799390196800232,
+      "rewards/rejected": -0.09695132076740265,
+      "step": 190
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.027565084226646e-07,
+      "logits/chosen": -2.559802770614624,
+      "logits/rejected": -2.4930777549743652,
+      "logps/chosen": -260.19580078125,
+      "logps/rejected": -215.35610961914062,
+      "loss": 0.6291,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": 0.04894023388624191,
+      "rewards/margins": 0.15646891295909882,
+      "rewards/rejected": -0.1075286716222763,
+      "step": 200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 3.9509954058192954e-07,
+      "logits/chosen": -2.5613999366760254,
+      "logits/rejected": -2.517932653427124,
+      "logps/chosen": -281.9465026855469,
+      "logps/rejected": -228.8275909423828,
+      "loss": 0.6252,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": 0.05419199541211128,
+      "rewards/margins": 0.1751035749912262,
+      "rewards/rejected": -0.12091157585382462,
+      "step": 210
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.874425727411945e-07,
+      "logits/chosen": -2.5146591663360596,
+      "logits/rejected": -2.488956928253174,
+      "logps/chosen": -274.73712158203125,
+      "logps/rejected": -226.3198699951172,
+      "loss": 0.6242,
+      "rewards/accuracies": 0.6859375238418579,
+      "rewards/chosen": 0.04880619794130325,
+      "rewards/margins": 0.178890660405159,
+      "rewards/rejected": -0.13008446991443634,
+      "step": 220
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.797856049004594e-07,
+      "logits/chosen": -2.5332489013671875,
+      "logits/rejected": -2.4790148735046387,
+      "logps/chosen": -262.056884765625,
+      "logps/rejected": -225.40811157226562,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.052485816180706024,
+      "rewards/margins": 0.17873048782348633,
+      "rewards/rejected": -0.1262446939945221,
+      "step": 230
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.7212863705972436e-07,
+      "logits/chosen": -2.5549776554107666,
+      "logits/rejected": -2.506992816925049,
+      "logps/chosen": -269.25286865234375,
+      "logps/rejected": -219.7020721435547,
+      "loss": 0.6213,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.053668104112148285,
+      "rewards/margins": 0.18163269758224487,
+      "rewards/rejected": -0.1279645711183548,
+      "step": 240
+    },
+    {
+      "epoch": 1.0,
+      "eval_logits/chosen": -2.2727341651916504,
+      "eval_logits/rejected": -2.160538911819458,
+      "eval_logps/chosen": -262.5110168457031,
+      "eval_logps/rejected": -218.72930908203125,
+      "eval_loss": 0.618248462677002,
+      "eval_rewards/accuracies": 0.6859999895095825,
+      "eval_rewards/chosen": 0.04256003722548485,
+      "eval_rewards/margins": 0.1994897723197937,
+      "eval_rewards/rejected": -0.15692974627017975,
+      "eval_runtime": 239.4377,
+      "eval_samples_per_second": 8.353,
+      "eval_steps_per_second": 0.522,
+      "step": 242
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 3.6447166921898925e-07,
+      "logits/chosen": -2.489406108856201,
+      "logits/rejected": -2.4407076835632324,
+      "logps/chosen": -255.3499298095703,
+      "logps/rejected": -207.3684539794922,
+      "loss": 0.6244,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": 0.039003290235996246,
+      "rewards/margins": 0.18538300693035126,
+      "rewards/rejected": -0.1463797241449356,
+      "step": 250
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 3.568147013782542e-07,
+      "logits/chosen": -2.511629581451416,
+      "logits/rejected": -2.4620814323425293,
+      "logps/chosen": -260.0677490234375,
+      "logps/rejected": -223.49169921875,
+      "loss": 0.6101,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": 0.04925510287284851,
+      "rewards/margins": 0.21666565537452698,
+      "rewards/rejected": -0.16741053760051727,
+      "step": 260
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 3.4915773353751913e-07,
+      "logits/chosen": -2.5199971199035645,
+      "logits/rejected": -2.4499754905700684,
+      "logps/chosen": -276.2830505371094,
+      "logps/rejected": -233.03201293945312,
+      "loss": 0.6031,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.06275217235088348,
+      "rewards/margins": 0.2755245864391327,
+      "rewards/rejected": -0.2127724140882492,
+      "step": 270
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 3.41500765696784e-07,
+      "logits/chosen": -2.527909517288208,
+      "logits/rejected": -2.4822728633880615,
+      "logps/chosen": -249.1805419921875,
+      "logps/rejected": -222.91024780273438,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.04066196084022522,
+      "rewards/margins": 0.20734688639640808,
+      "rewards/rejected": -0.16668489575386047,
+      "step": 280
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 3.33843797856049e-07,
+      "logits/chosen": -2.506227731704712,
+      "logits/rejected": -2.483896493911743,
+      "logps/chosen": -281.0603332519531,
+      "logps/rejected": -225.3281707763672,
+      "loss": 0.6033,
+      "rewards/accuracies": 0.729687511920929,
+      "rewards/chosen": 0.04976457357406616,
+      "rewards/margins": 0.2599830627441406,
+      "rewards/rejected": -0.21021847426891327,
+      "step": 290
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 3.2618683001531396e-07,
+      "logits/chosen": -2.4969980716705322,
+      "logits/rejected": -2.4238204956054688,
+      "logps/chosen": -259.1755065917969,
+      "logps/rejected": -224.92581176757812,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.0504862442612648,
+      "rewards/margins": 0.24086256325244904,
+      "rewards/rejected": -0.19037629663944244,
+      "step": 300
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 3.1852986217457885e-07,
+      "logits/chosen": -2.5485026836395264,
+      "logits/rejected": -2.4636754989624023,
+      "logps/chosen": -259.2119140625,
+      "logps/rejected": -225.5343017578125,
+      "loss": 0.6058,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": 0.04171649366617203,
+      "rewards/margins": 0.2371227741241455,
+      "rewards/rejected": -0.19540627300739288,
+      "step": 310
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 3.108728943338438e-07,
+      "logits/chosen": -2.5411689281463623,
+      "logits/rejected": -2.5233051776885986,
+      "logps/chosen": -250.4091796875,
+      "logps/rejected": -234.99911499023438,
+      "loss": 0.6045,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": 0.029940366744995117,
+      "rewards/margins": 0.20695392787456512,
+      "rewards/rejected": -0.17701356112957,
+      "step": 320
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 3.0321592649310873e-07,
+      "logits/chosen": -2.518437385559082,
+      "logits/rejected": -2.4455406665802,
+      "logps/chosen": -251.207275390625,
+      "logps/rejected": -221.4502410888672,
+      "loss": 0.5953,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": 0.05027127265930176,
+      "rewards/margins": 0.2602222263813019,
+      "rewards/rejected": -0.20995095372200012,
+      "step": 330
+    },
+    {
+      "epoch": 1.4,
+      "learning_rate": 2.955589586523736e-07,
+      "logits/chosen": -2.536722183227539,
+      "logits/rejected": -2.4708170890808105,
+      "logps/chosen": -263.5827941894531,
+      "logps/rejected": -223.34375,
+      "loss": 0.6006,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.0343378521502018,
+      "rewards/margins": 0.2729324400424957,
+      "rewards/rejected": -0.23859457671642303,
+      "step": 340
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 2.8790199081163856e-07,
+      "logits/chosen": -2.5344901084899902,
+      "logits/rejected": -2.457329511642456,
+      "logps/chosen": -263.54827880859375,
+      "logps/rejected": -222.186767578125,
+      "loss": 0.6012,
+      "rewards/accuracies": 0.692187488079071,
+      "rewards/chosen": 0.02179412916302681,
+      "rewards/margins": 0.24006810784339905,
+      "rewards/rejected": -0.21827396750450134,
+      "step": 350
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 2.802450229709035e-07,
+      "logits/chosen": -2.499368190765381,
+      "logits/rejected": -2.4326324462890625,
+      "logps/chosen": -252.73818969726562,
+      "logps/rejected": -210.35055541992188,
+      "loss": 0.5947,
+      "rewards/accuracies": 0.6890624761581421,
+      "rewards/chosen": 0.017355820164084435,
+      "rewards/margins": 0.2538844645023346,
+      "rewards/rejected": -0.236528679728508,
+      "step": 360
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 2.725880551301684e-07,
+      "logits/chosen": -2.5512120723724365,
+      "logits/rejected": -2.491529941558838,
+      "logps/chosen": -272.677978515625,
+      "logps/rejected": -230.0671844482422,
+      "loss": 0.6009,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.02877696417272091,
+      "rewards/margins": 0.27019625902175903,
+      "rewards/rejected": -0.24141927063465118,
+      "step": 370
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 2.649310872894334e-07,
+      "logits/chosen": -2.5440518856048584,
+      "logits/rejected": -2.498530149459839,
+      "logps/chosen": -272.7989196777344,
+      "logps/rejected": -219.9661865234375,
+      "loss": 0.5936,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.0328015573322773,
+      "rewards/margins": 0.30769142508506775,
+      "rewards/rejected": -0.27488988637924194,
+      "step": 380
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 2.572741194486983e-07,
+      "logits/chosen": -2.5445454120635986,
+      "logits/rejected": -2.478445529937744,
+      "logps/chosen": -271.42205810546875,
+      "logps/rejected": -205.12210083007812,
+      "loss": 0.5876,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": 0.02636413648724556,
+      "rewards/margins": 0.3068538010120392,
+      "rewards/rejected": -0.28048965334892273,
+      "step": 390
+    },
+    {
+      "epoch": 1.65,
+      "learning_rate": 2.496171516079632e-07,
+      "logits/chosen": -2.565645217895508,
+      "logits/rejected": -2.490952491760254,
+      "logps/chosen": -290.6482849121094,
+      "logps/rejected": -236.2904510498047,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": 0.05599096417427063,
+      "rewards/margins": 0.34153956174850464,
+      "rewards/rejected": -0.2855486273765564,
+      "step": 400
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 2.4196018376722816e-07,
+      "logits/chosen": -2.52652645111084,
+      "logits/rejected": -2.4913227558135986,
+      "logps/chosen": -276.25018310546875,
+      "logps/rejected": -225.22946166992188,
+      "loss": 0.5948,
+      "rewards/accuracies": 0.682812511920929,
+      "rewards/chosen": 0.0320248007774353,
+      "rewards/margins": 0.31556203961372375,
+      "rewards/rejected": -0.28353720903396606,
+      "step": 410
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 2.343032159264931e-07,
+      "logits/chosen": -2.500330686569214,
+      "logits/rejected": -2.426058530807495,
+      "logps/chosen": -266.7002868652344,
+      "logps/rejected": -220.94320678710938,
+      "loss": 0.5856,
+      "rewards/accuracies": 0.723437488079071,
+      "rewards/chosen": 0.03841843083500862,
+      "rewards/margins": 0.3689562678337097,
+      "rewards/rejected": -0.3305378556251526,
+      "step": 420
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 2.26646248085758e-07,
+      "logits/chosen": -2.489319324493408,
+      "logits/rejected": -2.429903745651245,
+      "logps/chosen": -257.18353271484375,
+      "logps/rejected": -214.43701171875,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.027863550931215286,
+      "rewards/margins": 0.3603667616844177,
+      "rewards/rejected": -0.3325032591819763,
+      "step": 430
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 2.1898928024502298e-07,
+      "logits/chosen": -2.546457290649414,
+      "logits/rejected": -2.479651927947998,
+      "logps/chosen": -261.133544921875,
+      "logps/rejected": -218.47610473632812,
+      "loss": 0.5824,
+      "rewards/accuracies": 0.698437511920929,
+      "rewards/chosen": 0.03679219260811806,
+      "rewards/margins": 0.3415088653564453,
+      "rewards/rejected": -0.30471667647361755,
+      "step": 440
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 2.113323124042879e-07,
+      "logits/chosen": -2.5231552124023438,
+      "logits/rejected": -2.479872465133667,
+      "logps/chosen": -268.78851318359375,
+      "logps/rejected": -228.0364990234375,
+      "loss": 0.5827,
+      "rewards/accuracies": 0.7203124761581421,
+      "rewards/chosen": 0.044063158333301544,
+      "rewards/margins": 0.36794179677963257,
+      "rewards/rejected": -0.32387861609458923,
+      "step": 450
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 2.036753445635528e-07,
+      "logits/chosen": -2.530987024307251,
+      "logits/rejected": -2.5036771297454834,
+      "logps/chosen": -281.70513916015625,
+      "logps/rejected": -234.73934936523438,
+      "loss": 0.5788,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": 0.032568901777267456,
+      "rewards/margins": 0.3853657841682434,
+      "rewards/rejected": -0.35279688239097595,
+      "step": 460
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 1.9601837672281775e-07,
+      "logits/chosen": -2.5145978927612305,
+      "logits/rejected": -2.474416971206665,
+      "logps/chosen": -267.7714538574219,
+      "logps/rejected": -233.2850341796875,
+      "loss": 0.5832,
+      "rewards/accuracies": 0.7328125238418579,
+      "rewards/chosen": 0.012133514508605003,
+      "rewards/margins": 0.3285755217075348,
+      "rewards/rejected": -0.31644195318222046,
+      "step": 470
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 1.883614088820827e-07,
+      "logits/chosen": -2.534013032913208,
+      "logits/rejected": -2.479665517807007,
+      "logps/chosen": -266.11285400390625,
+      "logps/rejected": -221.65478515625,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": 0.015116107650101185,
+      "rewards/margins": 0.34223589301109314,
+      "rewards/rejected": -0.32711976766586304,
+      "step": 480
+    },
+    {
+      "epoch": 2.0,
+      "eval_logits/chosen": -2.2634127140045166,
+      "eval_logits/rejected": -2.151700496673584,
+      "eval_logps/chosen": -262.890625,
+      "eval_logps/rejected": -220.66033935546875,
+      "eval_loss": 0.5826324224472046,
+      "eval_rewards/accuracies": 0.6940000057220459,
+      "eval_rewards/chosen": 0.004602876491844654,
+      "eval_rewards/margins": 0.35463690757751465,
+      "eval_rewards/rejected": -0.3500339984893799,
+      "eval_runtime": 238.0856,
+      "eval_samples_per_second": 8.4,
+      "eval_steps_per_second": 0.525,
+      "step": 484
+    },
+    {
+      "epoch": 2.02,
+      "learning_rate": 1.807044410413476e-07,
+      "logits/chosen": -2.509183645248413,
+      "logits/rejected": -2.474013566970825,
+      "logps/chosen": -255.69241333007812,
+      "logps/rejected": -233.1560516357422,
+      "loss": 0.586,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": 0.010258705355226994,
+      "rewards/margins": 0.35224205255508423,
+      "rewards/rejected": -0.3419833481311798,
+      "step": 490
+    },
+    {
+      "epoch": 2.07,
+      "learning_rate": 1.7304747320061255e-07,
+      "logits/chosen": -2.5044076442718506,
+      "logits/rejected": -2.4872841835021973,
+      "logps/chosen": -261.4200134277344,
+      "logps/rejected": -227.45669555664062,
+      "loss": 0.5865,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": 0.03717802092432976,
+      "rewards/margins": 0.35018208622932434,
+      "rewards/rejected": -0.3130040466785431,
+      "step": 500
+    },
+    {
+      "epoch": 2.11,
+      "learning_rate": 1.6539050535987747e-07,
+      "logits/chosen": -2.573312759399414,
+      "logits/rejected": -2.4606995582580566,
+      "logps/chosen": -267.6290588378906,
+      "logps/rejected": -221.90316772460938,
+      "loss": 0.5773,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": 0.02148844487965107,
+      "rewards/margins": 0.38462623953819275,
+      "rewards/rejected": -0.36313778162002563,
+      "step": 510
+    },
+    {
+      "epoch": 2.15,
+      "learning_rate": 1.5773353751914243e-07,
+      "logits/chosen": -2.583714723587036,
+      "logits/rejected": -2.502439022064209,
+      "logps/chosen": -282.7274475097656,
+      "logps/rejected": -226.8433074951172,
+      "loss": 0.5783,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.0283358097076416,
+      "rewards/margins": 0.40725621581077576,
+      "rewards/rejected": -0.37892037630081177,
+      "step": 520
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 1.5007656967840735e-07,
+      "logits/chosen": -2.4836316108703613,
+      "logits/rejected": -2.4393324851989746,
+      "logps/chosen": -251.7774200439453,
+      "logps/rejected": -227.4025421142578,
+      "loss": 0.5791,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": 0.011759527027606964,
+      "rewards/margins": 0.36558085680007935,
+      "rewards/rejected": -0.353821337223053,
+      "step": 530
+    },
+    {
+      "epoch": 2.23,
+      "learning_rate": 1.4241960183767226e-07,
+      "logits/chosen": -2.5330307483673096,
+      "logits/rejected": -2.468540668487549,
+      "logps/chosen": -280.36077880859375,
+      "logps/rejected": -232.63845825195312,
+      "loss": 0.5811,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": 0.0011851644376292825,
+      "rewards/margins": 0.37196025252342224,
+      "rewards/rejected": -0.37077510356903076,
+      "step": 540
+    },
+    {
+      "epoch": 2.27,
+      "learning_rate": 1.347626339969372e-07,
+      "logits/chosen": -2.519770383834839,
+      "logits/rejected": -2.4823544025421143,
+      "logps/chosen": -268.5936584472656,
+      "logps/rejected": -239.03897094726562,
+      "loss": 0.5826,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": 0.04637747257947922,
+      "rewards/margins": 0.3646572530269623,
+      "rewards/rejected": -0.31827980279922485,
+      "step": 550
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 1.2710566615620215e-07,
+      "logits/chosen": -2.47658109664917,
+      "logits/rejected": -2.4247078895568848,
+      "logps/chosen": -272.190673828125,
+      "logps/rejected": -221.92849731445312,
+      "loss": 0.5687,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": 0.01268075406551361,
+      "rewards/margins": 0.39336925745010376,
+      "rewards/rejected": -0.38068851828575134,
+      "step": 560
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 1.1944869831546706e-07,
+      "logits/chosen": -2.49946928024292,
+      "logits/rejected": -2.470533847808838,
+      "logps/chosen": -264.8629455566406,
+      "logps/rejected": -235.60354614257812,
+      "loss": 0.5808,
+      "rewards/accuracies": 0.692187488079071,
+      "rewards/chosen": -0.01222451962530613,
+      "rewards/margins": 0.35588544607162476,
+      "rewards/rejected": -0.36810994148254395,
+      "step": 570
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 1.11791730474732e-07,
+      "logits/chosen": -2.512359619140625,
+      "logits/rejected": -2.4669761657714844,
+      "logps/chosen": -260.2524108886719,
+      "logps/rejected": -223.0822296142578,
+      "loss": 0.5741,
+      "rewards/accuracies": 0.7484375238418579,
+      "rewards/chosen": -6.574243161594495e-05,
+      "rewards/margins": 0.42497771978378296,
+      "rewards/rejected": -0.4250434935092926,
+      "step": 580
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 1.0413476263399694e-07,
+      "logits/chosen": -2.5507304668426514,
+      "logits/rejected": -2.498631715774536,
+      "logps/chosen": -267.2727966308594,
+      "logps/rejected": -215.9676055908203,
+      "loss": 0.5743,
+      "rewards/accuracies": 0.7171875238418579,
+      "rewards/chosen": -0.014520371332764626,
+      "rewards/margins": 0.3767298758029938,
+      "rewards/rejected": -0.39125025272369385,
+      "step": 590
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 9.647779479326186e-08,
+      "logits/chosen": -2.4919803142547607,
+      "logits/rejected": -2.4784021377563477,
+      "logps/chosen": -248.50048828125,
+      "logps/rejected": -221.4516143798828,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.676562488079071,
+      "rewards/chosen": -0.003242678241804242,
+      "rewards/margins": 0.31586432456970215,
+      "rewards/rejected": -0.3191069960594177,
+      "step": 600
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 8.88208269525268e-08,
+      "logits/chosen": -2.4886717796325684,
+      "logits/rejected": -2.4773054122924805,
+      "logps/chosen": -258.47509765625,
+      "logps/rejected": -220.34323120117188,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.01594376005232334,
+      "rewards/margins": 0.3604745864868164,
+      "rewards/rejected": -0.3764183521270752,
+      "step": 610
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 8.116385911179173e-08,
+      "logits/chosen": -2.474463939666748,
+      "logits/rejected": -2.4515841007232666,
+      "logps/chosen": -277.49713134765625,
+      "logps/rejected": -216.4416046142578,
+      "loss": 0.5761,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.03217107802629471,
+      "rewards/margins": 0.41897639632225037,
+      "rewards/rejected": -0.38680535554885864,
+      "step": 620
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 7.350689127105667e-08,
+      "logits/chosen": -2.5482378005981445,
+      "logits/rejected": -2.4707536697387695,
+      "logps/chosen": -270.44488525390625,
+      "logps/rejected": -229.28286743164062,
+      "loss": 0.5673,
+      "rewards/accuracies": 0.698437511920929,
+      "rewards/chosen": 0.0038725235499441624,
+      "rewards/margins": 0.41428858041763306,
+      "rewards/rejected": -0.41041603684425354,
+      "step": 630
+    },
+    {
+      "epoch": 2.64,
+      "learning_rate": 6.584992343032159e-08,
+      "logits/chosen": -2.45513916015625,
+      "logits/rejected": -2.437708854675293,
+      "logps/chosen": -260.6726989746094,
+      "logps/rejected": -225.84298706054688,
+      "loss": 0.5806,
+      "rewards/accuracies": 0.682812511920929,
+      "rewards/chosen": 0.012735338881611824,
+      "rewards/margins": 0.38611000776290894,
+      "rewards/rejected": -0.3733746409416199,
+      "step": 640
+    },
+    {
+      "epoch": 2.69,
+      "learning_rate": 5.819295558958652e-08,
+      "logits/chosen": -2.4959685802459717,
+      "logits/rejected": -2.43827486038208,
+      "logps/chosen": -286.122802734375,
+      "logps/rejected": -219.0840606689453,
+      "loss": 0.5722,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.03723993897438049,
+      "rewards/margins": 0.4596535563468933,
+      "rewards/rejected": -0.4224136471748352,
+      "step": 650
+    },
+    {
+      "epoch": 2.73,
+      "learning_rate": 5.0535987748851455e-08,
+      "logits/chosen": -2.55987811088562,
+      "logits/rejected": -2.5002965927124023,
+      "logps/chosen": -273.2501220703125,
+      "logps/rejected": -228.8253631591797,
+      "loss": 0.5661,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": -0.019126426428556442,
+      "rewards/margins": 0.3617474436759949,
+      "rewards/rejected": -0.3808739185333252,
+      "step": 660
+    },
+    {
+      "epoch": 2.77,
+      "learning_rate": 4.287901990811638e-08,
+      "logits/chosen": -2.5399422645568848,
+      "logits/rejected": -2.4543564319610596,
+      "logps/chosen": -266.138427734375,
+      "logps/rejected": -241.64926147460938,
+      "loss": 0.5723,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": 0.01595628634095192,
+      "rewards/margins": 0.42130351066589355,
+      "rewards/rejected": -0.40534719824790955,
+      "step": 670
+    },
+    {
+      "epoch": 2.81,
+      "learning_rate": 3.522205206738132e-08,
+      "logits/chosen": -2.5083065032958984,
+      "logits/rejected": -2.465390682220459,
+      "logps/chosen": -272.95794677734375,
+      "logps/rejected": -227.048828125,
+      "loss": 0.5761,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -0.012082843109965324,
+      "rewards/margins": 0.39171308279037476,
+      "rewards/rejected": -0.40379589796066284,
+      "step": 680
+    },
+    {
+      "epoch": 2.85,
+      "learning_rate": 2.7565084226646246e-08,
+      "logits/chosen": -2.487924337387085,
+      "logits/rejected": -2.4710183143615723,
+      "logps/chosen": -262.0213317871094,
+      "logps/rejected": -233.55136108398438,
+      "loss": 0.5782,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": 0.0015483855968341231,
+      "rewards/margins": 0.4110836088657379,
+      "rewards/rejected": -0.4095352292060852,
+      "step": 690
+    },
+    {
+      "epoch": 2.89,
+      "learning_rate": 1.9908116385911178e-08,
+      "logits/chosen": -2.507427453994751,
+      "logits/rejected": -2.48207688331604,
+      "logps/chosen": -258.07647705078125,
+      "logps/rejected": -224.3450927734375,
+      "loss": 0.5765,
+      "rewards/accuracies": 0.6703125238418579,
+      "rewards/chosen": -0.017956208437681198,
+      "rewards/margins": 0.3418615460395813,
+      "rewards/rejected": -0.359817773103714,
+      "step": 700
+    },
+    {
+      "epoch": 2.93,
+      "learning_rate": 1.225114854517611e-08,
+      "logits/chosen": -2.5354719161987305,
+      "logits/rejected": -2.4671473503112793,
+      "logps/chosen": -278.4368896484375,
+      "logps/rejected": -218.9778594970703,
+      "loss": 0.5764,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -0.004084877669811249,
+      "rewards/margins": 0.3954611122608185,
+      "rewards/rejected": -0.39954596757888794,
+      "step": 710
+    },
+    {
+      "epoch": 2.97,
+      "learning_rate": 4.594180704441042e-09,
+      "logits/chosen": -2.500247001647949,
+      "logits/rejected": -2.4750142097473145,
+      "logps/chosen": -255.7807159423828,
+      "logps/rejected": -222.0048370361328,
+      "loss": 0.5743,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.0047062961384654045,
+      "rewards/margins": 0.3754323422908783,
+      "rewards/rejected": -0.380138635635376,
+      "step": 720
+    },
+    {
+      "epoch": 3.0,
+      "eval_logits/chosen": -2.259361505508423,
+      "eval_logits/rejected": -2.14780330657959,
+      "eval_logps/chosen": -263.0772705078125,
+      "eval_logps/rejected": -221.30685424804688,
+      "eval_loss": 0.5746620893478394,
+      "eval_rewards/accuracies": 0.7059999704360962,
+      "eval_rewards/chosen": -0.014065464027225971,
+      "eval_rewards/margins": 0.4006173312664032,
+      "eval_rewards/rejected": -0.41468286514282227,
+      "eval_runtime": 237.7658,
+      "eval_samples_per_second": 8.412,
+      "eval_steps_per_second": 0.526,
+      "step": 726
+    },
+    {
+      "epoch": 3.0,
+      "step": 726,
+      "total_flos": 0.0,
+      "train_loss": 0.6110922341996973,
+      "train_runtime": 33040.6759,
+      "train_samples_per_second": 5.626,
+      "train_steps_per_second": 0.022
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 726,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d66eafa67fbcd51a04a2d80677de1f28016eb42571ccc6a98b6cb6997b977e32
+size 4728