Model save

README.md

@@ -0,0 +1,71 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: DPO-PairRM-5-SMI-lr-1e6-iteration-5-t-7e-beta-15e3-1-iteration-6e1-confidence-D1-D2_smi
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# DPO-PairRM-5-SMI-lr-1e6-iteration-5-t-7e-beta-15e3-1-iteration-6e1-confidence-D1-D2_smi
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6444
+- Rewards/chosen: -2.4793
+- Rewards/rejected: -2.9560
+- Rewards/accuracies: 0.6667
+- Rewards/margins: 0.4767
+- Rewards/mix Margin: 0.1749
+- Logps/rejected: -481.8095
+- Logps/chosen: -453.2426
+- Logits/rejected: -1.7012
+- Logits/chosen: -1.7287
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-06
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 64
+- total_eval_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2
+- Datasets 2.17.1
+- Tokenizers 0.15.1

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -1.7287349700927734,
+    "eval_logits/rejected": -1.7011512517929077,
+    "eval_logps/chosen": -453.2426452636719,
+    "eval_logps/rejected": -481.8094787597656,
+    "eval_loss": 0.6444016695022583,
+    "eval_rewards/accuracies": 0.6666666865348816,
+    "eval_rewards/chosen": -2.479323625564575,
+    "eval_rewards/margins": 0.47671714425086975,
+    "eval_rewards/mix_margin": 0.17488937079906464,
+    "eval_rewards/rejected": -2.9560413360595703,
+    "eval_runtime": 716.6697,
+    "eval_samples": 1522,
+    "eval_samples_per_second": 2.124,
+    "eval_steps_per_second": 0.532,
+    "train_loss": 0.5442263817735228,
+    "train_runtime": 33688.2171,
+    "train_samples": 29394,
+    "train_samples_per_second": 0.873,
+    "train_steps_per_second": 0.014
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.2",
+  "use_cache": false,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -1.7287349700927734,
+    "eval_logits/rejected": -1.7011512517929077,
+    "eval_logps/chosen": -453.2426452636719,
+    "eval_logps/rejected": -481.8094787597656,
+    "eval_loss": 0.6444016695022583,
+    "eval_rewards/accuracies": 0.6666666865348816,
+    "eval_rewards/chosen": -2.479323625564575,
+    "eval_rewards/margins": 0.47671714425086975,
+    "eval_rewards/mix_margin": 0.17488937079906464,
+    "eval_rewards/rejected": -2.9560413360595703,
+    "eval_runtime": 716.6697,
+    "eval_samples": 1522,
+    "eval_samples_per_second": 2.124,
+    "eval_steps_per_second": 0.532
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01145455128921bf94f24e0c1a8f6fea6dabc01cc32be8f13574c23bffda35ff
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e69f3c130039a4d474ce0f547b42968d7e1763257e451ec31597f1049f5bb494
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0ef9c7bcdb279ba1a9d8c2dd9c0ddfb82b401680153b3728b772fffe4a8c69d
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Apr25_01-20-17_alin22/events.out.tfevents.1713975746.alin22.3792232.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2b6a2c4720840d27c586b81d2ecc18bac4f1a7a9949182ff4e8d3784582f69e
+size 50553

runs/Apr25_01-20-17_alin22/events.out.tfevents.1714010446.alin22.3792232.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14f69f05c2e9057754d2c13501454e231443297c0a73cc22a874e3855a6064f8
+size 890

runs/Apr25_12-37-07_alin22/events.out.tfevents.1714016355.alin22.4154124.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7792cf2bd89521f4cd807add85a15aa519fdc420c9728b900a51b7b3db8f2c4a
+size 49587

runs/Apr25_12-37-07_alin22/events.out.tfevents.1714050760.alin22.4154124.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1c7cf0c73f963aa39d7cdae7643fd79a954139f34ea41cb116c4facf3014b45
+size 890

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5442263817735228,
+    "train_runtime": 33688.2171,
+    "train_samples": 29394,
+    "train_samples_per_second": 0.873,
+    "train_steps_per_second": 0.014
+}

trainer_state.json

@@ -0,0 +1,900 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9993196353245339,
+  "eval_steps": 1000,
+  "global_step": 459,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.1739130434782606e-08,
+      "logits/chosen": -2.3381757736206055,
+      "logits/rejected": -2.3721675872802734,
+      "logps/chosen": -308.3559875488281,
+      "logps/rejected": -268.3885192871094,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/mix_margin": -1.0728836485895954e-08,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 2.1739130434782607e-07,
+      "logits/chosen": -2.3499717712402344,
+      "logits/rejected": -2.30268931388855,
+      "logps/chosen": -347.79754638671875,
+      "logps/rejected": -304.29705810546875,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.4513888955116272,
+      "rewards/chosen": 0.0014620342990383506,
+      "rewards/confidence": -0.011617942713201046,
+      "rewards/confidence_mean_diff": 0.011617942713201046,
+      "rewards/confidence_moving_diff": 0.0008721869089640677,
+      "rewards/margins": 0.0018856121459975839,
+      "rewards/mix_margin": 0.000961379031650722,
+      "rewards/real_percentage": 14.571428298950195,
+      "rewards/rejected": -0.00042357799247838557,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 4.3478260869565214e-07,
+      "logits/chosen": -2.4307799339294434,
+      "logits/rejected": -2.4313855171203613,
+      "logps/chosen": -337.871826171875,
+      "logps/rejected": -309.18695068359375,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.018211863934993744,
+      "rewards/confidence": -0.013249741867184639,
+      "rewards/confidence_mean_diff": 0.013249741867184639,
+      "rewards/confidence_moving_diff": 0.0012787743471562862,
+      "rewards/margins": 0.004662538878619671,
+      "rewards/mix_margin": 0.002308625727891922,
+      "rewards/real_percentage": 8.800000190734863,
+      "rewards/rejected": 0.013549325987696648,
+      "step": 20
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 6.521739130434782e-07,
+      "logits/chosen": -2.3925347328186035,
+      "logits/rejected": -2.3656105995178223,
+      "logps/chosen": -333.0788879394531,
+      "logps/rejected": -297.11865234375,
+      "loss": 0.679,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.07145825028419495,
+      "rewards/confidence": -0.036696191877126694,
+      "rewards/confidence_mean_diff": 0.036696191877126694,
+      "rewards/confidence_moving_diff": 0.0040471795946359634,
+      "rewards/margins": 0.04734768718481064,
+      "rewards/mix_margin": 0.024542126804590225,
+      "rewards/real_percentage": 7.400000095367432,
+      "rewards/rejected": 0.02411056123673916,
+      "step": 30
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 8.695652173913043e-07,
+      "logits/chosen": -2.276813507080078,
+      "logits/rejected": -2.2625725269317627,
+      "logps/chosen": -302.2676696777344,
+      "logps/rejected": -285.21478271484375,
+      "loss": 0.6483,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.041187357157468796,
+      "rewards/confidence": -0.08288822323083878,
+      "rewards/confidence_mean_diff": 0.08288822323083878,
+      "rewards/confidence_moving_diff": 0.0016981459921225905,
+      "rewards/margins": 0.11979661136865616,
+      "rewards/mix_margin": 0.06444107741117477,
+      "rewards/real_percentage": 5.599999904632568,
+      "rewards/rejected": -0.07860924303531647,
+      "step": 40
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.997685665512417e-07,
+      "logits/chosen": -2.103567600250244,
+      "logits/rejected": -2.0963990688323975,
+      "logps/chosen": -310.5840759277344,
+      "logps/rejected": -280.5057373046875,
+      "loss": 0.6262,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07748699188232422,
+      "rewards/confidence": -0.15422135591506958,
+      "rewards/confidence_mean_diff": 0.15422135591506958,
+      "rewards/confidence_moving_diff": 0.007699769921600819,
+      "rewards/margins": 0.17081865668296814,
+      "rewards/mix_margin": 0.09792973846197128,
+      "rewards/real_percentage": 6.599999904632568,
+      "rewards/rejected": -0.24830564856529236,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 9.971674001050686e-07,
+      "logits/chosen": -1.7708213329315186,
+      "logits/rejected": -1.7266209125518799,
+      "logps/chosen": -368.86602783203125,
+      "logps/rejected": -356.1971740722656,
+      "loss": 0.5962,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.36815953254699707,
+      "rewards/confidence": -0.21500273048877716,
+      "rewards/confidence_mean_diff": 0.21500273048877716,
+      "rewards/confidence_moving_diff": 0.00331355188973248,
+      "rewards/margins": 0.3264981508255005,
+      "rewards/mix_margin": 0.19259774684906006,
+      "rewards/real_percentage": 6.800000190734863,
+      "rewards/rejected": -0.6946576833724976,
+      "step": 60
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 9.916908694464492e-07,
+      "logits/chosen": -1.764725685119629,
+      "logits/rejected": -1.739040732383728,
+      "logps/chosen": -355.16314697265625,
+      "logps/rejected": -341.5367126464844,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7925843000411987,
+      "rewards/confidence": -0.2245199978351593,
+      "rewards/confidence_mean_diff": 0.2245199978351593,
+      "rewards/confidence_moving_diff": 0.017536107450723648,
+      "rewards/margins": 0.3716357350349426,
+      "rewards/mix_margin": 0.21924924850463867,
+      "rewards/real_percentage": 5.800000190734863,
+      "rewards/rejected": -1.1642199754714966,
+      "step": 70
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 9.833706480524877e-07,
+      "logits/chosen": -1.933165192604065,
+      "logits/rejected": -1.9561519622802734,
+      "logps/chosen": -402.5296325683594,
+      "logps/rejected": -389.52899169921875,
+      "loss": 0.5797,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0171725749969482,
+      "rewards/confidence": -0.4024506211280823,
+      "rewards/confidence_mean_diff": 0.4024506211280823,
+      "rewards/confidence_moving_diff": -0.013792142271995544,
+      "rewards/margins": 0.32845553755760193,
+      "rewards/mix_margin": 0.17139557003974915,
+      "rewards/real_percentage": 6.199999809265137,
+      "rewards/rejected": -1.345628023147583,
+      "step": 80
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 9.722548558659455e-07,
+      "logits/chosen": -1.8798820972442627,
+      "logits/rejected": -1.87497878074646,
+      "logps/chosen": -400.71405029296875,
+      "logps/rejected": -409.4770812988281,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0970375537872314,
+      "rewards/confidence": -0.29926934838294983,
+      "rewards/confidence_mean_diff": 0.29926934838294983,
+      "rewards/confidence_moving_diff": 0.015591544099152088,
+      "rewards/margins": 0.3549976646900177,
+      "rewards/mix_margin": 0.20659112930297852,
+      "rewards/real_percentage": 6.800000190734863,
+      "rewards/rejected": -1.4520353078842163,
+      "step": 90
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 9.584077809938854e-07,
+      "logits/chosen": -1.9064958095550537,
+      "logits/rejected": -1.8910179138183594,
+      "logps/chosen": -380.4066467285156,
+      "logps/rejected": -376.3758850097656,
+      "loss": 0.5538,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8957662582397461,
+      "rewards/confidence": -0.21787028014659882,
+      "rewards/confidence_mean_diff": 0.21787028014659882,
+      "rewards/confidence_moving_diff": -0.01832030713558197,
+      "rewards/margins": 0.5051447749137878,
+      "rewards/mix_margin": 0.28336718678474426,
+      "rewards/real_percentage": 5.599999904632568,
+      "rewards/rejected": -1.4009110927581787,
+      "step": 100
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 9.419095078978505e-07,
+      "logits/chosen": -1.9809643030166626,
+      "logits/rejected": -1.9503233432769775,
+      "logps/chosen": -386.52081298828125,
+      "logps/rejected": -377.4222717285156,
+      "loss": 0.5741,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7311219573020935,
+      "rewards/confidence": -0.30758336186408997,
+      "rewards/confidence_mean_diff": 0.30758336186408997,
+      "rewards/confidence_moving_diff": 0.007133707404136658,
+      "rewards/margins": 0.41949161887168884,
+      "rewards/mix_margin": 0.2570333480834961,
+      "rewards/real_percentage": 6.800000190734863,
+      "rewards/rejected": -1.15061354637146,
+      "step": 110
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 9.228554542259359e-07,
+      "logits/chosen": -2.112670421600342,
+      "logits/rejected": -2.097276210784912,
+      "logps/chosen": -356.1893615722656,
+      "logps/rejected": -397.5374450683594,
+      "loss": 0.5385,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8588116765022278,
+      "rewards/confidence": -0.3459985852241516,
+      "rewards/confidence_mean_diff": 0.3459985852241516,
+      "rewards/confidence_moving_diff": 0.003056983696296811,
+      "rewards/margins": 0.68636155128479,
+      "rewards/mix_margin": 0.2696496844291687,
+      "rewards/real_percentage": 6.400000095367432,
+      "rewards/rejected": -1.545173168182373,
+      "step": 120
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 9.013558189654817e-07,
+      "logits/chosen": -1.9888393878936768,
+      "logits/rejected": -1.9572227001190186,
+      "logps/chosen": -455.81561279296875,
+      "logps/rejected": -461.1678161621094,
+      "loss": 0.5316,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.197947382926941,
+      "rewards/confidence": -0.27709904313087463,
+      "rewards/confidence_mean_diff": 0.27709904313087463,
+      "rewards/confidence_moving_diff": 0.008776068687438965,
+      "rewards/margins": 0.6648138165473938,
+      "rewards/mix_margin": 0.34475114941596985,
+      "rewards/real_percentage": 5.400000095367432,
+      "rewards/rejected": -1.86276113986969,
+      "step": 130
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 8.775349451079948e-07,
+      "logits/chosen": -2.085186004638672,
+      "logits/rejected": -2.0593903064727783,
+      "logps/chosen": -428.7594299316406,
+      "logps/rejected": -433.14324951171875,
+      "loss": 0.536,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1666946411132812,
+      "rewards/confidence": -0.3819279074668884,
+      "rewards/confidence_mean_diff": 0.3819279074668884,
+      "rewards/confidence_moving_diff": -0.0010703206062316895,
+      "rewards/margins": 0.5809696912765503,
+      "rewards/mix_margin": 0.281490296125412,
+      "rewards/real_percentage": 5.199999809265137,
+      "rewards/rejected": -1.747664451599121,
+      "step": 140
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 8.515306005123179e-07,
+      "logits/chosen": -2.1095032691955566,
+      "logits/rejected": -2.10186505317688,
+      "logps/chosen": -407.43963623046875,
+      "logps/rejected": -430.080322265625,
+      "loss": 0.5424,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.9244325160980225,
+      "rewards/confidence": -0.3746878504753113,
+      "rewards/confidence_mean_diff": 0.3746878504753113,
+      "rewards/confidence_moving_diff": 0.001715996884740889,
+      "rewards/margins": 0.4158695340156555,
+      "rewards/mix_margin": 0.18484942615032196,
+      "rewards/real_percentage": 5.800000190734863,
+      "rewards/rejected": -2.340301990509033,
+      "step": 150
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 8.234931811251738e-07,
+      "logits/chosen": -2.0886027812957764,
+      "logits/rejected": -2.0623221397399902,
+      "logps/chosen": -434.42889404296875,
+      "logps/rejected": -436.99591064453125,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.801783561706543,
+      "rewards/confidence": -0.35509809851646423,
+      "rewards/confidence_mean_diff": 0.35509809851646423,
+      "rewards/confidence_moving_diff": -0.013272729702293873,
+      "rewards/margins": 0.5741153955459595,
+      "rewards/mix_margin": 0.3070274591445923,
+      "rewards/real_percentage": 6.400000095367432,
+      "rewards/rejected": -2.375898838043213,
+      "step": 160
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 7.935848411672499e-07,
+      "logits/chosen": -2.0530495643615723,
+      "logits/rejected": -2.0262792110443115,
+      "logps/chosen": -421.6261291503906,
+      "logps/rejected": -443.7255859375,
+      "loss": 0.5362,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8145627975463867,
+      "rewards/confidence": -0.3444231450557709,
+      "rewards/confidence_mean_diff": 0.3444231450557709,
+      "rewards/confidence_moving_diff": 0.011829972267150879,
+      "rewards/margins": 0.5010958909988403,
+      "rewards/mix_margin": 0.2390981912612915,
+      "rewards/real_percentage": 6.199999809265137,
+      "rewards/rejected": -2.3156585693359375,
+      "step": 170
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 7.619785553153864e-07,
+      "logits/chosen": -1.9314157962799072,
+      "logits/rejected": -1.9014371633529663,
+      "logps/chosen": -421.787353515625,
+      "logps/rejected": -416.24273681640625,
+      "loss": 0.5448,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.6310911178588867,
+      "rewards/confidence": -0.34312066435813904,
+      "rewards/confidence_mean_diff": 0.34312066435813904,
+      "rewards/confidence_moving_diff": -0.008907085284590721,
+      "rewards/margins": 0.6062546968460083,
+      "rewards/mix_margin": 0.31984299421310425,
+      "rewards/real_percentage": 6.0,
+      "rewards/rejected": -2.2373456954956055,
+      "step": 180
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 7.288571183047321e-07,
+      "logits/chosen": -1.8551080226898193,
+      "logits/rejected": -1.8303585052490234,
+      "logps/chosen": -430.4051208496094,
+      "logps/rejected": -433.05743408203125,
+      "loss": 0.5427,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.7081422805786133,
+      "rewards/confidence": -0.3352188169956207,
+      "rewards/confidence_mean_diff": 0.3352188169956207,
+      "rewards/confidence_moving_diff": 0.016140591353178024,
+      "rewards/margins": 0.6280410289764404,
+      "rewards/mix_margin": 0.3198699355125427,
+      "rewards/real_percentage": 6.199999809265137,
+      "rewards/rejected": -2.3361833095550537,
+      "step": 190
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 6.944120877366604e-07,
+      "logits/chosen": -1.8757364749908447,
+      "logits/rejected": -1.862011194229126,
+      "logps/chosen": -449.3797302246094,
+      "logps/rejected": -497.9132385253906,
+      "loss": 0.5307,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.089892864227295,
+      "rewards/confidence": -0.4818331301212311,
+      "rewards/confidence_mean_diff": 0.4818331301212311,
+      "rewards/confidence_moving_diff": 0.015796076506376266,
+      "rewards/margins": 0.6883096694946289,
+      "rewards/mix_margin": 0.28093141317367554,
+      "rewards/real_percentage": 6.400000095367432,
+      "rewards/rejected": -2.778202533721924,
+      "step": 200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 6.588426762067099e-07,
+      "logits/chosen": -1.7704166173934937,
+      "logits/rejected": -1.7364553213119507,
+      "logps/chosen": -506.0335998535156,
+      "logps/rejected": -564.422607421875,
+      "loss": 0.5411,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.170118570327759,
+      "rewards/confidence": -0.4565798342227936,
+      "rewards/confidence_mean_diff": 0.4565798342227936,
+      "rewards/confidence_moving_diff": -0.0030906558968126774,
+      "rewards/margins": 0.7355421781539917,
+      "rewards/mix_margin": 0.2624685764312744,
+      "rewards/real_percentage": 6.0,
+      "rewards/rejected": -2.905660390853882,
+      "step": 210
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 6.223545991599182e-07,
+      "logits/chosen": -1.8121325969696045,
+      "logits/rejected": -1.7733787298202515,
+      "logps/chosen": -460.2236328125,
+      "logps/rejected": -474.4139709472656,
+      "loss": 0.5638,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9526455402374268,
+      "rewards/confidence": -0.44291096925735474,
+      "rewards/confidence_mean_diff": 0.44291096925735474,
+      "rewards/confidence_moving_diff": -0.026620090007781982,
+      "rewards/margins": 0.5833476185798645,
+      "rewards/mix_margin": 0.28964775800704956,
+      "rewards/real_percentage": 5.599999904632568,
+      "rewards/rejected": -2.5359930992126465,
+      "step": 220
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.85158885136973e-07,
+      "logits/chosen": -1.9277960062026978,
+      "logits/rejected": -1.8827558755874634,
+      "logps/chosen": -423.71197509765625,
+      "logps/rejected": -446.4361267089844,
+      "loss": 0.5302,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8649892807006836,
+      "rewards/confidence": -0.3467513918876648,
+      "rewards/confidence_mean_diff": 0.3467513918876648,
+      "rewards/confidence_moving_diff": -0.005762821529060602,
+      "rewards/margins": 0.5457550883293152,
+      "rewards/mix_margin": 0.22559857368469238,
+      "rewards/real_percentage": 5.400000095367432,
+      "rewards/rejected": -2.4107441902160645,
+      "step": 230
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 5.474706552921074e-07,
+      "logits/chosen": -1.935349464416504,
+      "logits/rejected": -1.8973939418792725,
+      "logps/chosen": -395.6863708496094,
+      "logps/rejected": -423.58782958984375,
+      "loss": 0.5307,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.5539497137069702,
+      "rewards/confidence": -0.293173611164093,
+      "rewards/confidence_mean_diff": 0.293173611164093,
+      "rewards/confidence_moving_diff": -0.00015986264043021947,
+      "rewards/margins": 0.6723454594612122,
+      "rewards/mix_margin": 0.30846795439720154,
+      "rewards/real_percentage": 6.199999809265137,
+      "rewards/rejected": -2.226295232772827,
+      "step": 240
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 5.095078792413976e-07,
+      "logits/chosen": -1.9067703485488892,
+      "logits/rejected": -1.8853263854980469,
+      "logps/chosen": -412.2330627441406,
+      "logps/rejected": -446.505859375,
+      "loss": 0.5122,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.381047010421753,
+      "rewards/confidence": -0.4467477798461914,
+      "rewards/confidence_mean_diff": 0.4467477798461914,
+      "rewards/confidence_moving_diff": 0.001344837248325348,
+      "rewards/margins": 0.7658092975616455,
+      "rewards/mix_margin": 0.30207857489585876,
+      "rewards/real_percentage": 7.0,
+      "rewards/rejected": -2.1468563079833984,
+      "step": 250
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.714901144369981e-07,
+      "logits/chosen": -1.9301033020019531,
+      "logits/rejected": -1.8551814556121826,
+      "logps/chosen": -422.6144104003906,
+      "logps/rejected": -459.46405029296875,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.9298696517944336,
+      "rewards/confidence": -0.27530989050865173,
+      "rewards/confidence_mean_diff": 0.27530989050865173,
+      "rewards/confidence_moving_diff": -0.011671778745949268,
+      "rewards/margins": 0.7381618618965149,
+      "rewards/mix_margin": 0.3380785584449768,
+      "rewards/real_percentage": 6.199999809265137,
+      "rewards/rejected": -2.668031930923462,
+      "step": 260
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.33637236358139e-07,
+      "logits/chosen": -1.9588849544525146,
+      "logits/rejected": -1.9613529443740845,
+      "logps/chosen": -454.5,
+      "logps/rejected": -473.86376953125,
+      "loss": 0.5282,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8807973861694336,
+      "rewards/confidence": -0.46479693055152893,
+      "rewards/confidence_mean_diff": 0.46479693055152893,
+      "rewards/confidence_moving_diff": 0.034666456282138824,
+      "rewards/margins": 0.520012378692627,
+      "rewards/mix_margin": 0.2387557029724121,
+      "rewards/real_percentage": 7.0,
+      "rewards/rejected": -2.4008097648620605,
+      "step": 270
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9616816686281635e-07,
+      "logits/chosen": -1.82375168800354,
+      "logits/rejected": -1.8041833639144897,
+      "logps/chosen": -437.76922607421875,
+      "logps/rejected": -447.7157287597656,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.6059573888778687,
+      "rewards/confidence": -0.39664340019226074,
+      "rewards/confidence_mean_diff": 0.39664340019226074,
+      "rewards/confidence_moving_diff": -0.011675971560180187,
+      "rewards/margins": 0.5647900104522705,
+      "rewards/mix_margin": 0.2936910390853882,
+      "rewards/real_percentage": 6.199999809265137,
+      "rewards/rejected": -2.1707472801208496,
+      "step": 280
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.592996080547438e-07,
+      "logits/chosen": -1.786231279373169,
+      "logits/rejected": -1.7279274463653564,
+      "logps/chosen": -411.953125,
+      "logps/rejected": -432.6134338378906,
+      "loss": 0.5355,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9004911184310913,
+      "rewards/confidence": -0.3363880217075348,
+      "rewards/confidence_mean_diff": 0.3363880217075348,
+      "rewards/confidence_moving_diff": 0.010175123810768127,
+      "rewards/margins": 0.7472087144851685,
+      "rewards/mix_margin": 0.35827359557151794,
+      "rewards/real_percentage": 6.800000190734863,
+      "rewards/rejected": -2.6476995944976807,
+      "step": 290
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.23244788988237e-07,
+      "logits/chosen": -1.7489144802093506,
+      "logits/rejected": -1.6973133087158203,
+      "logps/chosen": -410.25103759765625,
+      "logps/rejected": -413.9371032714844,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.2288432121276855,
+      "rewards/confidence": -0.4106927514076233,
+      "rewards/confidence_mean_diff": 0.4106927514076233,
+      "rewards/confidence_moving_diff": 0.02881682850420475,
+      "rewards/margins": 0.6875227689743042,
+      "rewards/mix_margin": 0.32858434319496155,
+      "rewards/real_percentage": 6.400000095367432,
+      "rewards/rejected": -2.9163661003112793,
+      "step": 300
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 2.8821223245945747e-07,
+      "logits/chosen": -1.6137434244155884,
+      "logits/rejected": -1.552942156791687,
+      "logps/chosen": -464.08477783203125,
+      "logps/rejected": -489.72833251953125,
+      "loss": 0.5231,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.016467332839966,
+      "rewards/confidence": -0.37749093770980835,
+      "rewards/confidence_mean_diff": 0.37749093770980835,
+      "rewards/confidence_moving_diff": -0.04578464850783348,
+      "rewards/margins": 0.7573314905166626,
+      "rewards/mix_margin": 0.3511705994606018,
+      "rewards/real_percentage": 6.599999904632568,
+      "rewards/rejected": -2.773798704147339,
+      "step": 310
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.5440454901626487e-07,
+      "logits/chosen": -1.6135238409042358,
+      "logits/rejected": -1.5706686973571777,
+      "logps/chosen": -462.1180725097656,
+      "logps/rejected": -497.38104248046875,
+      "loss": 0.5229,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.4821410179138184,
+      "rewards/confidence": -0.35573774576187134,
+      "rewards/confidence_mean_diff": 0.35573774576187134,
+      "rewards/confidence_moving_diff": 0.010884365066885948,
+      "rewards/margins": 0.6469907760620117,
+      "rewards/mix_margin": 0.28311434388160706,
+      "rewards/real_percentage": 6.800000190734863,
+      "rewards/rejected": -3.12913179397583,
+      "step": 320
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.220172651615188e-07,
+      "logits/chosen": -1.5926752090454102,
+      "logits/rejected": -1.5634915828704834,
+      "logps/chosen": -456.4151306152344,
+      "logps/rejected": -458.0672912597656,
+      "loss": 0.4975,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9481618404388428,
+      "rewards/confidence": -0.35025110840797424,
+      "rewards/confidence_mean_diff": 0.35025110840797424,
+      "rewards/confidence_moving_diff": -0.0056252507492899895,
+      "rewards/margins": 0.725985050201416,
+      "rewards/mix_margin": 0.387787401676178,
+      "rewards/real_percentage": 5.400000095367432,
+      "rewards/rejected": -2.674147129058838,
+      "step": 330
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.9123769252690407e-07,
+      "logits/chosen": -1.6937768459320068,
+      "logits/rejected": -1.6783796548843384,
+      "logps/chosen": -435.18768310546875,
+      "logps/rejected": -442.1273498535156,
+      "loss": 0.4829,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.9751695394515991,
+      "rewards/confidence": -0.37928253412246704,
+      "rewards/confidence_mean_diff": 0.37928253412246704,
+      "rewards/confidence_moving_diff": -0.027161384001374245,
+      "rewards/margins": 0.6305011510848999,
+      "rewards/mix_margin": 0.3498138189315796,
+      "rewards/real_percentage": 5.800000190734863,
+      "rewards/rejected": -2.60567045211792,
+      "step": 340
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.6224384455740787e-07,
+      "logits/chosen": -1.710269570350647,
+      "logits/rejected": -1.652931809425354,
+      "logps/chosen": -467.62591552734375,
+      "logps/rejected": -491.04217529296875,
+      "loss": 0.5033,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.6455416679382324,
+      "rewards/confidence": -0.3581121265888214,
+      "rewards/confidence_mean_diff": 0.3581121265888214,
+      "rewards/confidence_moving_diff": 0.025283757597208023,
+      "rewards/margins": 0.623335063457489,
+      "rewards/mix_margin": 0.28707486391067505,
+      "rewards/real_percentage": 7.599999904632568,
+      "rewards/rejected": -3.268876552581787,
+      "step": 350
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.3520340697179406e-07,
+      "logits/chosen": -1.6668322086334229,
+      "logits/rejected": -1.6106383800506592,
+      "logps/chosen": -469.1095275878906,
+      "logps/rejected": -478.17449951171875,
+      "loss": 0.5166,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.123115062713623,
+      "rewards/confidence": -0.4552002549171448,
+      "rewards/confidence_mean_diff": 0.4552002549171448,
+      "rewards/confidence_moving_diff": 0.0014173805247992277,
+      "rewards/margins": 0.6338737607002258,
+      "rewards/mix_margin": 0.3297014832496643,
+      "rewards/real_percentage": 6.0,
+      "rewards/rejected": -2.756988763809204,
+      "step": 360
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.1027276795341134e-07,
+      "logits/chosen": -1.5943629741668701,
+      "logits/rejected": -1.6161006689071655,
+      "logps/chosen": -467.4046325683594,
+      "logps/rejected": -490.90777587890625,
+      "loss": 0.5108,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.3414371013641357,
+      "rewards/confidence": -0.44518932700157166,
+      "rewards/confidence_mean_diff": 0.44518932700157166,
+      "rewards/confidence_moving_diff": 0.0043337224051356316,
+      "rewards/margins": 0.6999163627624512,
+      "rewards/mix_margin": 0.32578808069229126,
+      "rewards/real_percentage": 6.0,
+      "rewards/rejected": -3.041353464126587,
+      "step": 370
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.75961136802183e-08,
+      "logits/chosen": -1.6814724206924438,
+      "logits/rejected": -1.6028457880020142,
+      "logps/chosen": -474.9368591308594,
+      "logps/rejected": -513.1763916015625,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.2711257934570312,
+      "rewards/confidence": -0.42974844574928284,
+      "rewards/confidence_mean_diff": 0.42974844574928284,
+      "rewards/confidence_moving_diff": 0.018308240920305252,
+      "rewards/margins": 0.8098933100700378,
+      "rewards/mix_margin": 0.3282521367073059,
+      "rewards/real_percentage": 7.0,
+      "rewards/rejected": -3.081019163131714,
+      "step": 380
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 6.730459442502328e-08,
+      "logits/chosen": -1.6732743978500366,
+      "logits/rejected": -1.6020500659942627,
+      "logps/chosen": -492.012939453125,
+      "logps/rejected": -516.7794189453125,
+      "loss": 0.5067,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.0890817642211914,
+      "rewards/confidence": -0.4012150168418884,
+      "rewards/confidence_mean_diff": 0.4012150168418884,
+      "rewards/confidence_moving_diff": -0.030136477202177048,
+      "rewards/margins": 0.7823335528373718,
+      "rewards/mix_margin": 0.3618648648262024,
+      "rewards/real_percentage": 5.800000190734863,
+      "rewards/rejected": -2.871415376663208,
+      "step": 390
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 4.951556604879048e-08,
+      "logits/chosen": -1.6667470932006836,
+      "logits/rejected": -1.634762167930603,
+      "logps/chosen": -512.33251953125,
+      "logps/rejected": -515.0343017578125,
+      "loss": 0.5576,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.8803839683532715,
+      "rewards/confidence": -0.4185534119606018,
+      "rewards/confidence_mean_diff": 0.4185534119606018,
+      "rewards/confidence_moving_diff": 0.005899274256080389,
+      "rewards/margins": 0.5610276460647583,
+      "rewards/mix_margin": 0.29152485728263855,
+      "rewards/real_percentage": 6.400000095367432,
+      "rewards/rejected": -3.4414114952087402,
+      "step": 400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 3.4331911273830784e-08,
+      "logits/chosen": -1.7730276584625244,
+      "logits/rejected": -1.7303050756454468,
+      "logps/chosen": -459.1253967285156,
+      "logps/rejected": -487.03375244140625,
+      "loss": 0.5318,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.33591365814209,
+      "rewards/confidence": -0.42727431654930115,
+      "rewards/confidence_mean_diff": 0.42727431654930115,
+      "rewards/confidence_moving_diff": 0.02859329618513584,
+      "rewards/margins": 0.6499987244606018,
+      "rewards/mix_margin": 0.3042239546775818,
+      "rewards/real_percentage": 7.199999809265137,
+      "rewards/rejected": -2.985912322998047,
+      "step": 410
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 2.184144466229154e-08,
+      "logits/chosen": -1.7364966869354248,
+      "logits/rejected": -1.716314673423767,
+      "logps/chosen": -482.951416015625,
+      "logps/rejected": -509.6161193847656,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.3884456157684326,
+      "rewards/confidence": -0.4644384980201721,
+      "rewards/confidence_mean_diff": 0.4644384980201721,
+      "rewards/confidence_moving_diff": 0.015452265739440918,
+      "rewards/margins": 0.7966889142990112,
+      "rewards/mix_margin": 0.33610981702804565,
+      "rewards/real_percentage": 6.400000095367432,
+      "rewards/rejected": -3.1851344108581543,
+      "step": 420
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.2116404741244202e-08,
+      "logits/chosen": -1.6397075653076172,
+      "logits/rejected": -1.581413984298706,
+      "logps/chosen": -482.9574279785156,
+      "logps/rejected": -507.77972412109375,
+      "loss": 0.4865,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.1570003032684326,
+      "rewards/confidence": -0.5490925908088684,
+      "rewards/confidence_mean_diff": 0.5490925908088684,
+      "rewards/confidence_moving_diff": -0.04263176769018173,
+      "rewards/margins": 0.6673628091812134,
+      "rewards/mix_margin": 0.2867775559425354,
+      "rewards/real_percentage": 5.599999904632568,
+      "rewards/rejected": -2.8243632316589355,
+      "step": 430
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 5.2130362116641905e-09,
+      "logits/chosen": -1.661386251449585,
+      "logits/rejected": -1.655612587928772,
+      "logps/chosen": -480.28643798828125,
+      "logps/rejected": -493.21697998046875,
+      "loss": 0.5137,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.342162609100342,
+      "rewards/confidence": -0.37716978788375854,
+      "rewards/confidence_mean_diff": 0.37716978788375854,
+      "rewards/confidence_moving_diff": 0.004406067542731762,
+      "rewards/margins": 0.6741171479225159,
+      "rewards/mix_margin": 0.2875633239746094,
+      "rewards/real_percentage": 6.599999904632568,
+      "rewards/rejected": -3.016279935836792,
+      "step": 440
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1712646575922636e-09,
+      "logits/chosen": -1.7303485870361328,
+      "logits/rejected": -1.7037239074707031,
+      "logps/chosen": -473.63946533203125,
+      "logps/rejected": -522.4127197265625,
+      "loss": 0.4808,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.1520164012908936,
+      "rewards/confidence": -0.3412160277366638,
+      "rewards/confidence_mean_diff": 0.3412160277366638,
+      "rewards/confidence_moving_diff": 0.019679775461554527,
+      "rewards/margins": 1.036991000175476,
+      "rewards/mix_margin": 0.3758584260940552,
+      "rewards/real_percentage": 6.800000190734863,
+      "rewards/rejected": -3.189007520675659,
+      "step": 450
+    },
+    {
+      "epoch": 1.0,
+      "step": 459,
+      "total_flos": 0.0,
+      "train_loss": 0.5442263817735228,
+      "train_runtime": 33688.2171,
+      "train_samples_per_second": 0.873,
+      "train_steps_per_second": 0.014
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 459,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9da3eb928776786e44106432d6b86ad9ea55535f232801cc5d780f06ca84011f
+size 6200