Model save

README.md

@@ -0,0 +1,59 @@
+---
+license: other
+base_model: Qwen/Qwen1.5-7B-Chat
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: qwen1.5-7b-chat-orpo-v2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# qwen1.5-7b-chat-orpo-v2
+
+This model is a fine-tuned version of [Qwen/Qwen1.5-7B-Chat](https://huggingface.co/Qwen/Qwen1.5-7B-Chat) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 16
+- total_train_batch_size: 16
+- total_eval_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.2
+- Pytorch 2.2.2
+- Datasets 2.19.2
+- Tokenizers 0.19.1

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 1.340359150177747,
+    "train_runtime": 13809.2478,
+    "train_samples": 8844,
+    "train_samples_per_second": 0.64,
+    "train_steps_per_second": 0.04
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen1.5-7B-Chat",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 151645,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151646
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "bos_token_id": 151644,
+  "do_sample": true,
+  "eos_token_id": 151645,
+  "pad_token_id": 151645,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.41.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3021cc6b515c51d70e9a53b5feab46786400ad6964de6fcd9232a077c40877f3
+size 4986106736

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b23957db4c417201cd121795c0b1ae06b7b7fed2bc8a57671311f0f0efd8d5d6
+size 4981244328

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27abf844e236e86c793aec186690dd9a4524887b6c87c1295419cdb6e13a709
+size 4228306968

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a08186486ebe7982a8238e7dd269cc4f67be0f700dd4aa42d170dc9fedebaa8a
+size 1242284160

model.safetensors.index.json

@@ -0,0 +1,394 @@
+{
+  "metadata": {
+    "total_size": 15437897728
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

runs/Jun06_10-27-14_e00094d41e75473dafd453e213587a54000001/events.out.tfevents.1717669929.e00094d41e75473dafd453e213587a54000001.1201.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:244b235ff9200a06f4718886852fe431f8adc5fd2dbcedff9f3a6ba959c06dde
+size 52826

special_tokens_map.json

@@ -0,0 +1,21 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>"
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|im_end|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 1.340359150177747,
+    "train_runtime": 13809.2478,
+    "train_samples": 8844,
+    "train_samples_per_second": 0.64,
+    "train_steps_per_second": 0.04
+}

trainer_state.json

@@ -0,0 +1,1032 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 553,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.018083182640144666,
+      "grad_norm": 270.0,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.524066150188446,
+      "log_odds_ratio": -0.5735815763473511,
+      "logits/chosen": -3.028451442718506,
+      "logits/rejected": -3.2061283588409424,
+      "logps/chosen": -1.6356823444366455,
+      "logps/rejected": -2.097788095474243,
+      "loss": 5.9583,
+      "nll_loss": 6.051669120788574,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.08178411424160004,
+      "rewards/margins": 0.02310529164969921,
+      "rewards/rejected": -0.1048894077539444,
+      "step": 10
+    },
+    {
+      "epoch": 0.03616636528028933,
+      "grad_norm": 338.0,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": 0.5587388873100281,
+      "log_odds_ratio": -0.5176494121551514,
+      "logits/chosen": -2.638868808746338,
+      "logits/rejected": -2.6223673820495605,
+      "logps/chosen": -1.23783278465271,
+      "logps/rejected": -1.646284818649292,
+      "loss": 5.6814,
+      "nll_loss": 3.5105323791503906,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.061891645193099976,
+      "rewards/margins": 0.020422598347067833,
+      "rewards/rejected": -0.08231424540281296,
+      "step": 20
+    },
+    {
+      "epoch": 0.054249547920433995,
+      "grad_norm": 197.0,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": -0.7765415906906128,
+      "log_odds_ratio": -1.4850478172302246,
+      "logits/chosen": -2.554340362548828,
+      "logits/rejected": -3.0053932666778564,
+      "logps/chosen": -2.7067830562591553,
+      "logps/rejected": -1.897303819656372,
+      "loss": 4.095,
+      "nll_loss": 5.7828497886657715,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.1353391706943512,
+      "rewards/margins": -0.04047396779060364,
+      "rewards/rejected": -0.09486518800258636,
+      "step": 30
+    },
+    {
+      "epoch": 0.07233273056057866,
+      "grad_norm": 64.0,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 1.3007441759109497,
+      "log_odds_ratio": -0.4158785343170166,
+      "logits/chosen": -2.897387981414795,
+      "logits/rejected": -3.0413200855255127,
+      "logps/chosen": -1.3996171951293945,
+      "logps/rejected": -2.517918109893799,
+      "loss": 2.6986,
+      "nll_loss": 2.556391954421997,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.06998086720705032,
+      "rewards/margins": 0.05591505020856857,
+      "rewards/rejected": -0.1258959025144577,
+      "step": 40
+    },
+    {
+      "epoch": 0.09041591320072333,
+      "grad_norm": 21.875,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 0.607527494430542,
+      "log_odds_ratio": -0.464752733707428,
+      "logits/chosen": -3.3353819847106934,
+      "logits/rejected": -3.417086124420166,
+      "logps/chosen": -1.5169802904129028,
+      "logps/rejected": -2.032203197479248,
+      "loss": 1.9263,
+      "nll_loss": 1.4955819845199585,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.0758490115404129,
+      "rewards/margins": 0.025761136785149574,
+      "rewards/rejected": -0.10161016136407852,
+      "step": 50
+    },
+    {
+      "epoch": 0.10849909584086799,
+      "grad_norm": 15.25,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 0.3995138704776764,
+      "log_odds_ratio": -0.6033788919448853,
+      "logits/chosen": -3.0420987606048584,
+      "logits/rejected": -3.2451672554016113,
+      "logps/chosen": -1.6125237941741943,
+      "logps/rejected": -1.9586118459701538,
+      "loss": 1.7326,
+      "nll_loss": 2.063459634780884,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.08062619715929031,
+      "rewards/margins": 0.017304399982094765,
+      "rewards/rejected": -0.09793059527873993,
+      "step": 60
+    },
+    {
+      "epoch": 0.12658227848101267,
+      "grad_norm": 8.5625,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 0.3118763566017151,
+      "log_odds_ratio": -0.5856896638870239,
+      "logits/chosen": -3.4981963634490967,
+      "logits/rejected": -3.5791831016540527,
+      "logps/chosen": -1.4380217790603638,
+      "logps/rejected": -1.686657190322876,
+      "loss": 1.4005,
+      "nll_loss": 1.4499210119247437,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07190109044313431,
+      "rewards/margins": 0.012431769631803036,
+      "rewards/rejected": -0.08433286100625992,
+      "step": 70
+    },
+    {
+      "epoch": 0.14466546112115733,
+      "grad_norm": 7.28125,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 0.6805863976478577,
+      "log_odds_ratio": -0.5004494190216064,
+      "logits/chosen": -3.5435702800750732,
+      "logits/rejected": -3.5884978771209717,
+      "logps/chosen": -1.284294843673706,
+      "logps/rejected": -1.8150221109390259,
+      "loss": 1.3307,
+      "nll_loss": 1.4621249437332153,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.06421474367380142,
+      "rewards/margins": 0.026536360383033752,
+      "rewards/rejected": -0.09075109660625458,
+      "step": 80
+    },
+    {
+      "epoch": 0.162748643761302,
+      "grad_norm": 6.6875,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 0.7451115846633911,
+      "log_odds_ratio": -0.7124490141868591,
+      "logits/chosen": -3.6752638816833496,
+      "logits/rejected": -3.662033796310425,
+      "logps/chosen": -1.0480401515960693,
+      "logps/rejected": -1.207778811454773,
+      "loss": 1.2034,
+      "nll_loss": 1.1693445444107056,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.05240201950073242,
+      "rewards/margins": 0.007986925542354584,
+      "rewards/rejected": -0.06038894131779671,
+      "step": 90
+    },
+    {
+      "epoch": 0.18083182640144665,
+      "grad_norm": 5.625,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 1.4625422954559326,
+      "log_odds_ratio": -0.2866816222667694,
+      "logits/chosen": -3.5862960815429688,
+      "logits/rejected": -3.631056547164917,
+      "logps/chosen": -0.616925060749054,
+      "logps/rejected": -1.2389600276947021,
+      "loss": 1.0472,
+      "nll_loss": 0.997981071472168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.030846258625388145,
+      "rewards/margins": 0.03110174462199211,
+      "rewards/rejected": -0.06194800138473511,
+      "step": 100
+    },
+    {
+      "epoch": 0.19891500904159132,
+      "grad_norm": 4.34375,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": 0.32260584831237793,
+      "log_odds_ratio": -0.7831681966781616,
+      "logits/chosen": -3.3512356281280518,
+      "logits/rejected": -3.317343235015869,
+      "logps/chosen": -0.8152042627334595,
+      "logps/rejected": -1.023037314414978,
+      "loss": 1.021,
+      "nll_loss": 0.8699595332145691,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.040760211646556854,
+      "rewards/margins": 0.010391652584075928,
+      "rewards/rejected": -0.05115186423063278,
+      "step": 110
+    },
+    {
+      "epoch": 0.21699819168173598,
+      "grad_norm": 4.46875,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": 0.06312495470046997,
+      "log_odds_ratio": -0.7186229228973389,
+      "logits/chosen": -3.7797393798828125,
+      "logits/rejected": -3.610599994659424,
+      "logps/chosen": -1.342027187347412,
+      "logps/rejected": -1.3699142932891846,
+      "loss": 1.2306,
+      "nll_loss": 1.2616217136383057,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0671013593673706,
+      "rewards/margins": 0.0013943525264039636,
+      "rewards/rejected": -0.06849571317434311,
+      "step": 120
+    },
+    {
+      "epoch": 0.23508137432188064,
+      "grad_norm": 3.40625,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": 0.3295042812824249,
+      "log_odds_ratio": -0.6021639704704285,
+      "logits/chosen": -3.6531271934509277,
+      "logits/rejected": -3.6605091094970703,
+      "logps/chosen": -1.1059828996658325,
+      "logps/rejected": -1.2798744440078735,
+      "loss": 1.0672,
+      "nll_loss": 1.0846788883209229,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.05529915168881416,
+      "rewards/margins": 0.008694576099514961,
+      "rewards/rejected": -0.06399371474981308,
+      "step": 130
+    },
+    {
+      "epoch": 0.25316455696202533,
+      "grad_norm": 3.921875,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": 0.005866944789886475,
+      "log_odds_ratio": -0.7822607755661011,
+      "logits/chosen": -3.4610767364501953,
+      "logits/rejected": -3.5588440895080566,
+      "logps/chosen": -1.0730628967285156,
+      "logps/rejected": -1.1153419017791748,
+      "loss": 1.0629,
+      "nll_loss": 0.9272462725639343,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.05365313962101936,
+      "rewards/margins": 0.0021139546297490597,
+      "rewards/rejected": -0.05576709657907486,
+      "step": 140
+    },
+    {
+      "epoch": 0.27124773960216997,
+      "grad_norm": 4.5625,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": 0.42503032088279724,
+      "log_odds_ratio": -0.8444482684135437,
+      "logits/chosen": -3.4901211261749268,
+      "logits/rejected": -3.6267387866973877,
+      "logps/chosen": -0.8993390798568726,
+      "logps/rejected": -1.1696584224700928,
+      "loss": 1.0394,
+      "nll_loss": 0.797405481338501,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.044966958463191986,
+      "rewards/margins": 0.013515966013073921,
+      "rewards/rejected": -0.05848292261362076,
+      "step": 150
+    },
+    {
+      "epoch": 0.28933092224231466,
+      "grad_norm": 4.75,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 0.5311036109924316,
+      "log_odds_ratio": -0.5203253030776978,
+      "logits/chosen": -3.2638163566589355,
+      "logits/rejected": -3.4405035972595215,
+      "logps/chosen": -1.0335638523101807,
+      "logps/rejected": -1.3805434703826904,
+      "loss": 1.1398,
+      "nll_loss": 1.0082361698150635,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.051678191870450974,
+      "rewards/margins": 0.017348987981677055,
+      "rewards/rejected": -0.06902717798948288,
+      "step": 160
+    },
+    {
+      "epoch": 0.3074141048824593,
+      "grad_norm": 4.125,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": 0.7853885293006897,
+      "log_odds_ratio": -0.5176795721054077,
+      "logits/chosen": -3.390453815460205,
+      "logits/rejected": -3.4858481884002686,
+      "logps/chosen": -1.0219676494598389,
+      "logps/rejected": -1.3882640600204468,
+      "loss": 1.0934,
+      "nll_loss": 0.9619676470756531,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.05109838768839836,
+      "rewards/margins": 0.018314816057682037,
+      "rewards/rejected": -0.0694132000207901,
+      "step": 170
+    },
+    {
+      "epoch": 0.325497287522604,
+      "grad_norm": 3.3125,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 0.2512182593345642,
+      "log_odds_ratio": -0.6799607276916504,
+      "logits/chosen": -3.8153457641601562,
+      "logits/rejected": -3.849152088165283,
+      "logps/chosen": -1.1906697750091553,
+      "logps/rejected": -1.386765718460083,
+      "loss": 1.0226,
+      "nll_loss": 1.331679105758667,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.059533488005399704,
+      "rewards/margins": 0.009804805740714073,
+      "rewards/rejected": -0.06933829933404922,
+      "step": 180
+    },
+    {
+      "epoch": 0.3435804701627486,
+      "grad_norm": 6.5625,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": 0.3922955393791199,
+      "log_odds_ratio": -0.5585602521896362,
+      "logits/chosen": -3.6088531017303467,
+      "logits/rejected": -3.6289172172546387,
+      "logps/chosen": -1.0057111978530884,
+      "logps/rejected": -1.2608778476715088,
+      "loss": 1.1235,
+      "nll_loss": 0.9645823240280151,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.05028555914759636,
+      "rewards/margins": 0.012758332304656506,
+      "rewards/rejected": -0.06304388493299484,
+      "step": 190
+    },
+    {
+      "epoch": 0.3616636528028933,
+      "grad_norm": 4.71875,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": 0.0035676597617566586,
+      "log_odds_ratio": -0.7242649793624878,
+      "logits/chosen": -3.624096632003784,
+      "logits/rejected": -3.6793160438537598,
+      "logps/chosen": -1.1755887269973755,
+      "logps/rejected": -1.200249195098877,
+      "loss": 1.0192,
+      "nll_loss": 0.9476919174194336,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05877942964434624,
+      "rewards/margins": 0.0012330322060734034,
+      "rewards/rejected": -0.06001245975494385,
+      "step": 200
+    },
+    {
+      "epoch": 0.379746835443038,
+      "grad_norm": 3.5625,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": 0.7048985362052917,
+      "log_odds_ratio": -0.6235366463661194,
+      "logits/chosen": -3.4578773975372314,
+      "logits/rejected": -3.6635658740997314,
+      "logps/chosen": -0.9914987683296204,
+      "logps/rejected": -1.1977266073226929,
+      "loss": 1.0038,
+      "nll_loss": 0.8347546458244324,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.04957494139671326,
+      "rewards/margins": 0.010311387479305267,
+      "rewards/rejected": -0.059886325150728226,
+      "step": 210
+    },
+    {
+      "epoch": 0.39783001808318263,
+      "grad_norm": 3.9375,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": -0.7274879217147827,
+      "log_odds_ratio": -1.5150617361068726,
+      "logits/chosen": -3.378265857696533,
+      "logits/rejected": -3.5164248943328857,
+      "logps/chosen": -1.3154796361923218,
+      "logps/rejected": -1.1096127033233643,
+      "loss": 1.0616,
+      "nll_loss": 0.9280896186828613,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.06577398627996445,
+      "rewards/margins": -0.010293351486325264,
+      "rewards/rejected": -0.055480629205703735,
+      "step": 220
+    },
+    {
+      "epoch": 0.4159132007233273,
+      "grad_norm": 4.96875,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": -0.029178833588957787,
+      "log_odds_ratio": -0.7706394195556641,
+      "logits/chosen": -3.5950729846954346,
+      "logits/rejected": -3.6548149585723877,
+      "logps/chosen": -1.2986187934875488,
+      "logps/rejected": -1.2398769855499268,
+      "loss": 1.0526,
+      "nll_loss": 1.0869041681289673,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.06493094563484192,
+      "rewards/margins": -0.0029370952397584915,
+      "rewards/rejected": -0.06199384853243828,
+      "step": 230
+    },
+    {
+      "epoch": 0.43399638336347196,
+      "grad_norm": 4.21875,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 0.16786038875579834,
+      "log_odds_ratio": -0.6685726046562195,
+      "logits/chosen": -3.688838243484497,
+      "logits/rejected": -3.749654769897461,
+      "logps/chosen": -1.374859094619751,
+      "logps/rejected": -1.5094839334487915,
+      "loss": 1.0482,
+      "nll_loss": 1.1397324800491333,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06874295324087143,
+      "rewards/margins": 0.0067312465980648994,
+      "rewards/rejected": -0.07547420263290405,
+      "step": 240
+    },
+    {
+      "epoch": 0.45207956600361665,
+      "grad_norm": 4.5,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 0.8509995341300964,
+      "log_odds_ratio": -0.5062636137008667,
+      "logits/chosen": -3.566221237182617,
+      "logits/rejected": -3.6127326488494873,
+      "logps/chosen": -0.9691341519355774,
+      "logps/rejected": -1.377387285232544,
+      "loss": 1.0446,
+      "nll_loss": 0.9793401956558228,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04845671355724335,
+      "rewards/margins": 0.02041265368461609,
+      "rewards/rejected": -0.06886936724185944,
+      "step": 250
+    },
+    {
+      "epoch": 0.4701627486437613,
+      "grad_norm": 2.640625,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 0.08310860395431519,
+      "log_odds_ratio": -0.720168948173523,
+      "logits/chosen": -3.4250590801239014,
+      "logits/rejected": -3.513597011566162,
+      "logps/chosen": -1.2283875942230225,
+      "logps/rejected": -1.3563426733016968,
+      "loss": 1.045,
+      "nll_loss": 1.0739576816558838,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06141937896609306,
+      "rewards/margins": 0.006397762335836887,
+      "rewards/rejected": -0.06781713664531708,
+      "step": 260
+    },
+    {
+      "epoch": 0.488245931283906,
+      "grad_norm": 3.734375,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 0.2674502432346344,
+      "log_odds_ratio": -0.6830952763557434,
+      "logits/chosen": -3.3503127098083496,
+      "logits/rejected": -3.4934239387512207,
+      "logps/chosen": -1.0327961444854736,
+      "logps/rejected": -1.2308813333511353,
+      "loss": 1.0153,
+      "nll_loss": 1.1966408491134644,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05163981392979622,
+      "rewards/margins": 0.009904252365231514,
+      "rewards/rejected": -0.061544060707092285,
+      "step": 270
+    },
+    {
+      "epoch": 0.5063291139240507,
+      "grad_norm": 7.75,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 0.9675368070602417,
+      "log_odds_ratio": -0.4681246876716614,
+      "logits/chosen": -2.9768073558807373,
+      "logits/rejected": -3.2737374305725098,
+      "logps/chosen": -0.9432920217514038,
+      "logps/rejected": -1.5204970836639404,
+      "loss": 1.0435,
+      "nll_loss": 0.8414273262023926,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.047164615243673325,
+      "rewards/margins": 0.02886023558676243,
+      "rewards/rejected": -0.07602483779191971,
+      "step": 280
+    },
+    {
+      "epoch": 0.5244122965641953,
+      "grad_norm": 4.3125,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 0.782937228679657,
+      "log_odds_ratio": -0.5052694082260132,
+      "logits/chosen": -3.4432454109191895,
+      "logits/rejected": -3.513373851776123,
+      "logps/chosen": -0.959903359413147,
+      "logps/rejected": -1.3499042987823486,
+      "loss": 1.034,
+      "nll_loss": 0.9008768200874329,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04799516871571541,
+      "rewards/margins": 0.019500048831105232,
+      "rewards/rejected": -0.06749521195888519,
+      "step": 290
+    },
+    {
+      "epoch": 0.5424954792043399,
+      "grad_norm": 3.328125,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": -0.06844829022884369,
+      "log_odds_ratio": -0.8229748606681824,
+      "logits/chosen": -3.6102004051208496,
+      "logits/rejected": -3.6592490673065186,
+      "logps/chosen": -1.3216373920440674,
+      "logps/rejected": -1.3158751726150513,
+      "loss": 1.0616,
+      "nll_loss": 1.050123929977417,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.06608187407255173,
+      "rewards/margins": -0.00028811878291890025,
+      "rewards/rejected": -0.06579376012086868,
+      "step": 300
+    },
+    {
+      "epoch": 0.5605786618444847,
+      "grad_norm": 3.796875,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 0.5723578333854675,
+      "log_odds_ratio": -0.5454049110412598,
+      "logits/chosen": -3.4600486755371094,
+      "logits/rejected": -3.52336049079895,
+      "logps/chosen": -0.9334306716918945,
+      "logps/rejected": -1.1987031698226929,
+      "loss": 1.0222,
+      "nll_loss": 0.9817255735397339,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.046671535819768906,
+      "rewards/margins": 0.013263625092804432,
+      "rewards/rejected": -0.05993515998125076,
+      "step": 310
+    },
+    {
+      "epoch": 0.5786618444846293,
+      "grad_norm": 3.53125,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 0.8614484667778015,
+      "log_odds_ratio": -0.4398893713951111,
+      "logits/chosen": -3.6512210369110107,
+      "logits/rejected": -3.76987886428833,
+      "logps/chosen": -0.744703471660614,
+      "logps/rejected": -1.220558762550354,
+      "loss": 1.0317,
+      "nll_loss": 0.9681469202041626,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03723517805337906,
+      "rewards/margins": 0.02379276230931282,
+      "rewards/rejected": -0.06102793663740158,
+      "step": 320
+    },
+    {
+      "epoch": 0.596745027124774,
+      "grad_norm": 3.703125,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 0.5696262121200562,
+      "log_odds_ratio": -0.8096323013305664,
+      "logits/chosen": -3.248673915863037,
+      "logits/rejected": -3.3223063945770264,
+      "logps/chosen": -1.0554739236831665,
+      "logps/rejected": -1.1397294998168945,
+      "loss": 1.0938,
+      "nll_loss": 1.0340250730514526,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.052773695439100266,
+      "rewards/margins": 0.004212772473692894,
+      "rewards/rejected": -0.05698646977543831,
+      "step": 330
+    },
+    {
+      "epoch": 0.6148282097649186,
+      "grad_norm": 4.46875,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 0.45444560050964355,
+      "log_odds_ratio": -0.556340754032135,
+      "logits/chosen": -3.499669313430786,
+      "logits/rejected": -3.5566048622131348,
+      "logps/chosen": -1.0938056707382202,
+      "logps/rejected": -1.4399141073226929,
+      "loss": 1.0092,
+      "nll_loss": 0.9630126953125,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05469028279185295,
+      "rewards/margins": 0.01730542816221714,
+      "rewards/rejected": -0.07199571281671524,
+      "step": 340
+    },
+    {
+      "epoch": 0.6329113924050633,
+      "grad_norm": 5.03125,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": -0.18927308917045593,
+      "log_odds_ratio": -1.1298543214797974,
+      "logits/chosen": -3.469010829925537,
+      "logits/rejected": -3.5768990516662598,
+      "logps/chosen": -1.1069543361663818,
+      "logps/rejected": -1.25455904006958,
+      "loss": 1.035,
+      "nll_loss": 1.04062819480896,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.055347710847854614,
+      "rewards/margins": 0.007380232214927673,
+      "rewards/rejected": -0.06272794306278229,
+      "step": 350
+    },
+    {
+      "epoch": 0.650994575045208,
+      "grad_norm": 4.125,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 1.1780674457550049,
+      "log_odds_ratio": -0.458904892206192,
+      "logits/chosen": -3.475041151046753,
+      "logits/rejected": -3.641082763671875,
+      "logps/chosen": -0.7767281532287598,
+      "logps/rejected": -1.3540589809417725,
+      "loss": 1.0549,
+      "nll_loss": 0.994460940361023,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03883640468120575,
+      "rewards/margins": 0.028866535052657127,
+      "rewards/rejected": -0.06770294159650803,
+      "step": 360
+    },
+    {
+      "epoch": 0.6690777576853526,
+      "grad_norm": 4.03125,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": -0.05644133687019348,
+      "log_odds_ratio": -0.7978056073188782,
+      "logits/chosen": -3.524318218231201,
+      "logits/rejected": -3.6522598266601562,
+      "logps/chosen": -1.1413499116897583,
+      "logps/rejected": -1.1321113109588623,
+      "loss": 0.9712,
+      "nll_loss": 0.9401092529296875,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.05706749111413956,
+      "rewards/margins": -0.00046192388981580734,
+      "rewards/rejected": -0.05660557001829147,
+      "step": 370
+    },
+    {
+      "epoch": 0.6871609403254972,
+      "grad_norm": 5.4375,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": -0.03757842257618904,
+      "log_odds_ratio": -0.9225370287895203,
+      "logits/chosen": -3.3350226879119873,
+      "logits/rejected": -3.4671382904052734,
+      "logps/chosen": -1.4029977321624756,
+      "logps/rejected": -1.209831714630127,
+      "loss": 1.0249,
+      "nll_loss": 1.2321926355361938,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.07014988362789154,
+      "rewards/margins": -0.009658301249146461,
+      "rewards/rejected": -0.06049158424139023,
+      "step": 380
+    },
+    {
+      "epoch": 0.705244122965642,
+      "grad_norm": 3.453125,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 0.38905367255210876,
+      "log_odds_ratio": -0.6244643330574036,
+      "logits/chosen": -3.652184009552002,
+      "logits/rejected": -3.6555867195129395,
+      "logps/chosen": -1.316056728363037,
+      "logps/rejected": -1.5403589010238647,
+      "loss": 0.9824,
+      "nll_loss": 1.1117520332336426,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06580284237861633,
+      "rewards/margins": 0.01121511310338974,
+      "rewards/rejected": -0.07701796293258667,
+      "step": 390
+    },
+    {
+      "epoch": 0.7233273056057866,
+      "grad_norm": 3.0,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 0.23738713562488556,
+      "log_odds_ratio": -0.6204290390014648,
+      "logits/chosen": -3.3568973541259766,
+      "logits/rejected": -3.591095447540283,
+      "logps/chosen": -1.1707801818847656,
+      "logps/rejected": -1.3350715637207031,
+      "loss": 1.071,
+      "nll_loss": 1.0477981567382812,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0585390105843544,
+      "rewards/margins": 0.008214573375880718,
+      "rewards/rejected": -0.0667535811662674,
+      "step": 400
+    },
+    {
+      "epoch": 0.7414104882459313,
+      "grad_norm": 3.5625,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 0.060622893273830414,
+      "log_odds_ratio": -0.7360556721687317,
+      "logits/chosen": -3.5519211292266846,
+      "logits/rejected": -3.641356945037842,
+      "logps/chosen": -1.2109748125076294,
+      "logps/rejected": -1.2431539297103882,
+      "loss": 1.0531,
+      "nll_loss": 1.063460111618042,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.06054873391985893,
+      "rewards/margins": 0.0016089621931314468,
+      "rewards/rejected": -0.06215769797563553,
+      "step": 410
+    },
+    {
+      "epoch": 0.759493670886076,
+      "grad_norm": 3.515625,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 0.27192121744155884,
+      "log_odds_ratio": -0.7382515668869019,
+      "logits/chosen": -3.4203600883483887,
+      "logits/rejected": -3.5196239948272705,
+      "logps/chosen": -0.8597354888916016,
+      "logps/rejected": -1.0601986646652222,
+      "loss": 1.0227,
+      "nll_loss": 0.8366381525993347,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.04298677295446396,
+      "rewards/margins": 0.010023154318332672,
+      "rewards/rejected": -0.05300993472337723,
+      "step": 420
+    },
+    {
+      "epoch": 0.7775768535262206,
+      "grad_norm": 4.6875,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 0.00424801092594862,
+      "log_odds_ratio": -0.7706916332244873,
+      "logits/chosen": -3.656468629837036,
+      "logits/rejected": -3.656891345977783,
+      "logps/chosen": -1.2077748775482178,
+      "logps/rejected": -1.210986852645874,
+      "loss": 0.9992,
+      "nll_loss": 1.0321767330169678,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.06038873642683029,
+      "rewards/margins": 0.00016060545749496669,
+      "rewards/rejected": -0.06054934114217758,
+      "step": 430
+    },
+    {
+      "epoch": 0.7956600361663653,
+      "grad_norm": 3.203125,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 0.15838010609149933,
+      "log_odds_ratio": -0.706937849521637,
+      "logits/chosen": -3.6181061267852783,
+      "logits/rejected": -3.649228572845459,
+      "logps/chosen": -1.0764718055725098,
+      "logps/rejected": -1.1391253471374512,
+      "loss": 1.0074,
+      "nll_loss": 1.0357991456985474,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.05382359027862549,
+      "rewards/margins": 0.003132682293653488,
+      "rewards/rejected": -0.05695626884698868,
+      "step": 440
+    },
+    {
+      "epoch": 0.8137432188065099,
+      "grad_norm": 3.609375,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 0.6918424367904663,
+      "log_odds_ratio": -0.5028156042098999,
+      "logits/chosen": -3.3800606727600098,
+      "logits/rejected": -3.511284351348877,
+      "logps/chosen": -0.7713460326194763,
+      "logps/rejected": -1.1150473356246948,
+      "loss": 1.0192,
+      "nll_loss": 0.9132420420646667,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.038567304611206055,
+      "rewards/margins": 0.017185065895318985,
+      "rewards/rejected": -0.05575236678123474,
+      "step": 450
+    },
+    {
+      "epoch": 0.8318264014466547,
+      "grad_norm": 4.46875,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 0.9962455630302429,
+      "log_odds_ratio": -0.4283738136291504,
+      "logits/chosen": -3.4526374340057373,
+      "logits/rejected": -3.6031575202941895,
+      "logps/chosen": -0.7248684167861938,
+      "logps/rejected": -1.235801100730896,
+      "loss": 0.9926,
+      "nll_loss": 0.7733514308929443,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03624342009425163,
+      "rewards/margins": 0.025546634569764137,
+      "rewards/rejected": -0.06179005652666092,
+      "step": 460
+    },
+    {
+      "epoch": 0.8499095840867993,
+      "grad_norm": 4.75,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 0.4537855088710785,
+      "log_odds_ratio": -0.5337843894958496,
+      "logits/chosen": -3.5173232555389404,
+      "logits/rejected": -3.6324150562286377,
+      "logps/chosen": -0.9342852830886841,
+      "logps/rejected": -1.2018686532974243,
+      "loss": 1.0222,
+      "nll_loss": 0.8243517875671387,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04671426862478256,
+      "rewards/margins": 0.013379162177443504,
+      "rewards/rejected": -0.06009342521429062,
+      "step": 470
+    },
+    {
+      "epoch": 0.8679927667269439,
+      "grad_norm": 5.09375,
+      "learning_rate": 2.2821773229381924e-06,
+      "log_odds_chosen": 1.8265974521636963,
+      "log_odds_ratio": -0.33439093828201294,
+      "logits/chosen": -3.333242416381836,
+      "logits/rejected": -3.549222469329834,
+      "logps/chosen": -0.7172473073005676,
+      "logps/rejected": -1.4388408660888672,
+      "loss": 0.9362,
+      "nll_loss": 0.9278656244277954,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03586236760020256,
+      "rewards/margins": 0.036079682409763336,
+      "rewards/rejected": -0.0719420462846756,
+      "step": 480
+    },
+    {
+      "epoch": 0.8860759493670886,
+      "grad_norm": 4.21875,
+      "learning_rate": 2.2587697572631284e-06,
+      "log_odds_chosen": 0.9842466115951538,
+      "log_odds_ratio": -0.42221134901046753,
+      "logits/chosen": -3.305194139480591,
+      "logits/rejected": -3.4940364360809326,
+      "logps/chosen": -0.8084889650344849,
+      "logps/rejected": -1.2992254495620728,
+      "loss": 1.0103,
+      "nll_loss": 0.9417991638183594,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.040424447506666183,
+      "rewards/margins": 0.024536821991205215,
+      "rewards/rejected": -0.0649612694978714,
+      "step": 490
+    },
+    {
+      "epoch": 0.9041591320072333,
+      "grad_norm": 3.40625,
+      "learning_rate": 2.23606797749979e-06,
+      "log_odds_chosen": 0.06368019431829453,
+      "log_odds_ratio": -0.6845844388008118,
+      "logits/chosen": -3.6345558166503906,
+      "logits/rejected": -3.47111177444458,
+      "logps/chosen": -1.1341079473495483,
+      "logps/rejected": -1.1745169162750244,
+      "loss": 1.062,
+      "nll_loss": 1.0511561632156372,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.05670539662241936,
+      "rewards/margins": 0.0020204458851367235,
+      "rewards/rejected": -0.05872584506869316,
+      "step": 500
+    },
+    {
+      "epoch": 0.9222423146473779,
+      "grad_norm": 4.15625,
+      "learning_rate": 2.2140372138502386e-06,
+      "log_odds_chosen": 1.0155892372131348,
+      "log_odds_ratio": -0.4495448172092438,
+      "logits/chosen": -3.242302656173706,
+      "logits/rejected": -3.3566043376922607,
+      "logps/chosen": -0.6464399099349976,
+      "logps/rejected": -1.0815407037734985,
+      "loss": 1.0125,
+      "nll_loss": 0.6781537532806396,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.032321996986866,
+      "rewards/margins": 0.02175503969192505,
+      "rewards/rejected": -0.054077040404081345,
+      "step": 510
+    },
+    {
+      "epoch": 0.9403254972875226,
+      "grad_norm": 3.5625,
+      "learning_rate": 2.1926450482675734e-06,
+      "log_odds_chosen": 0.5763598084449768,
+      "log_odds_ratio": -0.659260630607605,
+      "logits/chosen": -3.5601630210876465,
+      "logits/rejected": -3.527803897857666,
+      "logps/chosen": -1.2068111896514893,
+      "logps/rejected": -1.3772714138031006,
+      "loss": 1.0074,
+      "nll_loss": 1.2287429571151733,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06034056469798088,
+      "rewards/margins": 0.008523015305399895,
+      "rewards/rejected": -0.06886357814073563,
+      "step": 520
+    },
+    {
+      "epoch": 0.9584086799276673,
+      "grad_norm": 3.828125,
+      "learning_rate": 2.1718612138153473e-06,
+      "log_odds_chosen": 0.2709532380104065,
+      "log_odds_ratio": -0.6409106850624084,
+      "logits/chosen": -3.741631269454956,
+      "logits/rejected": -3.731243133544922,
+      "logps/chosen": -1.1519293785095215,
+      "logps/rejected": -1.3367632627487183,
+      "loss": 1.0247,
+      "nll_loss": 1.2208020687103271,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.05759647488594055,
+      "rewards/margins": 0.009241694584488869,
+      "rewards/rejected": -0.06683816015720367,
+      "step": 530
+    },
+    {
+      "epoch": 0.976491862567812,
+      "grad_norm": 3.203125,
+      "learning_rate": 2.151657414559676e-06,
+      "log_odds_chosen": 0.8211873173713684,
+      "log_odds_ratio": -0.5390130877494812,
+      "logits/chosen": -3.598482608795166,
+      "logits/rejected": -3.682385206222534,
+      "logps/chosen": -1.2253609895706177,
+      "logps/rejected": -1.6024954319000244,
+      "loss": 1.0344,
+      "nll_loss": 1.1462525129318237,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.061268050223588943,
+      "rewards/margins": 0.018856722861528397,
+      "rewards/rejected": -0.08012477308511734,
+      "step": 540
+    },
+    {
+      "epoch": 0.9945750452079566,
+      "grad_norm": 7.03125,
+      "learning_rate": 2.132007163556104e-06,
+      "log_odds_chosen": 1.1987555027008057,
+      "log_odds_ratio": -0.3731897473335266,
+      "logits/chosen": -3.2686665058135986,
+      "logits/rejected": -3.5461959838867188,
+      "logps/chosen": -0.7877604961395264,
+      "logps/rejected": -1.328150749206543,
+      "loss": 1.017,
+      "nll_loss": 0.808586597442627,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.0393880195915699,
+      "rewards/margins": 0.027019521221518517,
+      "rewards/rejected": -0.06640754640102386,
+      "step": 550
+    },
+    {
+      "epoch": 1.0,
+      "step": 553,
+      "total_flos": 0.0,
+      "train_loss": 1.340359150177747,
+      "train_runtime": 13809.2478,
+      "train_samples_per_second": 0.64,
+      "train_steps_per_second": 0.04
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 553,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:895133f0e6dcbcb4ee4eb5326d651ac40e358641b7cfb6ef761a68d32f339e64
+size 5496