Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: mistral-7b-orpo-capybara-reproduction
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral-7b-orpo-capybara-reproduction
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_ratio: 0.1
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.1+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 32001,
+  "<|im_start|>": 32000
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.5470885709871219,
+    "train_runtime": 7181.4173,
+    "train_samples": 7563,
+    "train_samples_per_second": 3.159,
+    "train_steps_per_second": 0.099
+}

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 32000,
+  "eos_token_id": 32001,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 32001,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32002
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 32000,
+  "eos_token_id": 32001,
+  "pad_token_id": 32001,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f4f713aa46386534737d41e3691dac2d6d0df04f5124a9120614532e1f9d38a
+size 4943178720

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02f36faeb11f6fdecf4caded61e7a3bb97f8b5f0286774a579dd9942876f26f4
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e44a886ec72238332a094e84aa693d4e8d59d8f4ffebda06c477b6e73ae142d9
+size 4540532728

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483496960
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Mar28_13-16-20_f3e5e3df8235/events.out.tfevents.1711631888.f3e5e3df8235.8414.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21d9ebeecf54833b28688fe18a75d3a90cebcd44bb710efc81f3fb33e9b448f0
+size 13529

runs/Mar28_13-38-07_f3e5e3df8235/events.out.tfevents.1711633111.f3e5e3df8235.17179.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8648122c588b5998c4bb3775d21ebbc5dfc4231f1dfc6add747f2c444d1d183e
+size 66312

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,62 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.5470885709871219,
+    "train_runtime": 7181.4173,
+    "train_samples": 7563,
+    "train_samples_per_second": 3.159,
+    "train_steps_per_second": 0.099
+}

trainer_state.json

@@ -0,0 +1,1308 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 711,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "grad_norm": 49.48526136809559,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.14267238974571228,
+      "log_odds_ratio": -0.6910250782966614,
+      "logits/chosen": -2.8734333515167236,
+      "logits/rejected": -2.8792202472686768,
+      "logps/chosen": -1.2169833183288574,
+      "logps/rejected": -1.3137216567993164,
+      "loss": 1.1778,
+      "nll_loss": 1.1760419607162476,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.06084916740655899,
+      "rewards/margins": 0.004836911801248789,
+      "rewards/rejected": -0.06568608433008194,
+      "step": 10
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 13.42628442764115,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": 0.2992035746574402,
+      "log_odds_ratio": -0.5965026617050171,
+      "logits/chosen": -2.9965202808380127,
+      "logits/rejected": -2.9929251670837402,
+      "logps/chosen": -0.9654132723808289,
+      "logps/rejected": -1.1591482162475586,
+      "loss": 1.0,
+      "nll_loss": 0.9463264346122742,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04827066510915756,
+      "rewards/margins": 0.009686745703220367,
+      "rewards/rejected": -0.05795741081237793,
+      "step": 20
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 9.276480900326913,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": 0.35957902669906616,
+      "log_odds_ratio": -0.5845057368278503,
+      "logits/chosen": -3.0192956924438477,
+      "logits/rejected": -3.018825054168701,
+      "logps/chosen": -0.8531354665756226,
+      "logps/rejected": -1.0771838426589966,
+      "loss": 0.9246,
+      "nll_loss": 0.8811793327331543,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04265677183866501,
+      "rewards/margins": 0.011202414520084858,
+      "rewards/rejected": -0.05385918542742729,
+      "step": 30
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 8.627451557441312,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 0.25122153759002686,
+      "log_odds_ratio": -0.6361113786697388,
+      "logits/chosen": -2.915778160095215,
+      "logits/rejected": -2.919457197189331,
+      "logps/chosen": -0.8648379445075989,
+      "logps/rejected": -0.9924612045288086,
+      "loss": 0.974,
+      "nll_loss": 0.9005156755447388,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.04324189946055412,
+      "rewards/margins": 0.0063811675645411015,
+      "rewards/rejected": -0.04962306469678879,
+      "step": 40
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 10.358595971591946,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 0.20706939697265625,
+      "log_odds_ratio": -0.6479698419570923,
+      "logits/chosen": -2.931480884552002,
+      "logits/rejected": -2.941484212875366,
+      "logps/chosen": -0.9367877244949341,
+      "logps/rejected": -1.0564123392105103,
+      "loss": 0.9112,
+      "nll_loss": 0.9363681674003601,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.046839382499456406,
+      "rewards/margins": 0.005981230176985264,
+      "rewards/rejected": -0.052820611745119095,
+      "step": 50
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 10.184305920593003,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 0.274976909160614,
+      "log_odds_ratio": -0.6372017860412598,
+      "logits/chosen": -2.887624979019165,
+      "logits/rejected": -2.902449369430542,
+      "logps/chosen": -0.8582770228385925,
+      "logps/rejected": -1.012649416923523,
+      "loss": 0.9268,
+      "nll_loss": 0.9307074546813965,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04291384667158127,
+      "rewards/margins": 0.007718615233898163,
+      "rewards/rejected": -0.05063246563076973,
+      "step": 60
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 10.247661529064816,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 0.29217422008514404,
+      "log_odds_ratio": -0.6399118304252625,
+      "logits/chosen": -2.9243764877319336,
+      "logits/rejected": -2.9292612075805664,
+      "logps/chosen": -0.8583773374557495,
+      "logps/rejected": -1.0340784788131714,
+      "loss": 0.9441,
+      "nll_loss": 0.9250191450119019,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.042918868362903595,
+      "rewards/margins": 0.008785055950284004,
+      "rewards/rejected": -0.05170391872525215,
+      "step": 70
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 8.823827784258985,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 0.33998143672943115,
+      "log_odds_ratio": -0.6087978482246399,
+      "logits/chosen": -2.901061773300171,
+      "logits/rejected": -2.909398078918457,
+      "logps/chosen": -0.8597785234451294,
+      "logps/rejected": -1.0702154636383057,
+      "loss": 0.9265,
+      "nll_loss": 0.883587658405304,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04298892617225647,
+      "rewards/margins": 0.010521845892071724,
+      "rewards/rejected": -0.053510766476392746,
+      "step": 80
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 9.3837801640132,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 0.32301634550094604,
+      "log_odds_ratio": -0.645144522190094,
+      "logits/chosen": -2.9571635723114014,
+      "logits/rejected": -2.9651668071746826,
+      "logps/chosen": -0.8539797067642212,
+      "logps/rejected": -1.0550405979156494,
+      "loss": 0.9336,
+      "nll_loss": 0.9078629612922668,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04269898682832718,
+      "rewards/margins": 0.010053041391074657,
+      "rewards/rejected": -0.05275202915072441,
+      "step": 90
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 8.451957287235773,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 0.27936068177223206,
+      "log_odds_ratio": -0.619978129863739,
+      "logits/chosen": -2.9700396060943604,
+      "logits/rejected": -2.9722213745117188,
+      "logps/chosen": -0.892767608165741,
+      "logps/rejected": -1.0630358457565308,
+      "loss": 0.9536,
+      "nll_loss": 0.9281327128410339,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04463838413357735,
+      "rewards/margins": 0.008513410575687885,
+      "rewards/rejected": -0.05315179377794266,
+      "step": 100
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 10.228379998303796,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": 0.18160709738731384,
+      "log_odds_ratio": -0.7056410908699036,
+      "logits/chosen": -2.980069637298584,
+      "logits/rejected": -2.9743785858154297,
+      "logps/chosen": -0.9338927268981934,
+      "logps/rejected": -1.0694698095321655,
+      "loss": 0.949,
+      "nll_loss": 0.9810144305229187,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.046694640070199966,
+      "rewards/margins": 0.006778855808079243,
+      "rewards/rejected": -0.053473494946956635,
+      "step": 110
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 15.501078885228303,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": 0.3752610683441162,
+      "log_odds_ratio": -0.5817548036575317,
+      "logits/chosen": -2.8941051959991455,
+      "logits/rejected": -2.898369073867798,
+      "logps/chosen": -0.8418710827827454,
+      "logps/rejected": -1.0447475910186768,
+      "loss": 0.9253,
+      "nll_loss": 0.908086895942688,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04209355264902115,
+      "rewards/margins": 0.010143828578293324,
+      "rewards/rejected": -0.0522373802959919,
+      "step": 120
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 8.598131028345922,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": 0.2913978397846222,
+      "log_odds_ratio": -0.6210304498672485,
+      "logits/chosen": -2.8913044929504395,
+      "logits/rejected": -2.900383234024048,
+      "logps/chosen": -0.9610312581062317,
+      "logps/rejected": -1.1445128917694092,
+      "loss": 0.979,
+      "nll_loss": 1.0131185054779053,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.048051562160253525,
+      "rewards/margins": 0.00917408149689436,
+      "rewards/rejected": -0.05722564458847046,
+      "step": 130
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 8.665950821616725,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": 0.31699851155281067,
+      "log_odds_ratio": -0.6054994463920593,
+      "logits/chosen": -2.866349697113037,
+      "logits/rejected": -2.872480869293213,
+      "logps/chosen": -0.8127169609069824,
+      "logps/rejected": -1.005972146987915,
+      "loss": 0.9264,
+      "nll_loss": 0.8691350817680359,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.040635846555233,
+      "rewards/margins": 0.009662764146924019,
+      "rewards/rejected": -0.05029861256480217,
+      "step": 140
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 7.610661708462692,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": 0.26200634241104126,
+      "log_odds_ratio": -0.635285496711731,
+      "logits/chosen": -2.813951253890991,
+      "logits/rejected": -2.831160306930542,
+      "logps/chosen": -0.8999479413032532,
+      "logps/rejected": -1.0443265438079834,
+      "loss": 0.9319,
+      "nll_loss": 0.9246827363967896,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.04499739781022072,
+      "rewards/margins": 0.0072189331986010075,
+      "rewards/rejected": -0.05221632868051529,
+      "step": 150
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 7.926717564356759,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 0.4248151183128357,
+      "log_odds_ratio": -0.5570122599601746,
+      "logits/chosen": -2.8344883918762207,
+      "logits/rejected": -2.8441624641418457,
+      "logps/chosen": -0.8424416780471802,
+      "logps/rejected": -1.1050217151641846,
+      "loss": 0.9371,
+      "nll_loss": 0.8941017985343933,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.04212208464741707,
+      "rewards/margins": 0.01312900148332119,
+      "rewards/rejected": -0.05525108426809311,
+      "step": 160
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 8.768416551022222,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": 0.280124694108963,
+      "log_odds_ratio": -0.6224346160888672,
+      "logits/chosen": -2.8287386894226074,
+      "logits/rejected": -2.83266019821167,
+      "logps/chosen": -0.8510664105415344,
+      "logps/rejected": -1.0343472957611084,
+      "loss": 0.9355,
+      "nll_loss": 0.8820816278457642,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04255332052707672,
+      "rewards/margins": 0.009164043702185154,
+      "rewards/rejected": -0.0517173632979393,
+      "step": 170
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 7.46076847939469,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 0.17997188866138458,
+      "log_odds_ratio": -0.6717725992202759,
+      "logits/chosen": -2.8428828716278076,
+      "logits/rejected": -2.8347556591033936,
+      "logps/chosen": -0.9121964573860168,
+      "logps/rejected": -0.9958919286727905,
+      "loss": 0.946,
+      "nll_loss": 0.9758418202400208,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04560982435941696,
+      "rewards/margins": 0.004184772726148367,
+      "rewards/rejected": -0.04979459568858147,
+      "step": 180
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 7.680949233012938,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": 0.35985735058784485,
+      "log_odds_ratio": -0.5938600301742554,
+      "logits/chosen": -2.859172821044922,
+      "logits/rejected": -2.8654944896698,
+      "logps/chosen": -0.847649097442627,
+      "logps/rejected": -1.0266352891921997,
+      "loss": 0.9285,
+      "nll_loss": 0.8830558657646179,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04238245636224747,
+      "rewards/margins": 0.008949300274252892,
+      "rewards/rejected": -0.05133175849914551,
+      "step": 190
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 9.078983143397938,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": 0.4591992497444153,
+      "log_odds_ratio": -0.5660054087638855,
+      "logits/chosen": -2.9353525638580322,
+      "logits/rejected": -2.9381651878356934,
+      "logps/chosen": -0.9107562303543091,
+      "logps/rejected": -1.1601167917251587,
+      "loss": 0.943,
+      "nll_loss": 1.0148041248321533,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.045537810772657394,
+      "rewards/margins": 0.012468031607568264,
+      "rewards/rejected": -0.05800584703683853,
+      "step": 200
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 8.44546881476589,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": 0.29461508989334106,
+      "log_odds_ratio": -0.6495056748390198,
+      "logits/chosen": -2.90964412689209,
+      "logits/rejected": -2.9097769260406494,
+      "logps/chosen": -0.8788071870803833,
+      "logps/rejected": -1.0280598402023315,
+      "loss": 0.9482,
+      "nll_loss": 0.9459528923034668,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.043940357863903046,
+      "rewards/margins": 0.007462637964636087,
+      "rewards/rejected": -0.051402997225522995,
+      "step": 210
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 8.193251853529034,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": 0.4032968580722809,
+      "log_odds_ratio": -0.568686842918396,
+      "logits/chosen": -2.9146342277526855,
+      "logits/rejected": -2.9228734970092773,
+      "logps/chosen": -0.8045447468757629,
+      "logps/rejected": -1.03438401222229,
+      "loss": 0.9203,
+      "nll_loss": 0.8637560606002808,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04022723808884621,
+      "rewards/margins": 0.011491960845887661,
+      "rewards/rejected": -0.05171919986605644,
+      "step": 220
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 8.324361077294697,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": 0.36563342809677124,
+      "log_odds_ratio": -0.5865170955657959,
+      "logits/chosen": -2.9637176990509033,
+      "logits/rejected": -2.9688830375671387,
+      "logps/chosen": -0.8463845252990723,
+      "logps/rejected": -1.061700701713562,
+      "loss": 0.9112,
+      "nll_loss": 0.9032902717590332,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04231923073530197,
+      "rewards/margins": 0.010765810497105122,
+      "rewards/rejected": -0.05308504030108452,
+      "step": 230
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 9.173894745062437,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 0.5646923184394836,
+      "log_odds_ratio": -0.550338625907898,
+      "logits/chosen": -2.9580464363098145,
+      "logits/rejected": -2.9572625160217285,
+      "logps/chosen": -0.7748560309410095,
+      "logps/rejected": -1.0799105167388916,
+      "loss": 0.8136,
+      "nll_loss": 0.8241481781005859,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.038742803037166595,
+      "rewards/margins": 0.015252722427248955,
+      "rewards/rejected": -0.0539955273270607,
+      "step": 240
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 9.099732749463024,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 1.4823018312454224,
+      "log_odds_ratio": -0.2659846544265747,
+      "logits/chosen": -2.9596664905548096,
+      "logits/rejected": -2.9568517208099365,
+      "logps/chosen": -0.47842296957969666,
+      "logps/rejected": -1.2101627588272095,
+      "loss": 0.5233,
+      "nll_loss": 0.5237408876419067,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.023921148851513863,
+      "rewards/margins": 0.03658699244260788,
+      "rewards/rejected": -0.06050813943147659,
+      "step": 250
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 9.707924912075288,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 1.2276315689086914,
+      "log_odds_ratio": -0.337185263633728,
+      "logits/chosen": -2.9387476444244385,
+      "logits/rejected": -2.947472333908081,
+      "logps/chosen": -0.507585346698761,
+      "logps/rejected": -1.0944607257843018,
+      "loss": 0.5243,
+      "nll_loss": 0.5015721321105957,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.02537926472723484,
+      "rewards/margins": 0.02934376522898674,
+      "rewards/rejected": -0.05472303181886673,
+      "step": 260
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 9.72287079567634,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 1.6538162231445312,
+      "log_odds_ratio": -0.23319308459758759,
+      "logits/chosen": -3.0101027488708496,
+      "logits/rejected": -3.0216686725616455,
+      "logps/chosen": -0.4229302406311035,
+      "logps/rejected": -1.2452623844146729,
+      "loss": 0.4944,
+      "nll_loss": 0.4760821759700775,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.021146511659026146,
+      "rewards/margins": 0.04111660644412041,
+      "rewards/rejected": -0.0622631199657917,
+      "step": 270
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 8.91121542892108,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 1.5969759225845337,
+      "log_odds_ratio": -0.24297292530536652,
+      "logits/chosen": -2.9810919761657715,
+      "logits/rejected": -2.984095335006714,
+      "logps/chosen": -0.4323214888572693,
+      "logps/rejected": -1.1894398927688599,
+      "loss": 0.4899,
+      "nll_loss": 0.4894355237483978,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.021616075187921524,
+      "rewards/margins": 0.03785592317581177,
+      "rewards/rejected": -0.05947200208902359,
+      "step": 280
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 9.172297878492826,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 1.5589730739593506,
+      "log_odds_ratio": -0.2480790615081787,
+      "logits/chosen": -2.936096668243408,
+      "logits/rejected": -2.9368419647216797,
+      "logps/chosen": -0.4527331292629242,
+      "logps/rejected": -1.1788005828857422,
+      "loss": 0.489,
+      "nll_loss": 0.48634010553359985,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.0226366575807333,
+      "rewards/margins": 0.03630336746573448,
+      "rewards/rejected": -0.05894003063440323,
+      "step": 290
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 9.341330783727024,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": 1.5156631469726562,
+      "log_odds_ratio": -0.2381705492734909,
+      "logits/chosen": -2.949042320251465,
+      "logits/rejected": -2.934840202331543,
+      "logps/chosen": -0.43807506561279297,
+      "logps/rejected": -1.1634423732757568,
+      "loss": 0.5031,
+      "nll_loss": 0.4857659935951233,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.0219037514179945,
+      "rewards/margins": 0.03626837208867073,
+      "rewards/rejected": -0.05817211791872978,
+      "step": 300
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 9.51833987789547,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 1.4176826477050781,
+      "log_odds_ratio": -0.281759113073349,
+      "logits/chosen": -2.953523874282837,
+      "logits/rejected": -2.954210042953491,
+      "logps/chosen": -0.47912734746932983,
+      "logps/rejected": -1.2172271013259888,
+      "loss": 0.5149,
+      "nll_loss": 0.4759243428707123,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.02395637147128582,
+      "rewards/margins": 0.03690498322248459,
+      "rewards/rejected": -0.060861360281705856,
+      "step": 310
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 9.733927381742513,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 1.33830726146698,
+      "log_odds_ratio": -0.2881857454776764,
+      "logits/chosen": -3.0379624366760254,
+      "logits/rejected": -3.0340828895568848,
+      "logps/chosen": -0.5138579607009888,
+      "logps/rejected": -1.1911945343017578,
+      "loss": 0.5378,
+      "nll_loss": 0.5391749739646912,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.025692898780107498,
+      "rewards/margins": 0.03386683017015457,
+      "rewards/rejected": -0.05955972522497177,
+      "step": 320
+    },
+    {
+      "epoch": 1.39,
+      "grad_norm": 8.811470336283001,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 1.485058307647705,
+      "log_odds_ratio": -0.26495322585105896,
+      "logits/chosen": -2.9619758129119873,
+      "logits/rejected": -2.954977035522461,
+      "logps/chosen": -0.440424382686615,
+      "logps/rejected": -1.131378412246704,
+      "loss": 0.4785,
+      "nll_loss": 0.4669272303581238,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0220212172716856,
+      "rewards/margins": 0.034547705203294754,
+      "rewards/rejected": -0.056568920612335205,
+      "step": 330
+    },
+    {
+      "epoch": 1.43,
+      "grad_norm": 8.015826159950372,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 1.5126841068267822,
+      "log_odds_ratio": -0.25113973021507263,
+      "logits/chosen": -2.979628801345825,
+      "logits/rejected": -2.9764034748077393,
+      "logps/chosen": -0.42568206787109375,
+      "logps/rejected": -1.1240874528884888,
+      "loss": 0.5006,
+      "nll_loss": 0.46929699182510376,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.021284103393554688,
+      "rewards/margins": 0.03492026776075363,
+      "rewards/rejected": -0.05620437115430832,
+      "step": 340
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 8.45163306773323,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": 1.3055660724639893,
+      "log_odds_ratio": -0.3173283338546753,
+      "logits/chosen": -2.961458683013916,
+      "logits/rejected": -2.9598679542541504,
+      "logps/chosen": -0.5269508957862854,
+      "logps/rejected": -1.1968458890914917,
+      "loss": 0.4854,
+      "nll_loss": 0.5489671230316162,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.02634754404425621,
+      "rewards/margins": 0.033494751900434494,
+      "rewards/rejected": -0.059842299669981,
+      "step": 350
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 8.682519620016398,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 1.644923210144043,
+      "log_odds_ratio": -0.23544283211231232,
+      "logits/chosen": -2.930689573287964,
+      "logits/rejected": -2.928385019302368,
+      "logps/chosen": -0.4198189675807953,
+      "logps/rejected": -1.2138855457305908,
+      "loss": 0.5002,
+      "nll_loss": 0.45420289039611816,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.020990949124097824,
+      "rewards/margins": 0.03970332816243172,
+      "rewards/rejected": -0.06069428473711014,
+      "step": 360
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 9.273291108083583,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": 1.5730557441711426,
+      "log_odds_ratio": -0.2532302439212799,
+      "logits/chosen": -2.9642691612243652,
+      "logits/rejected": -2.9628634452819824,
+      "logps/chosen": -0.4353133738040924,
+      "logps/rejected": -1.1966689825057983,
+      "loss": 0.5056,
+      "nll_loss": 0.48967617750167847,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.02176566980779171,
+      "rewards/margins": 0.0380677804350853,
+      "rewards/rejected": -0.05983344465494156,
+      "step": 370
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 8.84508067219799,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": 1.6267578601837158,
+      "log_odds_ratio": -0.24944674968719482,
+      "logits/chosen": -2.934312582015991,
+      "logits/rejected": -2.9307126998901367,
+      "logps/chosen": -0.44339004158973694,
+      "logps/rejected": -1.2226251363754272,
+      "loss": 0.4946,
+      "nll_loss": 0.44669169187545776,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.022169504314661026,
+      "rewards/margins": 0.038961753249168396,
+      "rewards/rejected": -0.06113125756382942,
+      "step": 380
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 7.973489385408531,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 1.4293986558914185,
+      "log_odds_ratio": -0.26934128999710083,
+      "logits/chosen": -2.927151918411255,
+      "logits/rejected": -2.9325613975524902,
+      "logps/chosen": -0.48316580057144165,
+      "logps/rejected": -1.1627215147018433,
+      "loss": 0.5043,
+      "nll_loss": 0.5102673172950745,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.024158289656043053,
+      "rewards/margins": 0.03397778421640396,
+      "rewards/rejected": -0.05813607573509216,
+      "step": 390
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 8.824910828626702,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 1.6508582830429077,
+      "log_odds_ratio": -0.24078328907489777,
+      "logits/chosen": -2.9304981231689453,
+      "logits/rejected": -2.9384522438049316,
+      "logps/chosen": -0.4645829200744629,
+      "logps/rejected": -1.2513707876205444,
+      "loss": 0.528,
+      "nll_loss": 0.5313082933425903,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.023229148238897324,
+      "rewards/margins": 0.03933940455317497,
+      "rewards/rejected": -0.0625685527920723,
+      "step": 400
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 9.94593557333145,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 1.5409417152404785,
+      "log_odds_ratio": -0.26054129004478455,
+      "logits/chosen": -2.8831980228424072,
+      "logits/rejected": -2.881943702697754,
+      "logps/chosen": -0.42643943428993225,
+      "logps/rejected": -1.1698906421661377,
+      "loss": 0.5037,
+      "nll_loss": 0.464557409286499,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.021321970969438553,
+      "rewards/margins": 0.03717255964875221,
+      "rewards/rejected": -0.05849452689290047,
+      "step": 410
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 8.290147408251284,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 1.3443034887313843,
+      "log_odds_ratio": -0.3111021816730499,
+      "logits/chosen": -2.867410182952881,
+      "logits/rejected": -2.867697238922119,
+      "logps/chosen": -0.5130647420883179,
+      "logps/rejected": -1.2247389554977417,
+      "loss": 0.4997,
+      "nll_loss": 0.5347863435745239,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.025653237476944923,
+      "rewards/margins": 0.03558371216058731,
+      "rewards/rejected": -0.06123695522546768,
+      "step": 420
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 10.095422116087253,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 1.538722276687622,
+      "log_odds_ratio": -0.25320082902908325,
+      "logits/chosen": -2.9051260948181152,
+      "logits/rejected": -2.907191038131714,
+      "logps/chosen": -0.49453073740005493,
+      "logps/rejected": -1.2816953659057617,
+      "loss": 0.5557,
+      "nll_loss": 0.5553343892097473,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.024726541712880135,
+      "rewards/margins": 0.0393582321703434,
+      "rewards/rejected": -0.06408476829528809,
+      "step": 430
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 9.482272989808482,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 1.701960802078247,
+      "log_odds_ratio": -0.23092034459114075,
+      "logits/chosen": -2.88254976272583,
+      "logits/rejected": -2.880296230316162,
+      "logps/chosen": -0.41986674070358276,
+      "logps/rejected": -1.1859886646270752,
+      "loss": 0.5171,
+      "nll_loss": 0.4521302282810211,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.020993337035179138,
+      "rewards/margins": 0.0383060947060585,
+      "rewards/rejected": -0.05929943174123764,
+      "step": 440
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 10.287479798932107,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 1.5226378440856934,
+      "log_odds_ratio": -0.26804596185684204,
+      "logits/chosen": -2.9180657863616943,
+      "logits/rejected": -2.9176905155181885,
+      "logps/chosen": -0.4231870174407959,
+      "logps/rejected": -1.1624265909194946,
+      "loss": 0.4775,
+      "nll_loss": 0.42117613554000854,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.021159350872039795,
+      "rewards/margins": 0.036961980164051056,
+      "rewards/rejected": -0.05812133103609085,
+      "step": 450
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 8.629730433025097,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 1.3161282539367676,
+      "log_odds_ratio": -0.3122040629386902,
+      "logits/chosen": -2.9203555583953857,
+      "logits/rejected": -2.9158670902252197,
+      "logps/chosen": -0.46862277388572693,
+      "logps/rejected": -1.085384726524353,
+      "loss": 0.5357,
+      "nll_loss": 0.4903482496738434,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.023431137204170227,
+      "rewards/margins": 0.030838098376989365,
+      "rewards/rejected": -0.05426923185586929,
+      "step": 460
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 9.462108983797318,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 1.5001277923583984,
+      "log_odds_ratio": -0.2745407521724701,
+      "logits/chosen": -2.918792724609375,
+      "logits/rejected": -2.920560598373413,
+      "logps/chosen": -0.45306238532066345,
+      "logps/rejected": -1.1488702297210693,
+      "loss": 0.5047,
+      "nll_loss": 0.46945127844810486,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.02265312150120735,
+      "rewards/margins": 0.034790389239788055,
+      "rewards/rejected": -0.05744351074099541,
+      "step": 470
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 12.033704149037684,
+      "learning_rate": 2.2821773229381924e-06,
+      "log_odds_chosen": 2.368060350418091,
+      "log_odds_ratio": -0.13723622262477875,
+      "logits/chosen": -2.896536111831665,
+      "logits/rejected": -2.9012863636016846,
+      "logps/chosen": -0.26962071657180786,
+      "logps/rejected": -1.277259349822998,
+      "loss": 0.2775,
+      "nll_loss": 0.2636214792728424,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.013481035828590393,
+      "rewards/margins": 0.05038193613290787,
+      "rewards/rejected": -0.06386296451091766,
+      "step": 480
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 8.31587372487098,
+      "learning_rate": 2.2587697572631284e-06,
+      "log_odds_chosen": 3.0361766815185547,
+      "log_odds_ratio": -0.0762939602136612,
+      "logits/chosen": -2.872748374938965,
+      "logits/rejected": -2.8818652629852295,
+      "logps/chosen": -0.17086732387542725,
+      "logps/rejected": -1.341355562210083,
+      "loss": 0.1837,
+      "nll_loss": 0.18102502822875977,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.008543366566300392,
+      "rewards/margins": 0.05852441117167473,
+      "rewards/rejected": -0.06706777215003967,
+      "step": 490
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 9.109087690728854,
+      "learning_rate": 2.23606797749979e-06,
+      "log_odds_chosen": 3.1204092502593994,
+      "log_odds_ratio": -0.0752432718873024,
+      "logits/chosen": -2.842435121536255,
+      "logits/rejected": -2.851132869720459,
+      "logps/chosen": -0.1684246063232422,
+      "logps/rejected": -1.4159977436065674,
+      "loss": 0.1865,
+      "nll_loss": 0.1788669377565384,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008421231061220169,
+      "rewards/margins": 0.0623786523938179,
+      "rewards/rejected": -0.07079987227916718,
+      "step": 500
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 11.342602042204765,
+      "learning_rate": 2.2140372138502386e-06,
+      "log_odds_chosen": 3.1781134605407715,
+      "log_odds_ratio": -0.06865429133176804,
+      "logits/chosen": -2.842808485031128,
+      "logits/rejected": -2.8529012203216553,
+      "logps/chosen": -0.15066179633140564,
+      "logps/rejected": -1.4150397777557373,
+      "loss": 0.1774,
+      "nll_loss": 0.15856973826885223,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.007533090654760599,
+      "rewards/margins": 0.06321889907121658,
+      "rewards/rejected": -0.07075198739767075,
+      "step": 510
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 9.015217844997702,
+      "learning_rate": 2.1926450482675734e-06,
+      "log_odds_chosen": 3.1590893268585205,
+      "log_odds_ratio": -0.05701795965433121,
+      "logits/chosen": -2.8566677570343018,
+      "logits/rejected": -2.8613815307617188,
+      "logps/chosen": -0.17371872067451477,
+      "logps/rejected": -1.5091571807861328,
+      "loss": 0.1875,
+      "nll_loss": 0.1826210618019104,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008685936219990253,
+      "rewards/margins": 0.06677192449569702,
+      "rewards/rejected": -0.0754578709602356,
+      "step": 520
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 7.705666021658087,
+      "learning_rate": 2.1718612138153473e-06,
+      "log_odds_chosen": 3.129737615585327,
+      "log_odds_ratio": -0.07283337414264679,
+      "logits/chosen": -2.857102632522583,
+      "logits/rejected": -2.8588953018188477,
+      "logps/chosen": -0.16752207279205322,
+      "logps/rejected": -1.4430062770843506,
+      "loss": 0.1864,
+      "nll_loss": 0.18135443329811096,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.008376102894544601,
+      "rewards/margins": 0.0637742131948471,
+      "rewards/rejected": -0.072150319814682,
+      "step": 530
+    },
+    {
+      "epoch": 2.28,
+      "grad_norm": 10.453820153776633,
+      "learning_rate": 2.151657414559676e-06,
+      "log_odds_chosen": 3.152796983718872,
+      "log_odds_ratio": -0.07412304729223251,
+      "logits/chosen": -2.8558595180511475,
+      "logits/rejected": -2.8634510040283203,
+      "logps/chosen": -0.1711394190788269,
+      "logps/rejected": -1.419811725616455,
+      "loss": 0.1867,
+      "nll_loss": 0.18491069972515106,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.0085569703951478,
+      "rewards/margins": 0.06243361160159111,
+      "rewards/rejected": -0.07099058479070663,
+      "step": 540
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 9.218290542920707,
+      "learning_rate": 2.132007163556104e-06,
+      "log_odds_chosen": 2.85337233543396,
+      "log_odds_ratio": -0.10947513580322266,
+      "logits/chosen": -2.851966142654419,
+      "logits/rejected": -2.8540444374084473,
+      "logps/chosen": -0.19084951281547546,
+      "logps/rejected": -1.3400590419769287,
+      "loss": 0.1955,
+      "nll_loss": 0.19958652555942535,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.009542474523186684,
+      "rewards/margins": 0.057460468262434006,
+      "rewards/rejected": -0.06700295209884644,
+      "step": 550
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 10.317365217804,
+      "learning_rate": 2.1128856368212917e-06,
+      "log_odds_chosen": 2.9810779094696045,
+      "log_odds_ratio": -0.09573061764240265,
+      "logits/chosen": -2.842881441116333,
+      "logits/rejected": -2.8447251319885254,
+      "logps/chosen": -0.19754521548748016,
+      "logps/rejected": -1.4727751016616821,
+      "loss": 0.1741,
+      "nll_loss": 0.20112493634223938,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.009877260774374008,
+      "rewards/margins": 0.06376149505376816,
+      "rewards/rejected": -0.07363875210285187,
+      "step": 560
+    },
+    {
+      "epoch": 2.41,
+      "grad_norm": 8.85810509927659,
+      "learning_rate": 2.0942695414584777e-06,
+      "log_odds_chosen": 3.2707130908966064,
+      "log_odds_ratio": -0.05988941714167595,
+      "logits/chosen": -2.8249504566192627,
+      "logits/rejected": -2.835737705230713,
+      "logps/chosen": -0.1614939272403717,
+      "logps/rejected": -1.4856724739074707,
+      "loss": 0.188,
+      "nll_loss": 0.17816588282585144,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008074694313108921,
+      "rewards/margins": 0.06620893627405167,
+      "rewards/rejected": -0.07428362965583801,
+      "step": 570
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 8.66523647376059,
+      "learning_rate": 2.0761369963434992e-06,
+      "log_odds_chosen": 2.9631845951080322,
+      "log_odds_ratio": -0.08499319851398468,
+      "logits/chosen": -2.785752058029175,
+      "logits/rejected": -2.796713352203369,
+      "logps/chosen": -0.18658441305160522,
+      "logps/rejected": -1.3940823078155518,
+      "loss": 0.1968,
+      "nll_loss": 0.1999494731426239,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.009329221211373806,
+      "rewards/margins": 0.060374896973371506,
+      "rewards/rejected": -0.06970412284135818,
+      "step": 580
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 7.663377813440287,
+      "learning_rate": 2.058467423981546e-06,
+      "log_odds_chosen": 3.1809768676757812,
+      "log_odds_ratio": -0.0660170167684555,
+      "logits/chosen": -2.7790160179138184,
+      "logits/rejected": -2.7909960746765137,
+      "logps/chosen": -0.16522589325904846,
+      "logps/rejected": -1.5188618898391724,
+      "loss": 0.1828,
+      "nll_loss": 0.18212279677391052,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.008261295035481453,
+      "rewards/margins": 0.06768180429935455,
+      "rewards/rejected": -0.07594309747219086,
+      "step": 590
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 9.43229140775389,
+      "learning_rate": 2.0412414523193154e-06,
+      "log_odds_chosen": 3.2196316719055176,
+      "log_odds_ratio": -0.05672148987650871,
+      "logits/chosen": -2.8662352561950684,
+      "logits/rejected": -2.872044086456299,
+      "logps/chosen": -0.16352586448192596,
+      "logps/rejected": -1.5207058191299438,
+      "loss": 0.1882,
+      "nll_loss": 0.1710083782672882,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008176293224096298,
+      "rewards/margins": 0.06785898655653,
+      "rewards/rejected": -0.0760352835059166,
+      "step": 600
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 9.251985707632262,
+      "learning_rate": 2.0244408254472904e-06,
+      "log_odds_chosen": 3.181028366088867,
+      "log_odds_ratio": -0.06368444114923477,
+      "logits/chosen": -2.8114702701568604,
+      "logits/rejected": -2.827327013015747,
+      "logps/chosen": -0.16818083822727203,
+      "logps/rejected": -1.5173419713974,
+      "loss": 0.1928,
+      "nll_loss": 0.19379279017448425,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008409040980041027,
+      "rewards/margins": 0.06745805591344833,
+      "rewards/rejected": -0.07586710155010223,
+      "step": 610
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 8.85560770094724,
+      "learning_rate": 2.0080483222562476e-06,
+      "log_odds_chosen": 3.0775837898254395,
+      "log_odds_ratio": -0.08850983530282974,
+      "logits/chosen": -2.8276491165161133,
+      "logits/rejected": -2.8322935104370117,
+      "logps/chosen": -0.17984047532081604,
+      "logps/rejected": -1.4078468084335327,
+      "loss": 0.1874,
+      "nll_loss": 0.18593069911003113,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.008992023766040802,
+      "rewards/margins": 0.061400316655635834,
+      "rewards/rejected": -0.07039234042167664,
+      "step": 620
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 9.092588797512901,
+      "learning_rate": 1.9920476822239895e-06,
+      "log_odds_chosen": 3.452129364013672,
+      "log_odds_ratio": -0.048345282673835754,
+      "logits/chosen": -2.828294277191162,
+      "logits/rejected": -2.8251712322235107,
+      "logps/chosen": -0.15338510274887085,
+      "logps/rejected": -1.581225872039795,
+      "loss": 0.1829,
+      "nll_loss": 0.18267366290092468,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0076692551374435425,
+      "rewards/margins": 0.07139204442501068,
+      "rewards/rejected": -0.07906129956245422,
+      "step": 630
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 8.85624863375054,
+      "learning_rate": 1.976423537605237e-06,
+      "log_odds_chosen": 3.381225109100342,
+      "log_odds_ratio": -0.0466490238904953,
+      "logits/chosen": -2.8369762897491455,
+      "logits/rejected": -2.8300728797912598,
+      "logps/chosen": -0.13137118518352509,
+      "logps/rejected": -1.4386910200119019,
+      "loss": 0.1705,
+      "nll_loss": 0.1446365863084793,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.006568559445440769,
+      "rewards/margins": 0.0653659924864769,
+      "rewards/rejected": -0.07193455845117569,
+      "step": 640
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 9.126901201726705,
+      "learning_rate": 1.961161351381841e-06,
+      "log_odds_chosen": 3.2348713874816895,
+      "log_odds_ratio": -0.059200920164585114,
+      "logits/chosen": -2.8560492992401123,
+      "logits/rejected": -2.8655595779418945,
+      "logps/chosen": -0.15543992817401886,
+      "logps/rejected": -1.4683492183685303,
+      "loss": 0.1718,
+      "nll_loss": 0.17798814177513123,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.007771995849907398,
+      "rewards/margins": 0.06564547121524811,
+      "rewards/rejected": -0.07341746240854263,
+      "step": 650
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 9.459348581011165,
+      "learning_rate": 1.9462473604038077e-06,
+      "log_odds_chosen": 3.150841236114502,
+      "log_odds_ratio": -0.08523111045360565,
+      "logits/chosen": -2.883835792541504,
+      "logits/rejected": -2.880549669265747,
+      "logps/chosen": -0.16044911742210388,
+      "logps/rejected": -1.4550909996032715,
+      "loss": 0.1827,
+      "nll_loss": 0.1803114414215088,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.008022455498576164,
+      "rewards/margins": 0.06473208963871002,
+      "rewards/rejected": -0.07275454699993134,
+      "step": 660
+    },
+    {
+      "epoch": 2.83,
+      "grad_norm": 8.641861937738083,
+      "learning_rate": 1.9316685232156397e-06,
+      "log_odds_chosen": 3.220848798751831,
+      "log_odds_ratio": -0.06002454087138176,
+      "logits/chosen": -2.8484086990356445,
+      "logits/rejected": -2.8489136695861816,
+      "logps/chosen": -0.15638573467731476,
+      "logps/rejected": -1.4806524515151978,
+      "loss": 0.1855,
+      "nll_loss": 0.177657350897789,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.007819287478923798,
+      "rewards/margins": 0.06621333211660385,
+      "rewards/rejected": -0.07403261959552765,
+      "step": 670
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 9.590977394670045,
+      "learning_rate": 1.917412472118426e-06,
+      "log_odds_chosen": 3.190727949142456,
+      "log_odds_ratio": -0.06804339587688446,
+      "logits/chosen": -2.8769946098327637,
+      "logits/rejected": -2.8714823722839355,
+      "logps/chosen": -0.16566260159015656,
+      "logps/rejected": -1.4919040203094482,
+      "loss": 0.1888,
+      "nll_loss": 0.1725473403930664,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008283129893243313,
+      "rewards/margins": 0.06631207466125488,
+      "rewards/rejected": -0.07459519803524017,
+      "step": 680
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 8.352762820112602,
+      "learning_rate": 1.9034674690672024e-06,
+      "log_odds_chosen": 3.2079219818115234,
+      "log_odds_ratio": -0.08390762656927109,
+      "logits/chosen": -2.8269476890563965,
+      "logits/rejected": -2.837005376815796,
+      "logps/chosen": -0.18829426169395447,
+      "logps/rejected": -1.5218613147735596,
+      "loss": 0.1972,
+      "nll_loss": 0.18950611352920532,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.009414712898433208,
+      "rewards/margins": 0.06667836010456085,
+      "rewards/rejected": -0.07609307020902634,
+      "step": 690
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 7.962144706780938,
+      "learning_rate": 1.8898223650461362e-06,
+      "log_odds_chosen": 3.216644763946533,
+      "log_odds_ratio": -0.060684747993946075,
+      "logits/chosen": -2.856433868408203,
+      "logits/rejected": -2.8612186908721924,
+      "logps/chosen": -0.16995587944984436,
+      "logps/rejected": -1.478044867515564,
+      "loss": 0.1852,
+      "nll_loss": 0.18319742381572723,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008497794158756733,
+      "rewards/margins": 0.06540445238351822,
+      "rewards/rejected": -0.07390224188566208,
+      "step": 700
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 8.826772322191102,
+      "learning_rate": 1.876466562602004e-06,
+      "log_odds_chosen": 3.2584176063537598,
+      "log_odds_ratio": -0.05594608932733536,
+      "logits/chosen": -2.808353900909424,
+      "logits/rejected": -2.8018672466278076,
+      "logps/chosen": -0.1637522280216217,
+      "logps/rejected": -1.5413471460342407,
+      "loss": 0.1864,
+      "nll_loss": 0.16807685792446136,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008187612518668175,
+      "rewards/margins": 0.06887974590063095,
+      "rewards/rejected": -0.07706736028194427,
+      "step": 710
+    },
+    {
+      "epoch": 3.0,
+      "step": 711,
+      "total_flos": 0.0,
+      "train_loss": 0.5470885709871219,
+      "train_runtime": 7181.4173,
+      "train_samples_per_second": 3.159,
+      "train_steps_per_second": 0.099
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 711,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1bb626a5d0074ea6e691a2d2a143be2c2490ace4eac9f05b57ed37d27dc56d6
+size 6456