Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: inverted-orpo-mistral-capybara
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# inverted-orpo-mistral-capybara
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.38.1
+- Pytorch 2.1.2+cu121
+- Datasets 2.16.1
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.9363477402682757,
+    "train_runtime": 28864.8871,
+    "train_samples": 14154,
+    "train_samples_per_second": 1.471,
+    "train_steps_per_second": 0.023
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.38.1"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52565d8426d487c0d065ff7530c197f7b71bd0a425fbeab09953b469ea2d4ca4
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:561cba56192299fe4327e54e6f665ae123ded33b0fc3b17129772fa73b0531f8
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d05e90c9eab30ff5ec3716816875a2657e928d36ea060f621c7f7f24c95b96bb
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/May08_18-19-16_rtx6000gpu1.xfact.net/events.out.tfevents.1715160238.rtx6000gpu1.xfact.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6ab613afc839712e3a09211a0f45bd2ef9dfd11bb83248533654f8114158e07
+size 23733

runs/May08_21-15-06_rtx6000gpu1.xfact.net/events.out.tfevents.1715170766.rtx6000gpu1.xfact.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:722dafbdc1aae4f1ea4a80f36185409c919054270aa9975bb656014cf13b48ef
+size 61927

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.9363477402682757,
+    "train_runtime": 28864.8871,
+    "train_samples": 14154,
+    "train_samples_per_second": 1.471,
+    "train_steps_per_second": 0.023
+}

trainer_state.json

@@ -0,0 +1,1218 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9966101694915253,
+  "eval_steps": 500,
+  "global_step": 663,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.05,
+      "grad_norm": 10.625,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": -0.21723361313343048,
+      "log_odds_ratio": -0.8646289706230164,
+      "logits/chosen": -2.376826286315918,
+      "logits/rejected": -2.3396213054656982,
+      "logps/chosen": -1.1265513896942139,
+      "logps/rejected": -0.9904217720031738,
+      "loss": 1.1515,
+      "nll_loss": 1.1495189666748047,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": -0.05632757022976875,
+      "rewards/margins": -0.006806477904319763,
+      "rewards/rejected": -0.04952109605073929,
+      "step": 10
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 8.125,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": -0.2775025963783264,
+      "log_odds_ratio": -0.8951504826545715,
+      "logits/chosen": -2.366422653198242,
+      "logits/rejected": -2.338573694229126,
+      "logps/chosen": -1.1216810941696167,
+      "logps/rejected": -0.9544132351875305,
+      "loss": 1.1414,
+      "nll_loss": 1.1651629209518433,
+      "rewards/accuracies": 0.32499998807907104,
+      "rewards/chosen": -0.056084055453538895,
+      "rewards/margins": -0.00836339220404625,
+      "rewards/rejected": -0.047720663249492645,
+      "step": 20
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 7.03125,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": -0.1686987578868866,
+      "log_odds_ratio": -0.8350237011909485,
+      "logits/chosen": -2.346161365509033,
+      "logits/rejected": -2.3042690753936768,
+      "logps/chosen": -1.0229403972625732,
+      "logps/rejected": -0.916629433631897,
+      "loss": 1.0927,
+      "nll_loss": 1.063353419303894,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.051147013902664185,
+      "rewards/margins": -0.005315541289746761,
+      "rewards/rejected": -0.045831479132175446,
+      "step": 30
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 7.09375,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": -0.2760249376296997,
+      "log_odds_ratio": -0.8817641139030457,
+      "logits/chosen": -2.3606069087982178,
+      "logits/rejected": -2.3503973484039307,
+      "logps/chosen": -0.9669380187988281,
+      "logps/rejected": -0.8209754228591919,
+      "loss": 1.0772,
+      "nll_loss": 0.9983038902282715,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": -0.04834689944982529,
+      "rewards/margins": -0.007298128213733435,
+      "rewards/rejected": -0.041048772633075714,
+      "step": 40
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 7.1875,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": -0.2603610157966614,
+      "log_odds_ratio": -0.8878246545791626,
+      "logits/chosen": -2.4249422550201416,
+      "logits/rejected": -2.325108766555786,
+      "logps/chosen": -1.055864930152893,
+      "logps/rejected": -0.8944637179374695,
+      "loss": 1.1001,
+      "nll_loss": 1.1144287586212158,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": -0.05279325321316719,
+      "rewards/margins": -0.008070068433880806,
+      "rewards/rejected": -0.044723186641931534,
+      "step": 50
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 6.84375,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": -0.31346869468688965,
+      "log_odds_ratio": -0.9280476570129395,
+      "logits/chosen": -2.419933319091797,
+      "logits/rejected": -2.3587355613708496,
+      "logps/chosen": -1.0151221752166748,
+      "logps/rejected": -0.8353691101074219,
+      "loss": 1.0398,
+      "nll_loss": 1.0006625652313232,
+      "rewards/accuracies": 0.3187499940395355,
+      "rewards/chosen": -0.05075611546635628,
+      "rewards/margins": -0.00898765493184328,
+      "rewards/rejected": -0.04176845774054527,
+      "step": 60
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 6.65625,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": -0.21962186694145203,
+      "log_odds_ratio": -0.8736317753791809,
+      "logits/chosen": -2.3793551921844482,
+      "logits/rejected": -2.3073837757110596,
+      "logps/chosen": -0.9888677597045898,
+      "logps/rejected": -0.8603803515434265,
+      "loss": 1.0517,
+      "nll_loss": 1.0219568014144897,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.04944339022040367,
+      "rewards/margins": -0.006424369756132364,
+      "rewards/rejected": -0.043019019067287445,
+      "step": 70
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 6.65625,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": -0.28525999188423157,
+      "log_odds_ratio": -0.9010836482048035,
+      "logits/chosen": -2.3715453147888184,
+      "logits/rejected": -2.297002077102661,
+      "logps/chosen": -0.999737560749054,
+      "logps/rejected": -0.8434433937072754,
+      "loss": 1.074,
+      "nll_loss": 1.0341795682907104,
+      "rewards/accuracies": 0.3125,
+      "rewards/chosen": -0.04998688027262688,
+      "rewards/margins": -0.007814710959792137,
+      "rewards/rejected": -0.04217217117547989,
+      "step": 80
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 6.3125,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": -0.18699277937412262,
+      "log_odds_ratio": -0.8439871668815613,
+      "logits/chosen": -2.3840651512145996,
+      "logits/rejected": -2.3138346672058105,
+      "logps/chosen": -0.9523512721061707,
+      "logps/rejected": -0.8433036804199219,
+      "loss": 1.0374,
+      "nll_loss": 0.9886404275894165,
+      "rewards/accuracies": 0.39375001192092896,
+      "rewards/chosen": -0.04761756584048271,
+      "rewards/margins": -0.005452378187328577,
+      "rewards/rejected": -0.04216518625617027,
+      "step": 90
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 7.3125,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": -0.20422692596912384,
+      "log_odds_ratio": -0.8546068072319031,
+      "logits/chosen": -2.344386577606201,
+      "logits/rejected": -2.282938003540039,
+      "logps/chosen": -1.053534984588623,
+      "logps/rejected": -0.9320154190063477,
+      "loss": 1.0879,
+      "nll_loss": 1.0812509059906006,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.05267674848437309,
+      "rewards/margins": -0.006075979210436344,
+      "rewards/rejected": -0.04660077020525932,
+      "step": 100
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 6.4375,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": -0.24251386523246765,
+      "log_odds_ratio": -0.8658794164657593,
+      "logits/chosen": -2.3458285331726074,
+      "logits/rejected": -2.298800230026245,
+      "logps/chosen": -1.0196887254714966,
+      "logps/rejected": -0.8735698461532593,
+      "loss": 1.099,
+      "nll_loss": 1.0551105737686157,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": -0.05098443105816841,
+      "rewards/margins": -0.007305940147489309,
+      "rewards/rejected": -0.043678492307662964,
+      "step": 110
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 7.5,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": -0.29140716791152954,
+      "log_odds_ratio": -0.9143481254577637,
+      "logits/chosen": -2.3598785400390625,
+      "logits/rejected": -2.299387216567993,
+      "logps/chosen": -1.0395349264144897,
+      "logps/rejected": -0.8586183786392212,
+      "loss": 1.0884,
+      "nll_loss": 1.0841031074523926,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.05197674781084061,
+      "rewards/margins": -0.009045831859111786,
+      "rewards/rejected": -0.04293091595172882,
+      "step": 120
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 7.1875,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": -0.19871631264686584,
+      "log_odds_ratio": -0.8721719980239868,
+      "logits/chosen": -2.4017045497894287,
+      "logits/rejected": -2.3413734436035156,
+      "logps/chosen": -0.9847584962844849,
+      "logps/rejected": -0.8684871792793274,
+      "loss": 1.0817,
+      "nll_loss": 1.0258110761642456,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.0492379292845726,
+      "rewards/margins": -0.005813568830490112,
+      "rewards/rejected": -0.04342436045408249,
+      "step": 130
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 6.375,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": -0.2122315913438797,
+      "log_odds_ratio": -0.8670593500137329,
+      "logits/chosen": -2.3736579418182373,
+      "logits/rejected": -2.2930188179016113,
+      "logps/chosen": -0.9382177591323853,
+      "logps/rejected": -0.821458637714386,
+      "loss": 1.0268,
+      "nll_loss": 0.9702949523925781,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.04691087827086449,
+      "rewards/margins": -0.00583794666454196,
+      "rewards/rejected": -0.04107293859124184,
+      "step": 140
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 7.25,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": -0.22532574832439423,
+      "log_odds_ratio": -0.8638848066329956,
+      "logits/chosen": -2.442842960357666,
+      "logits/rejected": -2.3551251888275146,
+      "logps/chosen": -1.0039573907852173,
+      "logps/rejected": -0.8670075535774231,
+      "loss": 1.0895,
+      "nll_loss": 1.044359564781189,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.05019787698984146,
+      "rewards/margins": -0.006847499404102564,
+      "rewards/rejected": -0.043350376188755035,
+      "step": 150
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 6.875,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": -0.22583532333374023,
+      "log_odds_ratio": -0.8749126195907593,
+      "logits/chosen": -2.354644298553467,
+      "logits/rejected": -2.2900655269622803,
+      "logps/chosen": -0.9405444264411926,
+      "logps/rejected": -0.80528724193573,
+      "loss": 1.0844,
+      "nll_loss": 1.0093518495559692,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.04702722281217575,
+      "rewards/margins": -0.006762862205505371,
+      "rewards/rejected": -0.04026436060667038,
+      "step": 160
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 6.0,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": -0.30897724628448486,
+      "log_odds_ratio": -0.9169195294380188,
+      "logits/chosen": -2.4037585258483887,
+      "logits/rejected": -2.382966995239258,
+      "logps/chosen": -1.0549595355987549,
+      "logps/rejected": -0.8608039617538452,
+      "loss": 1.1012,
+      "nll_loss": 1.069485068321228,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.052747976034879684,
+      "rewards/margins": -0.009707780554890633,
+      "rewards/rejected": -0.0430401973426342,
+      "step": 170
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 6.65625,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": -0.151263028383255,
+      "log_odds_ratio": -0.8355720639228821,
+      "logits/chosen": -2.3262200355529785,
+      "logits/rejected": -2.2469918727874756,
+      "logps/chosen": -1.0002092123031616,
+      "logps/rejected": -0.9038441777229309,
+      "loss": 1.0736,
+      "nll_loss": 1.0467875003814697,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": -0.05001046508550644,
+      "rewards/margins": -0.004818252753466368,
+      "rewards/rejected": -0.045192211866378784,
+      "step": 180
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 6.1875,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": -0.21370911598205566,
+      "log_odds_ratio": -0.8563556671142578,
+      "logits/chosen": -2.3353800773620605,
+      "logits/rejected": -2.302805185317993,
+      "logps/chosen": -0.9616407155990601,
+      "logps/rejected": -0.8339935541152954,
+      "loss": 1.0925,
+      "nll_loss": 1.0070997476577759,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.04808203876018524,
+      "rewards/margins": -0.006382361985743046,
+      "rewards/rejected": -0.04169967398047447,
+      "step": 190
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 6.875,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": -0.2662477195262909,
+      "log_odds_ratio": -0.8836930394172668,
+      "logits/chosen": -2.3342254161834717,
+      "logits/rejected": -2.279543876647949,
+      "logps/chosen": -0.9679354429244995,
+      "logps/rejected": -0.812696635723114,
+      "loss": 1.0688,
+      "nll_loss": 0.9891126751899719,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.04839677736163139,
+      "rewards/margins": -0.007761943154036999,
+      "rewards/rejected": -0.04063483327627182,
+      "step": 200
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 6.5,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": -0.352990984916687,
+      "log_odds_ratio": -0.9537938833236694,
+      "logits/chosen": -2.2331907749176025,
+      "logits/rejected": -2.1765122413635254,
+      "logps/chosen": -0.999859631061554,
+      "logps/rejected": -0.8078708648681641,
+      "loss": 1.0732,
+      "nll_loss": 1.0370758771896362,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.049992986023426056,
+      "rewards/margins": -0.00959943700581789,
+      "rewards/rejected": -0.04039354994893074,
+      "step": 210
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 6.6875,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": -0.24687866866588593,
+      "log_odds_ratio": -0.885057270526886,
+      "logits/chosen": -2.32244873046875,
+      "logits/rejected": -2.2400941848754883,
+      "logps/chosen": -0.9790719151496887,
+      "logps/rejected": -0.8183375597000122,
+      "loss": 1.0485,
+      "nll_loss": 1.0295348167419434,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.0489535927772522,
+      "rewards/margins": -0.008036715909838676,
+      "rewards/rejected": -0.04091687500476837,
+      "step": 220
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 7.0625,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": 0.004374316427856684,
+      "log_odds_ratio": -0.7569075226783752,
+      "logits/chosen": -2.33986234664917,
+      "logits/rejected": -2.275083065032959,
+      "logps/chosen": -0.8708177804946899,
+      "logps/rejected": -0.8615075349807739,
+      "loss": 0.9401,
+      "nll_loss": 0.9019868969917297,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.043540894985198975,
+      "rewards/margins": -0.00046551605919376016,
+      "rewards/rejected": -0.043075378984212875,
+      "step": 230
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 6.9375,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 0.136603444814682,
+      "log_odds_ratio": -0.6914254426956177,
+      "logits/chosen": -2.302295207977295,
+      "logits/rejected": -2.263159990310669,
+      "logps/chosen": -0.8053948283195496,
+      "logps/rejected": -0.8713942766189575,
+      "loss": 0.9052,
+      "nll_loss": 0.8538738489151001,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04026973992586136,
+      "rewards/margins": 0.003299972740933299,
+      "rewards/rejected": -0.04356971010565758,
+      "step": 240
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 7.59375,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 0.1561969816684723,
+      "log_odds_ratio": -0.7018811106681824,
+      "logits/chosen": -2.317003011703491,
+      "logits/rejected": -2.2483248710632324,
+      "logps/chosen": -0.8204513788223267,
+      "logps/rejected": -0.8764976263046265,
+      "loss": 0.9152,
+      "nll_loss": 0.865095317363739,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.04102257266640663,
+      "rewards/margins": 0.0028023170307278633,
+      "rewards/rejected": -0.04382488504052162,
+      "step": 250
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 8.5,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 0.07478588074445724,
+      "log_odds_ratio": -0.7376130223274231,
+      "logits/chosen": -2.3320107460021973,
+      "logits/rejected": -2.273357391357422,
+      "logps/chosen": -0.8615882992744446,
+      "logps/rejected": -0.8847838640213013,
+      "loss": 0.9318,
+      "nll_loss": 0.8836938142776489,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.04307941347360611,
+      "rewards/margins": 0.0011597753036767244,
+      "rewards/rejected": -0.04423918575048447,
+      "step": 260
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 7.125,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 0.193772554397583,
+      "log_odds_ratio": -0.6876689791679382,
+      "logits/chosen": -2.303919553756714,
+      "logits/rejected": -2.2494609355926514,
+      "logps/chosen": -0.7990426421165466,
+      "logps/rejected": -0.8900362253189087,
+      "loss": 0.8974,
+      "nll_loss": 0.848175048828125,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.03995212912559509,
+      "rewards/margins": 0.0045496742241084576,
+      "rewards/rejected": -0.044501807540655136,
+      "step": 270
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 7.0,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 0.1328398734331131,
+      "log_odds_ratio": -0.6926364302635193,
+      "logits/chosen": -2.3918557167053223,
+      "logits/rejected": -2.3479630947113037,
+      "logps/chosen": -0.7991486191749573,
+      "logps/rejected": -0.8500739336013794,
+      "loss": 0.9058,
+      "nll_loss": 0.8436633348464966,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.039957426488399506,
+      "rewards/margins": 0.0025462661869823933,
+      "rewards/rejected": -0.04250369220972061,
+      "step": 280
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 7.625,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 0.1771097183227539,
+      "log_odds_ratio": -0.6848055124282837,
+      "logits/chosen": -2.3858330249786377,
+      "logits/rejected": -2.3222715854644775,
+      "logps/chosen": -0.8123289346694946,
+      "logps/rejected": -0.8936681747436523,
+      "loss": 0.9117,
+      "nll_loss": 0.8546589016914368,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.04061644524335861,
+      "rewards/margins": 0.004066964611411095,
+      "rewards/rejected": -0.044683415442705154,
+      "step": 290
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 6.84375,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": 0.154900923371315,
+      "log_odds_ratio": -0.6884406805038452,
+      "logits/chosen": -2.3433773517608643,
+      "logits/rejected": -2.337531089782715,
+      "logps/chosen": -0.8393154144287109,
+      "logps/rejected": -0.8944731950759888,
+      "loss": 0.9008,
+      "nll_loss": 0.8844978213310242,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04196577146649361,
+      "rewards/margins": 0.0027578892186284065,
+      "rewards/rejected": -0.04472366347908974,
+      "step": 300
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 7.21875,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 0.12805406749248505,
+      "log_odds_ratio": -0.7049511671066284,
+      "logits/chosen": -2.3576130867004395,
+      "logits/rejected": -2.306063652038574,
+      "logps/chosen": -0.842370867729187,
+      "logps/rejected": -0.8757230639457703,
+      "loss": 0.9307,
+      "nll_loss": 0.8907486200332642,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.04211854189634323,
+      "rewards/margins": 0.0016676082741469145,
+      "rewards/rejected": -0.04378615692257881,
+      "step": 310
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 6.6875,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 0.21001560986042023,
+      "log_odds_ratio": -0.68244469165802,
+      "logits/chosen": -2.40632700920105,
+      "logits/rejected": -2.3724734783172607,
+      "logps/chosen": -0.8295213580131531,
+      "logps/rejected": -0.942533016204834,
+      "loss": 0.921,
+      "nll_loss": 0.8767648935317993,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04147607088088989,
+      "rewards/margins": 0.005650580860674381,
+      "rewards/rejected": -0.0471266508102417,
+      "step": 320
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 7.53125,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 0.15271909534931183,
+      "log_odds_ratio": -0.6864677667617798,
+      "logits/chosen": -2.375275135040283,
+      "logits/rejected": -2.3332605361938477,
+      "logps/chosen": -0.8226965665817261,
+      "logps/rejected": -0.8755828738212585,
+      "loss": 0.9249,
+      "nll_loss": 0.8956872820854187,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.04113483056426048,
+      "rewards/margins": 0.0026443139649927616,
+      "rewards/rejected": -0.04377914220094681,
+      "step": 330
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 7.75,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 0.11997294425964355,
+      "log_odds_ratio": -0.6974139213562012,
+      "logits/chosen": -2.3423027992248535,
+      "logits/rejected": -2.3120710849761963,
+      "logps/chosen": -0.8107906579971313,
+      "logps/rejected": -0.8437153100967407,
+      "loss": 0.9476,
+      "nll_loss": 0.8603833913803101,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.040539536625146866,
+      "rewards/margins": 0.0016462260391563177,
+      "rewards/rejected": -0.04218576103448868,
+      "step": 340
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 7.4375,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": 0.12810608744621277,
+      "log_odds_ratio": -0.7091251015663147,
+      "logits/chosen": -2.353813886642456,
+      "logits/rejected": -2.307131052017212,
+      "logps/chosen": -0.7978275418281555,
+      "logps/rejected": -0.8407192230224609,
+      "loss": 0.922,
+      "nll_loss": 0.8526071310043335,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.039891377091407776,
+      "rewards/margins": 0.002144585130736232,
+      "rewards/rejected": -0.04203595966100693,
+      "step": 350
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 6.46875,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 0.14650364220142365,
+      "log_odds_ratio": -0.6985893845558167,
+      "logits/chosen": -2.434299945831299,
+      "logits/rejected": -2.3850483894348145,
+      "logps/chosen": -0.8027324676513672,
+      "logps/rejected": -0.8706743121147156,
+      "loss": 0.932,
+      "nll_loss": 0.840488076210022,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04013662412762642,
+      "rewards/margins": 0.003397094551473856,
+      "rewards/rejected": -0.04353372007608414,
+      "step": 360
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 7.0,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": 0.07762489467859268,
+      "log_odds_ratio": -0.7249036431312561,
+      "logits/chosen": -2.3616278171539307,
+      "logits/rejected": -2.2978177070617676,
+      "logps/chosen": -0.811935544013977,
+      "logps/rejected": -0.837403416633606,
+      "loss": 0.9067,
+      "nll_loss": 0.8698326945304871,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04059677943587303,
+      "rewards/margins": 0.0012733942130580544,
+      "rewards/rejected": -0.04187016934156418,
+      "step": 370
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 7.5,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": 0.10106128454208374,
+      "log_odds_ratio": -0.6977318525314331,
+      "logits/chosen": -2.368786334991455,
+      "logits/rejected": -2.305479049682617,
+      "logps/chosen": -0.8443781137466431,
+      "logps/rejected": -0.8856738209724426,
+      "loss": 0.902,
+      "nll_loss": 0.886816143989563,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.04221891239285469,
+      "rewards/margins": 0.0020647819619625807,
+      "rewards/rejected": -0.04428369551897049,
+      "step": 380
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 7.0,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 0.22741122543811798,
+      "log_odds_ratio": -0.6608260869979858,
+      "logits/chosen": -2.3913867473602295,
+      "logits/rejected": -2.3317337036132812,
+      "logps/chosen": -0.8110629916191101,
+      "logps/rejected": -0.9145916104316711,
+      "loss": 0.9266,
+      "nll_loss": 0.8723472356796265,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.040553152561187744,
+      "rewards/margins": 0.005176428239792585,
+      "rewards/rejected": -0.04572957754135132,
+      "step": 390
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 7.125,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 0.09950557351112366,
+      "log_odds_ratio": -0.7172703742980957,
+      "logits/chosen": -2.3557465076446533,
+      "logits/rejected": -2.3033339977264404,
+      "logps/chosen": -0.9033384323120117,
+      "logps/rejected": -0.936965823173523,
+      "loss": 0.9442,
+      "nll_loss": 0.9896990060806274,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.045166920870542526,
+      "rewards/margins": 0.0016813663532957435,
+      "rewards/rejected": -0.04684828966856003,
+      "step": 400
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 6.9375,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 0.1489110141992569,
+      "log_odds_ratio": -0.6765201687812805,
+      "logits/chosen": -2.3149614334106445,
+      "logits/rejected": -2.264227867126465,
+      "logps/chosen": -0.774600625038147,
+      "logps/rejected": -0.8234383463859558,
+      "loss": 0.8682,
+      "nll_loss": 0.8227639198303223,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.03873003274202347,
+      "rewards/margins": 0.002441887278109789,
+      "rewards/rejected": -0.04117191582918167,
+      "step": 410
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 7.21875,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 0.26809313893318176,
+      "log_odds_ratio": -0.6399468183517456,
+      "logits/chosen": -2.3514695167541504,
+      "logits/rejected": -2.292144298553467,
+      "logps/chosen": -0.767677903175354,
+      "logps/rejected": -0.8895092010498047,
+      "loss": 0.8851,
+      "nll_loss": 0.8227022886276245,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.03838389366865158,
+      "rewards/margins": 0.006091567222028971,
+      "rewards/rejected": -0.04447546228766441,
+      "step": 420
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 8.0,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 0.10217361152172089,
+      "log_odds_ratio": -0.7120336294174194,
+      "logits/chosen": -2.332167148590088,
+      "logits/rejected": -2.262996196746826,
+      "logps/chosen": -0.861347496509552,
+      "logps/rejected": -0.9110593795776367,
+      "loss": 0.9142,
+      "nll_loss": 0.9066430330276489,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.04306737333536148,
+      "rewards/margins": 0.0024855921510607004,
+      "rewards/rejected": -0.04555296525359154,
+      "step": 430
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 7.09375,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 0.06129971146583557,
+      "log_odds_ratio": -0.7196843028068542,
+      "logits/chosen": -2.345583915710449,
+      "logits/rejected": -2.3343563079833984,
+      "logps/chosen": -0.8849897384643555,
+      "logps/rejected": -0.8971759080886841,
+      "loss": 0.9506,
+      "nll_loss": 0.9216864705085754,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.04424949362874031,
+      "rewards/margins": 0.0006093080155551434,
+      "rewards/rejected": -0.04485879838466644,
+      "step": 440
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 12.1875,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 0.3704330623149872,
+      "log_odds_ratio": -0.6121096611022949,
+      "logits/chosen": -2.294252634048462,
+      "logits/rejected": -2.2701363563537598,
+      "logps/chosen": -0.7903162837028503,
+      "logps/rejected": -0.9567170143127441,
+      "loss": 0.8449,
+      "nll_loss": 0.8487945795059204,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.039515819400548935,
+      "rewards/margins": 0.008320031687617302,
+      "rewards/rejected": -0.04783584922552109,
+      "step": 450
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 8.8125,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 0.571201741695404,
+      "log_odds_ratio": -0.5243600606918335,
+      "logits/chosen": -2.3143715858459473,
+      "logits/rejected": -2.277233600616455,
+      "logps/chosen": -0.7199083566665649,
+      "logps/rejected": -0.9758491516113281,
+      "loss": 0.8267,
+      "nll_loss": 0.8100913763046265,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.03599541634321213,
+      "rewards/margins": 0.012797033414244652,
+      "rewards/rejected": -0.048792459070682526,
+      "step": 460
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 7.34375,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 0.5478326082229614,
+      "log_odds_ratio": -0.5462910532951355,
+      "logits/chosen": -2.284961223602295,
+      "logits/rejected": -2.2431790828704834,
+      "logps/chosen": -0.7308935523033142,
+      "logps/rejected": -0.952447235584259,
+      "loss": 0.7979,
+      "nll_loss": 0.8102613687515259,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03654467687010765,
+      "rewards/margins": 0.011077682487666607,
+      "rewards/rejected": -0.04762236028909683,
+      "step": 470
+    },
+    {
+      "epoch": 2.17,
+      "grad_norm": 8.3125,
+      "learning_rate": 2.2821773229381924e-06,
+      "log_odds_chosen": 0.5902162790298462,
+      "log_odds_ratio": -0.5309671759605408,
+      "logits/chosen": -2.303758144378662,
+      "logits/rejected": -2.2506182193756104,
+      "logps/chosen": -0.7160202264785767,
+      "logps/rejected": -0.9725910425186157,
+      "loss": 0.8048,
+      "nll_loss": 0.7902317047119141,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.03580101206898689,
+      "rewards/margins": 0.012828533537685871,
+      "rewards/rejected": -0.04862954840064049,
+      "step": 480
+    },
+    {
+      "epoch": 2.21,
+      "grad_norm": 7.78125,
+      "learning_rate": 2.2587697572631284e-06,
+      "log_odds_chosen": 0.4584142565727234,
+      "log_odds_ratio": -0.578834593296051,
+      "logits/chosen": -2.3179421424865723,
+      "logits/rejected": -2.2865946292877197,
+      "logps/chosen": -0.7205600738525391,
+      "logps/rejected": -0.899226188659668,
+      "loss": 0.7868,
+      "nll_loss": 0.7924690246582031,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03602800518274307,
+      "rewards/margins": 0.008933312259614468,
+      "rewards/rejected": -0.044961318373680115,
+      "step": 490
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 6.9375,
+      "learning_rate": 2.23606797749979e-06,
+      "log_odds_chosen": 0.5127681493759155,
+      "log_odds_ratio": -0.5489556789398193,
+      "logits/chosen": -2.3282065391540527,
+      "logits/rejected": -2.263848066329956,
+      "logps/chosen": -0.7146658897399902,
+      "logps/rejected": -0.9588983654975891,
+      "loss": 0.8044,
+      "nll_loss": 0.7427123188972473,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.03573329374194145,
+      "rewards/margins": 0.012211622670292854,
+      "rewards/rejected": -0.04794491454958916,
+      "step": 500
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 7.59375,
+      "learning_rate": 2.2140372138502386e-06,
+      "log_odds_chosen": 0.5753137469291687,
+      "log_odds_ratio": -0.5367614030838013,
+      "logits/chosen": -2.2426199913024902,
+      "logits/rejected": -2.2221579551696777,
+      "logps/chosen": -0.68536376953125,
+      "logps/rejected": -0.9230566024780273,
+      "loss": 0.8109,
+      "nll_loss": 0.7715431451797485,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.03426819294691086,
+      "rewards/margins": 0.011884642764925957,
+      "rewards/rejected": -0.046152833849191666,
+      "step": 510
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 7.96875,
+      "learning_rate": 2.1926450482675734e-06,
+      "log_odds_chosen": 0.3541458547115326,
+      "log_odds_ratio": -0.6071102619171143,
+      "logits/chosen": -2.31622576713562,
+      "logits/rejected": -2.2613685131073,
+      "logps/chosen": -0.7861618995666504,
+      "logps/rejected": -0.941517174243927,
+      "loss": 0.8433,
+      "nll_loss": 0.8615577816963196,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.0393080934882164,
+      "rewards/margins": 0.007767767645418644,
+      "rewards/rejected": -0.04707586020231247,
+      "step": 520
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 8.125,
+      "learning_rate": 2.1718612138153473e-06,
+      "log_odds_chosen": 0.5630629062652588,
+      "log_odds_ratio": -0.5376764535903931,
+      "logits/chosen": -2.322035551071167,
+      "logits/rejected": -2.2575039863586426,
+      "logps/chosen": -0.728439450263977,
+      "logps/rejected": -0.9899078607559204,
+      "loss": 0.7861,
+      "nll_loss": 0.7822965383529663,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.03642197325825691,
+      "rewards/margins": 0.013073420152068138,
+      "rewards/rejected": -0.0494953915476799,
+      "step": 530
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 7.78125,
+      "learning_rate": 2.151657414559676e-06,
+      "log_odds_chosen": 0.37775903940200806,
+      "log_odds_ratio": -0.5940607190132141,
+      "logits/chosen": -2.375549793243408,
+      "logits/rejected": -2.321992874145508,
+      "logps/chosen": -0.7584763169288635,
+      "logps/rejected": -0.9299229383468628,
+      "loss": 0.7987,
+      "nll_loss": 0.8176478147506714,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.037923816591501236,
+      "rewards/margins": 0.00857233814895153,
+      "rewards/rejected": -0.04649615287780762,
+      "step": 540
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 7.59375,
+      "learning_rate": 2.132007163556104e-06,
+      "log_odds_chosen": 0.4968021512031555,
+      "log_odds_ratio": -0.559329092502594,
+      "logits/chosen": -2.27695894241333,
+      "logits/rejected": -2.2415664196014404,
+      "logps/chosen": -0.7272771596908569,
+      "logps/rejected": -0.9639213681221008,
+      "loss": 0.8073,
+      "nll_loss": 0.7941919565200806,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.036363862454891205,
+      "rewards/margins": 0.011832201853394508,
+      "rewards/rejected": -0.04819606989622116,
+      "step": 550
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 8.4375,
+      "learning_rate": 2.1128856368212917e-06,
+      "log_odds_chosen": 0.5834923982620239,
+      "log_odds_ratio": -0.544324517250061,
+      "logits/chosen": -2.2980105876922607,
+      "logits/rejected": -2.26395845413208,
+      "logps/chosen": -0.7306932210922241,
+      "logps/rejected": -0.9826458096504211,
+      "loss": 0.8055,
+      "nll_loss": 0.7919368743896484,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.036534663289785385,
+      "rewards/margins": 0.012597626075148582,
+      "rewards/rejected": -0.049132294952869415,
+      "step": 560
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 8.5625,
+      "learning_rate": 2.0942695414584777e-06,
+      "log_odds_chosen": 0.35887032747268677,
+      "log_odds_ratio": -0.6235562562942505,
+      "logits/chosen": -2.361316204071045,
+      "logits/rejected": -2.294520854949951,
+      "logps/chosen": -0.7684736251831055,
+      "logps/rejected": -0.9107894897460938,
+      "loss": 0.8402,
+      "nll_loss": 0.8586353063583374,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.03842368349432945,
+      "rewards/margins": 0.007115787826478481,
+      "rewards/rejected": -0.04553947597742081,
+      "step": 570
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 7.90625,
+      "learning_rate": 2.0761369963434992e-06,
+      "log_odds_chosen": 0.6161745190620422,
+      "log_odds_ratio": -0.5137969851493835,
+      "logits/chosen": -2.321749448776245,
+      "logits/rejected": -2.320282459259033,
+      "logps/chosen": -0.686724841594696,
+      "logps/rejected": -0.961820125579834,
+      "loss": 0.7929,
+      "nll_loss": 0.7309606671333313,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.03433624282479286,
+      "rewards/margins": 0.013754759915173054,
+      "rewards/rejected": -0.04809100180864334,
+      "step": 580
+    },
+    {
+      "epoch": 2.67,
+      "grad_norm": 7.84375,
+      "learning_rate": 2.058467423981546e-06,
+      "log_odds_chosen": 0.5524671077728271,
+      "log_odds_ratio": -0.5460283756256104,
+      "logits/chosen": -2.3575046062469482,
+      "logits/rejected": -2.301823616027832,
+      "logps/chosen": -0.6897765398025513,
+      "logps/rejected": -0.9405665397644043,
+      "loss": 0.8173,
+      "nll_loss": 0.7388589382171631,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.034488826990127563,
+      "rewards/margins": 0.012539500370621681,
+      "rewards/rejected": -0.047028329223394394,
+      "step": 590
+    },
+    {
+      "epoch": 2.71,
+      "grad_norm": 8.0625,
+      "learning_rate": 2.0412414523193154e-06,
+      "log_odds_chosen": 0.4845600128173828,
+      "log_odds_ratio": -0.5647393465042114,
+      "logits/chosen": -2.317967176437378,
+      "logits/rejected": -2.288545846939087,
+      "logps/chosen": -0.7509777545928955,
+      "logps/rejected": -0.971537709236145,
+      "loss": 0.8197,
+      "nll_loss": 0.8222864270210266,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.037548888474702835,
+      "rewards/margins": 0.011028001084923744,
+      "rewards/rejected": -0.04857688397169113,
+      "step": 600
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 7.3125,
+      "learning_rate": 2.0244408254472904e-06,
+      "log_odds_chosen": 0.4420229494571686,
+      "log_odds_ratio": -0.5893286466598511,
+      "logits/chosen": -2.3047995567321777,
+      "logits/rejected": -2.318437099456787,
+      "logps/chosen": -0.7420299053192139,
+      "logps/rejected": -0.9326221346855164,
+      "loss": 0.7964,
+      "nll_loss": 0.7895435094833374,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03710149601101875,
+      "rewards/margins": 0.009529616683721542,
+      "rewards/rejected": -0.04663110896945,
+      "step": 610
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 7.375,
+      "learning_rate": 2.0080483222562476e-06,
+      "log_odds_chosen": 0.44583791494369507,
+      "log_odds_ratio": -0.5645539164543152,
+      "logits/chosen": -2.3024137020111084,
+      "logits/rejected": -2.244235038757324,
+      "logps/chosen": -0.748982310295105,
+      "logps/rejected": -0.9248701930046082,
+      "loss": 0.8177,
+      "nll_loss": 0.8224814534187317,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.03744911774992943,
+      "rewards/margins": 0.008794394321739674,
+      "rewards/rejected": -0.04624350741505623,
+      "step": 620
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 7.4375,
+      "learning_rate": 1.9920476822239895e-06,
+      "log_odds_chosen": 0.3953856825828552,
+      "log_odds_ratio": -0.5933858156204224,
+      "logits/chosen": -2.347012758255005,
+      "logits/rejected": -2.2884857654571533,
+      "logps/chosen": -0.7635630965232849,
+      "logps/rejected": -0.9257116317749023,
+      "loss": 0.8293,
+      "nll_loss": 0.8256405591964722,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.03817816078662872,
+      "rewards/margins": 0.008107425644993782,
+      "rewards/rejected": -0.046285584568977356,
+      "step": 630
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 7.3125,
+      "learning_rate": 1.976423537605237e-06,
+      "log_odds_chosen": 0.5705650448799133,
+      "log_odds_ratio": -0.547913134098053,
+      "logits/chosen": -2.304575204849243,
+      "logits/rejected": -2.2619354724884033,
+      "logps/chosen": -0.6860629916191101,
+      "logps/rejected": -0.9498222470283508,
+      "loss": 0.8156,
+      "nll_loss": 0.7438285946846008,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.034303151071071625,
+      "rewards/margins": 0.01318796444684267,
+      "rewards/rejected": -0.04749111086130142,
+      "step": 640
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 7.5625,
+      "learning_rate": 1.961161351381841e-06,
+      "log_odds_chosen": 0.5407294034957886,
+      "log_odds_ratio": -0.5611921548843384,
+      "logits/chosen": -2.366647243499756,
+      "logits/rejected": -2.3279268741607666,
+      "logps/chosen": -0.7047677636146545,
+      "logps/rejected": -0.9417093396186829,
+      "loss": 0.8069,
+      "nll_loss": 0.7865225076675415,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.03523838892579079,
+      "rewards/margins": 0.011847076937556267,
+      "rewards/rejected": -0.0470854714512825,
+      "step": 650
+    },
+    {
+      "epoch": 2.98,
+      "grad_norm": 7.8125,
+      "learning_rate": 1.9462473604038077e-06,
+      "log_odds_chosen": 0.4032713770866394,
+      "log_odds_ratio": -0.5882443189620972,
+      "logits/chosen": -2.343567371368408,
+      "logits/rejected": -2.28145432472229,
+      "logps/chosen": -0.7442666292190552,
+      "logps/rejected": -0.9124624133110046,
+      "loss": 0.7916,
+      "nll_loss": 0.8193455934524536,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.03721333295106888,
+      "rewards/margins": 0.008409791626036167,
+      "rewards/rejected": -0.04562311992049217,
+      "step": 660
+    },
+    {
+      "epoch": 3.0,
+      "step": 663,
+      "total_flos": 0.0,
+      "train_loss": 0.9363477402682757,
+      "train_runtime": 28864.8871,
+      "train_samples_per_second": 1.471,
+      "train_steps_per_second": 0.023
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 663,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1699ac37006f71a342ac203a105cf538c43bc8907626c13921563cfd7d7d36ce
+size 5304