Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: Positive_like_DPO_Chat-zephyr-7b-sft-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Positive_like_DPO_Chat-zephyr-7b-sft-full
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 244.87338864628373,
+    "train_runtime": 12795.3282,
+    "train_samples": 61135,
+    "train_samples_per_second": 4.778,
+    "train_steps_per_second": 0.037
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.2",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.41.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf6d0b46939441c91d0eb967e404759e346cf61c45a684972f08a92926dc610a
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:602ad54b55433436a3bdd772a46c9626bff47503675a67c1c0fb058c81be3525
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e6b8bb2952a88ef1d915f406f38779d26089fe7ea07c670b5be444bcc4444cf
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 244.87338864628373,
+    "train_runtime": 12795.3282,
+    "train_samples": 61135,
+    "train_samples_per_second": 4.778,
+    "train_steps_per_second": 0.037
+}

trainer_state.json

@@ -0,0 +1,762 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998691442030882,
+  "eval_steps": 500,
+  "global_step": 477,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002093692750588851,
+      "grad_norm": 3535.8126325665735,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": 5002.53564453125,
+      "logits/rejected": 4591.72021484375,
+      "logps/chosen": -265.6396789550781,
+      "logps/rejected": -206.22401428222656,
+      "loss": 280.8964,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 3583.3887780950436,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": 5976.96337890625,
+      "logits/rejected": 5015.21337890625,
+      "logps/chosen": -292.321044921875,
+      "logps/rejected": -259.08831787109375,
+      "loss": 287.6974,
+      "rewards/accuracies": 0.4548611044883728,
+      "rewards/chosen": 0.00239731022156775,
+      "rewards/margins": 4.432235436979681e-05,
+      "rewards/rejected": 0.0023529878817498684,
+      "step": 10
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 1937.0995896878487,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": 5865.89453125,
+      "logits/rejected": 4845.67138671875,
+      "logps/chosen": -283.9303283691406,
+      "logps/rejected": -243.36111450195312,
+      "loss": 276.4462,
+      "rewards/accuracies": 0.534375011920929,
+      "rewards/chosen": 0.06663157045841217,
+      "rewards/margins": 0.0029086393769830465,
+      "rewards/rejected": 0.06372293084859848,
+      "step": 20
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 1357.5939676600453,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 5566.55859375,
+      "logits/rejected": 4849.14208984375,
+      "logps/chosen": -256.52099609375,
+      "logps/rejected": -223.9065399169922,
+      "loss": 259.9777,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.15716445446014404,
+      "rewards/margins": 0.008207684382796288,
+      "rewards/rejected": 0.1489567756652832,
+      "step": 30
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 1133.6293010511845,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": 5576.7578125,
+      "logits/rejected": 4843.4580078125,
+      "logps/chosen": -252.6702423095703,
+      "logps/rejected": -238.50485229492188,
+      "loss": 262.5837,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": 0.21286813914775848,
+      "rewards/margins": 0.013377232477068901,
+      "rewards/rejected": 0.19949090480804443,
+      "step": 40
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 1156.1788135816998,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": 5815.3984375,
+      "logits/rejected": 5046.6923828125,
+      "logps/chosen": -254.2016143798828,
+      "logps/rejected": -244.6356658935547,
+      "loss": 256.2513,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.24416108429431915,
+      "rewards/margins": 0.015325096435844898,
+      "rewards/rejected": 0.22883598506450653,
+      "step": 50
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 1067.5976503019779,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": 5575.5048828125,
+      "logits/rejected": 4749.95703125,
+      "logps/chosen": -249.46359252929688,
+      "logps/rejected": -224.8580322265625,
+      "loss": 242.7049,
+      "rewards/accuracies": 0.5406249761581421,
+      "rewards/chosen": 0.2895886301994324,
+      "rewards/margins": 0.023379303514957428,
+      "rewards/rejected": 0.26620933413505554,
+      "step": 60
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 1053.4897470709532,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": 5370.1044921875,
+      "logits/rejected": 4955.6025390625,
+      "logps/chosen": -248.7747039794922,
+      "logps/rejected": -239.68154907226562,
+      "loss": 242.2774,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": 0.29533231258392334,
+      "rewards/margins": 0.02411496639251709,
+      "rewards/rejected": 0.27121734619140625,
+      "step": 70
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 1023.2390689553637,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": 5459.8798828125,
+      "logits/rejected": 4685.32275390625,
+      "logps/chosen": -244.05105590820312,
+      "logps/rejected": -224.86630249023438,
+      "loss": 242.3456,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": 0.32035860419273376,
+      "rewards/margins": 0.03829706832766533,
+      "rewards/rejected": 0.28206151723861694,
+      "step": 80
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 1040.9032310821044,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": 5580.447265625,
+      "logits/rejected": 4722.6728515625,
+      "logps/chosen": -243.3023223876953,
+      "logps/rejected": -223.93185424804688,
+      "loss": 242.2394,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": 0.3440525531768799,
+      "rewards/margins": 0.0336349718272686,
+      "rewards/rejected": 0.3104175925254822,
+      "step": 90
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 1122.126586948237,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": 5649.5400390625,
+      "logits/rejected": 5218.107421875,
+      "logps/chosen": -244.94400024414062,
+      "logps/rejected": -239.73779296875,
+      "loss": 242.0253,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": 0.34924736618995667,
+      "rewards/margins": 0.0014709115494042635,
+      "rewards/rejected": 0.34777647256851196,
+      "step": 100
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 997.7885835864053,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": 5795.6767578125,
+      "logits/rejected": 4699.9208984375,
+      "logps/chosen": -237.4701690673828,
+      "logps/rejected": -226.93594360351562,
+      "loss": 240.8889,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.37152451276779175,
+      "rewards/margins": 0.05216788500547409,
+      "rewards/rejected": 0.31935662031173706,
+      "step": 110
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 1067.2738806193756,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": 5398.6474609375,
+      "logits/rejected": 5022.0517578125,
+      "logps/chosen": -232.2193603515625,
+      "logps/rejected": -233.2302703857422,
+      "loss": 245.0688,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.3495965003967285,
+      "rewards/margins": 0.013141113333404064,
+      "rewards/rejected": 0.3364553451538086,
+      "step": 120
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 1101.9933538265811,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": 5447.984375,
+      "logits/rejected": 4856.47216796875,
+      "logps/chosen": -234.07583618164062,
+      "logps/rejected": -235.1103973388672,
+      "loss": 245.9864,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": 0.3444148600101471,
+      "rewards/margins": 0.005432566162198782,
+      "rewards/rejected": 0.3389822840690613,
+      "step": 130
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 1062.9847754936125,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": 5238.33349609375,
+      "logits/rejected": 4664.427734375,
+      "logps/chosen": -236.94302368164062,
+      "logps/rejected": -218.821044921875,
+      "loss": 247.7552,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.3496176600456238,
+      "rewards/margins": 0.026499617844820023,
+      "rewards/rejected": 0.32311806082725525,
+      "step": 140
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 1368.1757226951288,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": 5794.70458984375,
+      "logits/rejected": 4785.361328125,
+      "logps/chosen": -253.90975952148438,
+      "logps/rejected": -218.62548828125,
+      "loss": 241.2088,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.35205191373825073,
+      "rewards/margins": 0.023399515077471733,
+      "rewards/rejected": 0.32865244150161743,
+      "step": 150
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 1052.0276681831142,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": 5505.08154296875,
+      "logits/rejected": 4592.50927734375,
+      "logps/chosen": -242.8617706298828,
+      "logps/rejected": -221.708251953125,
+      "loss": 243.5824,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.4173237681388855,
+      "rewards/margins": 0.07062368839979172,
+      "rewards/rejected": 0.34670013189315796,
+      "step": 160
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 1072.8144440956282,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": 5766.25244140625,
+      "logits/rejected": 4925.43017578125,
+      "logps/chosen": -249.1059112548828,
+      "logps/rejected": -209.27963256835938,
+      "loss": 248.3713,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.3320981562137604,
+      "rewards/margins": 0.01844380795955658,
+      "rewards/rejected": 0.3136543333530426,
+      "step": 170
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 983.0618691372457,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": 5427.0126953125,
+      "logits/rejected": 4671.05224609375,
+      "logps/chosen": -251.17276000976562,
+      "logps/rejected": -214.9893035888672,
+      "loss": 245.643,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": 0.3612250089645386,
+      "rewards/margins": 0.011310195550322533,
+      "rewards/rejected": 0.3499148190021515,
+      "step": 180
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 1005.4792953444737,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": 5555.00634765625,
+      "logits/rejected": 4690.0107421875,
+      "logps/chosen": -232.8914337158203,
+      "logps/rejected": -213.5168914794922,
+      "loss": 244.4388,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.3686008155345917,
+      "rewards/margins": 0.024937449023127556,
+      "rewards/rejected": 0.34366339445114136,
+      "step": 190
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 1095.9872681623817,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": 5196.1396484375,
+      "logits/rejected": 4525.6201171875,
+      "logps/chosen": -218.52597045898438,
+      "logps/rejected": -209.1753692626953,
+      "loss": 242.2741,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": 0.3582485318183899,
+      "rewards/margins": 0.02139531448483467,
+      "rewards/rejected": 0.33685317635536194,
+      "step": 200
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 1032.5136886828875,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": 5817.4052734375,
+      "logits/rejected": 4964.2431640625,
+      "logps/chosen": -250.0350341796875,
+      "logps/rejected": -227.2427978515625,
+      "loss": 242.9751,
+      "rewards/accuracies": 0.5406249761581421,
+      "rewards/chosen": 0.3673235774040222,
+      "rewards/margins": 0.022716034203767776,
+      "rewards/rejected": 0.34460753202438354,
+      "step": 210
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 1069.3112273505162,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": 5231.8330078125,
+      "logits/rejected": 4868.0009765625,
+      "logps/chosen": -235.4297332763672,
+      "logps/rejected": -224.67996215820312,
+      "loss": 247.8473,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.35685989260673523,
+      "rewards/margins": 0.00022467300004791468,
+      "rewards/rejected": 0.3566352427005768,
+      "step": 220
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 1038.3755229403357,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": 5424.3896484375,
+      "logits/rejected": 4603.22265625,
+      "logps/chosen": -229.6114959716797,
+      "logps/rejected": -212.58535766601562,
+      "loss": 235.744,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.3865725100040436,
+      "rewards/margins": 0.05810902267694473,
+      "rewards/rejected": 0.32846349477767944,
+      "step": 230
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 1058.0373868868155,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": 5579.22265625,
+      "logits/rejected": 4746.73291015625,
+      "logps/chosen": -228.1630401611328,
+      "logps/rejected": -214.03872680664062,
+      "loss": 246.103,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.3741385042667389,
+      "rewards/margins": 0.01671653240919113,
+      "rewards/rejected": 0.35742202401161194,
+      "step": 240
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 1066.8993672604806,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": 5711.28759765625,
+      "logits/rejected": 4824.6904296875,
+      "logps/chosen": -245.55361938476562,
+      "logps/rejected": -227.17333984375,
+      "loss": 243.4184,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.4039505124092102,
+      "rewards/margins": 0.054931532591581345,
+      "rewards/rejected": 0.34901896119117737,
+      "step": 250
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 983.2388962195462,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": 5527.0869140625,
+      "logits/rejected": 5007.7177734375,
+      "logps/chosen": -246.20358276367188,
+      "logps/rejected": -224.95919799804688,
+      "loss": 244.2827,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": 0.3986549377441406,
+      "rewards/margins": 0.037424616515636444,
+      "rewards/rejected": 0.36123034358024597,
+      "step": 260
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 1105.5507497790131,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": 5603.63916015625,
+      "logits/rejected": 5142.90576171875,
+      "logps/chosen": -234.740478515625,
+      "logps/rejected": -224.26904296875,
+      "loss": 234.9617,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.39873379468917847,
+      "rewards/margins": 0.03745921701192856,
+      "rewards/rejected": 0.3612746000289917,
+      "step": 270
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 1034.2532678101588,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": 5626.45556640625,
+      "logits/rejected": 4683.701171875,
+      "logps/chosen": -242.08908081054688,
+      "logps/rejected": -211.0960235595703,
+      "loss": 233.7415,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": 0.3863658607006073,
+      "rewards/margins": 0.02948523685336113,
+      "rewards/rejected": 0.35688066482543945,
+      "step": 280
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 1148.0848744279972,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": 5876.2919921875,
+      "logits/rejected": 5516.24072265625,
+      "logps/chosen": -250.70028686523438,
+      "logps/rejected": -249.01510620117188,
+      "loss": 245.2798,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.40733808279037476,
+      "rewards/margins": -0.0817265659570694,
+      "rewards/rejected": 0.48906460404396057,
+      "step": 290
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 1067.1492136203478,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": 5382.34765625,
+      "logits/rejected": 5070.5087890625,
+      "logps/chosen": -234.3385467529297,
+      "logps/rejected": -230.91720581054688,
+      "loss": 239.1968,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.37011438608169556,
+      "rewards/margins": -0.029902536422014236,
+      "rewards/rejected": 0.4000169336795807,
+      "step": 300
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 1038.567182284818,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": 6013.2529296875,
+      "logits/rejected": 5287.40478515625,
+      "logps/chosen": -247.5303192138672,
+      "logps/rejected": -233.228759765625,
+      "loss": 237.5939,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.43299251794815063,
+      "rewards/margins": 0.06971301883459091,
+      "rewards/rejected": 0.3632794916629791,
+      "step": 310
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 1204.0142191428097,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": 5575.4404296875,
+      "logits/rejected": 4841.73974609375,
+      "logps/chosen": -233.0708465576172,
+      "logps/rejected": -221.49600219726562,
+      "loss": 242.2518,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": 0.38792601227760315,
+      "rewards/margins": 0.004761689808219671,
+      "rewards/rejected": 0.38316434621810913,
+      "step": 320
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 1019.620900443292,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": 5387.96044921875,
+      "logits/rejected": 5011.3662109375,
+      "logps/chosen": -238.8975067138672,
+      "logps/rejected": -226.3376007080078,
+      "loss": 238.7415,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.3788672685623169,
+      "rewards/margins": -0.0010509025305509567,
+      "rewards/rejected": 0.3799181580543518,
+      "step": 330
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 997.6449734870048,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": 5218.39013671875,
+      "logits/rejected": 4540.35986328125,
+      "logps/chosen": -217.21044921875,
+      "logps/rejected": -205.6111297607422,
+      "loss": 231.9898,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.403564989566803,
+      "rewards/margins": 0.04764997214078903,
+      "rewards/rejected": 0.35591498017311096,
+      "step": 340
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 1066.0615606169838,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": 5892.0224609375,
+      "logits/rejected": 4797.64306640625,
+      "logps/chosen": -258.8974914550781,
+      "logps/rejected": -231.6051025390625,
+      "loss": 251.9084,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": 0.42017000913619995,
+      "rewards/margins": 0.05283670499920845,
+      "rewards/rejected": 0.367333322763443,
+      "step": 350
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 1000.4777940737159,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": 5596.86962890625,
+      "logits/rejected": 4888.3076171875,
+      "logps/chosen": -236.48135375976562,
+      "logps/rejected": -220.30322265625,
+      "loss": 233.4907,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.4124221205711365,
+      "rewards/margins": 0.03293180465698242,
+      "rewards/rejected": 0.37949031591415405,
+      "step": 360
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 937.2926675342543,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": 5473.3173828125,
+      "logits/rejected": 4680.34521484375,
+      "logps/chosen": -251.95211791992188,
+      "logps/rejected": -224.4751739501953,
+      "loss": 239.6144,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.3684094548225403,
+      "rewards/margins": 0.04351681098341942,
+      "rewards/rejected": 0.3248926103115082,
+      "step": 370
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 1050.3155040091867,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": 5725.0947265625,
+      "logits/rejected": 5000.07470703125,
+      "logps/chosen": -246.28152465820312,
+      "logps/rejected": -213.35775756835938,
+      "loss": 252.1425,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.38419944047927856,
+      "rewards/margins": 0.00920055340975523,
+      "rewards/rejected": 0.37499889731407166,
+      "step": 380
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 1053.9561839695411,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": 5992.890625,
+      "logits/rejected": 5294.5634765625,
+      "logps/chosen": -247.9290313720703,
+      "logps/rejected": -235.806640625,
+      "loss": 238.8614,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.438203901052475,
+      "rewards/margins": 0.048503682017326355,
+      "rewards/rejected": 0.3897002339363098,
+      "step": 390
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 938.4991044304853,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": 5561.79296875,
+      "logits/rejected": 4942.47314453125,
+      "logps/chosen": -247.83755493164062,
+      "logps/rejected": -216.7385711669922,
+      "loss": 244.8828,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.4306658208370209,
+      "rewards/margins": 0.03947380185127258,
+      "rewards/rejected": 0.3911920487880707,
+      "step": 400
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 991.4623358360578,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": 5575.3115234375,
+      "logits/rejected": 4932.56201171875,
+      "logps/chosen": -235.03713989257812,
+      "logps/rejected": -220.0222625732422,
+      "loss": 239.7671,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": 0.4683019518852234,
+      "rewards/margins": -0.14257735013961792,
+      "rewards/rejected": 0.6108792424201965,
+      "step": 410
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 1039.9341468951118,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": 5915.91748046875,
+      "logits/rejected": 5172.326171875,
+      "logps/chosen": -250.53573608398438,
+      "logps/rejected": -234.00204467773438,
+      "loss": 240.5354,
+      "rewards/accuracies": 0.5406249761581421,
+      "rewards/chosen": 0.4144343435764313,
+      "rewards/margins": 0.029571443796157837,
+      "rewards/rejected": 0.38486289978027344,
+      "step": 420
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 1393.4874470921877,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": 5823.2763671875,
+      "logits/rejected": 4703.76025390625,
+      "logps/chosen": -245.7346954345703,
+      "logps/rejected": -221.4534149169922,
+      "loss": 239.5021,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.41867977380752563,
+      "rewards/margins": 0.05862581729888916,
+      "rewards/rejected": 0.3600539565086365,
+      "step": 430
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 974.1988194474727,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": 5649.8154296875,
+      "logits/rejected": 5389.01953125,
+      "logps/chosen": -241.44961547851562,
+      "logps/rejected": -239.0779266357422,
+      "loss": 232.5532,
+      "rewards/accuracies": 0.4906249940395355,
+      "rewards/chosen": 0.37093910574913025,
+      "rewards/margins": 0.0012592204147949815,
+      "rewards/rejected": 0.36967989802360535,
+      "step": 440
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 1054.0987809731553,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": 5798.8427734375,
+      "logits/rejected": 4999.23828125,
+      "logps/chosen": -249.7338104248047,
+      "logps/rejected": -220.97769165039062,
+      "loss": 238.5285,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.4060141146183014,
+      "rewards/margins": 0.0012865230673924088,
+      "rewards/rejected": 0.40472760796546936,
+      "step": 450
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 1120.0717473345896,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": 5987.23388671875,
+      "logits/rejected": 5070.90625,
+      "logps/chosen": -269.4898376464844,
+      "logps/rejected": -243.72256469726562,
+      "loss": 256.2592,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": 0.435092031955719,
+      "rewards/margins": 0.015190321020781994,
+      "rewards/rejected": 0.41990166902542114,
+      "step": 460
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 1126.7552362991885,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": 5581.7841796875,
+      "logits/rejected": 4960.31689453125,
+      "logps/chosen": -238.2722930908203,
+      "logps/rejected": -217.5271453857422,
+      "loss": 245.3545,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.42566928267478943,
+      "rewards/margins": 0.022912535816431046,
+      "rewards/rejected": 0.40275678038597107,
+      "step": 470
+    },
+    {
+      "epoch": 0.998691442030882,
+      "step": 477,
+      "total_flos": 0.0,
+      "train_loss": 244.87338864628373,
+      "train_runtime": 12795.3282,
+      "train_samples_per_second": 4.778,
+      "train_steps_per_second": 0.037
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 477,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b357f0f37a9f0ab171d4093877641370186158ab284b271fd0a157ff29caa633
+size 7352