update

iter0/README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- alignment-handbook
+- generated_from_trainer
+datasets:
+- ./data_may/train/reward_max_optune/n_sample2_filter1.0/iter0
+model-index:
+- name: iter0_0.1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# iter0_0.1
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the ./data_may/train/reward_max_optune/n_sample2_filter1.0/iter0 dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.37.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

iter0/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.2572693068186442,
+    "train_runtime": 6192.5005,
+    "train_samples": 48000,
+    "train_samples_per_second": 7.751,
+    "train_steps_per_second": 0.061
+}

iter0/checkpoint-375/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

iter0/checkpoint-375/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.37.0"
+}

iter0/checkpoint-375/global_step375/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37fec716d65a951da558ffad957a64fd9c8f7ecc05bc53f089233758c52a0cef
+size 7241735056

iter0/checkpoint-375/global_step375/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cb9cff4032ffcdea1e133dba6d74afdf34444b02d20e04ef4981f34dff6e273
+size 7241735056

iter0/checkpoint-375/global_step375/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95a3d3cc13054e29ef78e1ccec049c8d711ab39efc847aef466754305af27a3c
+size 7241735056

iter0/checkpoint-375/global_step375/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58bbb16b1f72b7c67fa0060e814d701763252c939df0e20fb6215cb19695a07b
+size 7241735056

iter0/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

iter0/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.37.0"
+}

iter0/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58624d559edea9d97e28af2bfbb6abed968f44cd0dd1e21f68048839c279da71
+size 4943162336

iter0/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a97a64d20f2985250b0eee1ee5c1115b69cdb8d095bb02a97812ab429d6c847c
+size 4999819336

iter0/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d3c18bae5a8f67f5569344f4e0409dd2225ec4336baf966bf316b11b7536e3e
+size 4540516344

iter0/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

iter0/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

iter0/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}

iter0/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.2572693068186442,
+    "train_runtime": 6192.5005,
+    "train_samples": 48000,
+    "train_samples_per_second": 7.751,
+    "train_steps_per_second": 0.061
+}

iter0/trainer_state.json

@@ -0,0 +1,562 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 375,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3157894736842104e-08,
+      "logits/generated": -3.0232396125793457,
+      "logits/real": -2.996844530105591,
+      "logps/generated": -291.56793212890625,
+      "logps/real": -340.7873840332031,
+      "loss": 0.3645,
+      "rewards/accuracies": 0.0,
+      "rewards/generated": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/real": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3157894736842104e-07,
+      "logits/generated": -2.977639675140381,
+      "logits/real": -2.9781062602996826,
+      "logps/generated": -338.7113037109375,
+      "logps/real": -360.56146240234375,
+      "loss": 0.3584,
+      "rewards/accuracies": 0.5555555820465088,
+      "rewards/generated": 0.10237760096788406,
+      "rewards/margins": 0.04294492304325104,
+      "rewards/real": 0.1453225314617157,
+      "step": 10
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.631578947368421e-07,
+      "logits/generated": -2.9867026805877686,
+      "logits/real": -2.990659236907959,
+      "logps/generated": -371.62164306640625,
+      "logps/real": -372.09954833984375,
+      "loss": 0.3391,
+      "rewards/accuracies": 0.59375,
+      "rewards/generated": 0.5660532712936401,
+      "rewards/margins": 0.15894225239753723,
+      "rewards/real": 0.7249955534934998,
+      "step": 20
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.9473684210526315e-07,
+      "logits/generated": -2.940207004547119,
+      "logits/real": -2.945539951324463,
+      "logps/generated": -323.21282958984375,
+      "logps/real": -323.20733642578125,
+      "loss": 0.3061,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/generated": 0.2973577380180359,
+      "rewards/margins": 0.29796674847602844,
+      "rewards/real": 0.5953244566917419,
+      "step": 30
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.970326409495548e-07,
+      "logits/generated": -2.849879026412964,
+      "logits/real": -2.868879556655884,
+      "logps/generated": -339.9267578125,
+      "logps/real": -348.660400390625,
+      "loss": 0.3043,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/generated": -0.18336713314056396,
+      "rewards/margins": 0.4493914246559143,
+      "rewards/real": 0.26602429151535034,
+      "step": 40
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.821958456973294e-07,
+      "logits/generated": -2.8244385719299316,
+      "logits/real": -2.819532871246338,
+      "logps/generated": -345.12353515625,
+      "logps/real": -345.24334716796875,
+      "loss": 0.2707,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/generated": -1.0958898067474365,
+      "rewards/margins": 0.7357537150382996,
+      "rewards/real": -0.36013612151145935,
+      "step": 50
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.673590504451038e-07,
+      "logits/generated": -2.7510242462158203,
+      "logits/real": -2.744049549102783,
+      "logps/generated": -343.3367614746094,
+      "logps/real": -353.568115234375,
+      "loss": 0.2658,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/generated": -1.349844217300415,
+      "rewards/margins": 0.7489473819732666,
+      "rewards/real": -0.6008970141410828,
+      "step": 60
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.5252225519287835e-07,
+      "logits/generated": -2.787135362625122,
+      "logits/real": -2.7906911373138428,
+      "logps/generated": -380.27276611328125,
+      "logps/real": -390.9748840332031,
+      "loss": 0.2682,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/generated": -1.3749873638153076,
+      "rewards/margins": 0.8838955760002136,
+      "rewards/real": -0.49109163880348206,
+      "step": 70
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.376854599406528e-07,
+      "logits/generated": -2.7812376022338867,
+      "logits/real": -2.79952073097229,
+      "logps/generated": -352.7367858886719,
+      "logps/real": -343.9632873535156,
+      "loss": 0.2784,
+      "rewards/accuracies": 0.78125,
+      "rewards/generated": -1.779193639755249,
+      "rewards/margins": 1.1407415866851807,
+      "rewards/real": -0.6384519934654236,
+      "step": 80
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.228486646884273e-07,
+      "logits/generated": -2.80656099319458,
+      "logits/real": -2.7876017093658447,
+      "logps/generated": -369.83990478515625,
+      "logps/real": -381.7880859375,
+      "loss": 0.2742,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/generated": -1.4693442583084106,
+      "rewards/margins": 0.8362933993339539,
+      "rewards/real": -0.6330507397651672,
+      "step": 90
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.0801186943620176e-07,
+      "logits/generated": -2.7452383041381836,
+      "logits/real": -2.7657182216644287,
+      "logps/generated": -354.4010314941406,
+      "logps/real": -359.81219482421875,
+      "loss": 0.2657,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/generated": -1.3197325468063354,
+      "rewards/margins": 0.8461551666259766,
+      "rewards/real": -0.4735774099826813,
+      "step": 100
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.931750741839762e-07,
+      "logits/generated": -2.8132920265197754,
+      "logits/real": -2.8043882846832275,
+      "logps/generated": -357.61383056640625,
+      "logps/real": -354.3050537109375,
+      "loss": 0.2716,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/generated": -1.6764816045761108,
+      "rewards/margins": 0.9828389883041382,
+      "rewards/real": -0.6936424374580383,
+      "step": 110
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.7833827893175073e-07,
+      "logits/generated": -2.825605869293213,
+      "logits/real": -2.8103888034820557,
+      "logps/generated": -365.675537109375,
+      "logps/real": -368.09197998046875,
+      "loss": 0.2543,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/generated": -1.922782301902771,
+      "rewards/margins": 1.292311191558838,
+      "rewards/real": -0.6304711103439331,
+      "step": 120
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.635014836795252e-07,
+      "logits/generated": -2.7962846755981445,
+      "logits/real": -2.795644521713257,
+      "logps/generated": -340.1669006347656,
+      "logps/real": -348.66583251953125,
+      "loss": 0.2602,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/generated": -1.8341821432113647,
+      "rewards/margins": 0.9289523959159851,
+      "rewards/real": -0.9052297472953796,
+      "step": 130
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.486646884272997e-07,
+      "logits/generated": -2.7868337631225586,
+      "logits/real": -2.7795639038085938,
+      "logps/generated": -358.3647766113281,
+      "logps/real": -362.6192321777344,
+      "loss": 0.2584,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/generated": -1.9304630756378174,
+      "rewards/margins": 1.0024542808532715,
+      "rewards/real": -0.9280086755752563,
+      "step": 140
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3382789317507414e-07,
+      "logits/generated": -2.7806317806243896,
+      "logits/real": -2.773284435272217,
+      "logps/generated": -392.99273681640625,
+      "logps/real": -388.6888732910156,
+      "loss": 0.2429,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/generated": -2.132871389389038,
+      "rewards/margins": 0.8601642847061157,
+      "rewards/real": -1.272707223892212,
+      "step": 150
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.189910979228487e-07,
+      "logits/generated": -2.7479450702667236,
+      "logits/real": -2.7415106296539307,
+      "logps/generated": -384.2051086425781,
+      "logps/real": -382.9107360839844,
+      "loss": 0.2518,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/generated": -2.3066015243530273,
+      "rewards/margins": 1.2394059896469116,
+      "rewards/real": -1.0671956539154053,
+      "step": 160
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.0415430267062316e-07,
+      "logits/generated": -2.7207372188568115,
+      "logits/real": -2.6968023777008057,
+      "logps/generated": -351.6153259277344,
+      "logps/real": -358.0864562988281,
+      "loss": 0.247,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/generated": -2.2227485179901123,
+      "rewards/margins": 1.4021742343902588,
+      "rewards/real": -0.820574164390564,
+      "step": 170
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.893175074183976e-07,
+      "logits/generated": -2.687243700027466,
+      "logits/real": -2.6896092891693115,
+      "logps/generated": -340.67498779296875,
+      "logps/real": -325.22259521484375,
+      "loss": 0.2683,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/generated": -2.342029094696045,
+      "rewards/margins": 1.0233131647109985,
+      "rewards/real": -1.3187161684036255,
+      "step": 180
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.744807121661721e-07,
+      "logits/generated": -2.709791421890259,
+      "logits/real": -2.73317289352417,
+      "logps/generated": -396.40606689453125,
+      "logps/real": -388.1844482421875,
+      "loss": 0.2442,
+      "rewards/accuracies": 0.78125,
+      "rewards/generated": -2.465156078338623,
+      "rewards/margins": 1.4016426801681519,
+      "rewards/real": -1.063513159751892,
+      "step": 190
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.596439169139466e-07,
+      "logits/generated": -2.7428107261657715,
+      "logits/real": -2.7355589866638184,
+      "logps/generated": -368.4299011230469,
+      "logps/real": -373.0939025878906,
+      "loss": 0.2451,
+      "rewards/accuracies": 0.75,
+      "rewards/generated": -2.1397910118103027,
+      "rewards/margins": 1.308272123336792,
+      "rewards/real": -0.8315190076828003,
+      "step": 200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4480712166172106e-07,
+      "logits/generated": -2.703258752822876,
+      "logits/real": -2.693305015563965,
+      "logps/generated": -339.4871826171875,
+      "logps/real": -326.2037658691406,
+      "loss": 0.2395,
+      "rewards/accuracies": 0.78125,
+      "rewards/generated": -2.3122100830078125,
+      "rewards/margins": 1.2954694032669067,
+      "rewards/real": -1.0167406797409058,
+      "step": 210
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.2997032640949554e-07,
+      "logits/generated": -2.7212119102478027,
+      "logits/real": -2.716545581817627,
+      "logps/generated": -339.74267578125,
+      "logps/real": -346.297607421875,
+      "loss": 0.2458,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/generated": -1.9660396575927734,
+      "rewards/margins": 1.078840970993042,
+      "rewards/real": -0.8871987462043762,
+      "step": 220
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1513353115727e-07,
+      "logits/generated": -2.77765154838562,
+      "logits/real": -2.7591769695281982,
+      "logps/generated": -386.1648864746094,
+      "logps/real": -381.2674560546875,
+      "loss": 0.2324,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/generated": -2.414008617401123,
+      "rewards/margins": 1.5327675342559814,
+      "rewards/real": -0.8812410235404968,
+      "step": 230
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 2.0029673590504451e-07,
+      "logits/generated": -2.7021536827087402,
+      "logits/real": -2.70768666267395,
+      "logps/generated": -354.3561706542969,
+      "logps/real": -353.68212890625,
+      "loss": 0.2492,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/generated": -2.4238786697387695,
+      "rewards/margins": 1.27177894115448,
+      "rewards/real": -1.152099847793579,
+      "step": 240
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.8545994065281897e-07,
+      "logits/generated": -2.7076125144958496,
+      "logits/real": -2.7352890968322754,
+      "logps/generated": -365.26214599609375,
+      "logps/real": -355.78564453125,
+      "loss": 0.2426,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/generated": -2.567624092102051,
+      "rewards/margins": 1.3714964389801025,
+      "rewards/real": -1.1961278915405273,
+      "step": 250
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.7062314540059346e-07,
+      "logits/generated": -2.7474026679992676,
+      "logits/real": -2.733513593673706,
+      "logps/generated": -370.26568603515625,
+      "logps/real": -366.9493713378906,
+      "loss": 0.2496,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/generated": -2.5266714096069336,
+      "rewards/margins": 1.390491247177124,
+      "rewards/real": -1.1361799240112305,
+      "step": 260
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.5578635014836795e-07,
+      "logits/generated": -2.7382729053497314,
+      "logits/real": -2.7590155601501465,
+      "logps/generated": -339.4982604980469,
+      "logps/real": -354.5415954589844,
+      "loss": 0.2407,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/generated": -2.284700870513916,
+      "rewards/margins": 1.1465342044830322,
+      "rewards/real": -1.1381666660308838,
+      "step": 270
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.4094955489614243e-07,
+      "logits/generated": -2.6945815086364746,
+      "logits/real": -2.695988416671753,
+      "logps/generated": -373.51385498046875,
+      "logps/real": -350.8352966308594,
+      "loss": 0.2303,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/generated": -2.4922609329223633,
+      "rewards/margins": 1.3119118213653564,
+      "rewards/real": -1.1803491115570068,
+      "step": 280
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.261127596439169e-07,
+      "logits/generated": -2.7670834064483643,
+      "logits/real": -2.7600436210632324,
+      "logps/generated": -344.09136962890625,
+      "logps/real": -337.3023376464844,
+      "loss": 0.2435,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/generated": -2.637000799179077,
+      "rewards/margins": 1.540818452835083,
+      "rewards/real": -1.0961825847625732,
+      "step": 290
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1127596439169139e-07,
+      "logits/generated": -2.6610119342803955,
+      "logits/real": -2.6668756008148193,
+      "logps/generated": -342.7873229980469,
+      "logps/real": -330.2555847167969,
+      "loss": 0.2424,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/generated": -2.5096726417541504,
+      "rewards/margins": 1.4919517040252686,
+      "rewards/real": -1.0177206993103027,
+      "step": 300
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.643916913946587e-08,
+      "logits/generated": -2.7119061946868896,
+      "logits/real": -2.736443519592285,
+      "logps/generated": -364.1079406738281,
+      "logps/real": -365.68963623046875,
+      "loss": 0.2316,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/generated": -3.0718140602111816,
+      "rewards/margins": 1.5135959386825562,
+      "rewards/real": -1.5582183599472046,
+      "step": 310
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.160237388724035e-08,
+      "logits/generated": -2.7183382511138916,
+      "logits/real": -2.735018253326416,
+      "logps/generated": -394.9755859375,
+      "logps/real": -377.31427001953125,
+      "loss": 0.2359,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/generated": -2.73679256439209,
+      "rewards/margins": 1.1780710220336914,
+      "rewards/real": -1.558721661567688,
+      "step": 320
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.676557863501484e-08,
+      "logits/generated": -2.7515358924865723,
+      "logits/real": -2.742940664291382,
+      "logps/generated": -388.3130187988281,
+      "logps/real": -372.29437255859375,
+      "loss": 0.234,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/generated": -2.38558030128479,
+      "rewards/margins": 1.1235764026641846,
+      "rewards/real": -1.2620038986206055,
+      "step": 330
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 5.192878338278932e-08,
+      "logits/generated": -2.695279121398926,
+      "logits/real": -2.6978631019592285,
+      "logps/generated": -365.2856140136719,
+      "logps/real": -363.0904235839844,
+      "loss": 0.2303,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/generated": -2.723789930343628,
+      "rewards/margins": 1.42342209815979,
+      "rewards/real": -1.300368070602417,
+      "step": 340
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.709198813056379e-08,
+      "logits/generated": -2.662724018096924,
+      "logits/real": -2.675875186920166,
+      "logps/generated": -338.28704833984375,
+      "logps/real": -342.17462158203125,
+      "loss": 0.2333,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/generated": -2.581637144088745,
+      "rewards/margins": 1.3430696725845337,
+      "rewards/real": -1.2385674715042114,
+      "step": 350
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.225519287833828e-08,
+      "logits/generated": -2.6803088188171387,
+      "logits/real": -2.704144239425659,
+      "logps/generated": -356.77703857421875,
+      "logps/real": -359.313720703125,
+      "loss": 0.2368,
+      "rewards/accuracies": 0.75,
+      "rewards/generated": -2.8369853496551514,
+      "rewards/margins": 1.2829147577285767,
+      "rewards/real": -1.554070234298706,
+      "step": 360
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.418397626112759e-09,
+      "logits/generated": -2.7113311290740967,
+      "logits/real": -2.7457308769226074,
+      "logps/generated": -404.06756591796875,
+      "logps/real": -393.70843505859375,
+      "loss": 0.2369,
+      "rewards/accuracies": 0.78125,
+      "rewards/generated": -2.8109331130981445,
+      "rewards/margins": 1.4008702039718628,
+      "rewards/real": -1.4100630283355713,
+      "step": 370
+    },
+    {
+      "epoch": 1.0,
+      "step": 375,
+      "total_flos": 0.0,
+      "train_loss": 0.2572693068186442,
+      "train_runtime": 6192.5005,
+      "train_samples_per_second": 7.751,
+      "train_steps_per_second": 0.061
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 375,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

iter0/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad1a8e08a20d9849ba4854028576185ed1bfd04bb523ad1f1c37cd201163361
+size 5944