Training in progress, step 180

Browse files

Files changed (15) hide show

README.md +85 -0
added_tokens.json +24 -0
all_results.json +23 -0
config.json +30 -0
eval_results.json +17 -0
generation_config.json +7 -0
merges.txt +0 -0
model.safetensors +3 -0
special_tokens_map.json +27 -0
tokenizer.json +0 -0
tokenizer_config.json +196 -0
train_results.json +9 -0
trainer_state.json +583 -0
training_args.bin +3 -0
vocab.json +0 -0

README.md ADDED Viewed

	@@ -0,0 +1,85 @@

+---
+license: apache-2.0
+base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
+tags:
+- alignment-handbook
+- ndcg
+- trl
+- expo
+- generated_from_trainer
+- trl
+- expo
+- generated_from_trainer
+datasets:
+- hZzy/train_pairwise
+model-index:
+- name: qwen2.5-0.5b-expo-DPO-ES-TRY
+  results: []
+---
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/dd7oiosi)
+# qwen2.5-0.5b-expo-DPO-ES-TRY
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on the hZzy/train_pairwise dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6866
+- Logps: -91.4116
+- Logits: -1.5339
+- Objective: 0.6926
+- Dpo Loss: 0.6926
+- Regularize: 0.6926
+- Ranking Simple: 0.5052
+- Ranking Idealized: 0.5888
+- Ranking Idealized Expo: 0.5093
+- Dpo Wo Beta: -0.9551
+## Model description
+More information needed
+## Intended uses & limitations
+More information needed
+## Training and evaluation data
+More information needed
+## Training procedure
+### Training hyperparameters
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 6
+- gradient_accumulation_steps: 12
+- total_train_batch_size: 288
+- total_eval_batch_size: 24
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 2
+### Training results
+| Training Loss | Epoch  | Step | Validation Loss | Logps    | Logits  | Objective | Dpo Loss | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo | Dpo Wo Beta |
+|:-------------:|:------:|:----:|:---------------:|:--------:|:-------:|:---------:|:--------:|:----------:|:--------------:|:-----------------:|:----------------------:|:-----------:|
+| 0.672         | 0.2041 | 36   | 0.6866          | -91.4116 | -1.5339 | 0.6926    | 0.6926   | 0.6926     | 0.5052         | 0.5888            | 0.5093                 | -0.9551     |
+| 0.6533        | 0.4081 | 72   | 0.6769          | -92.4758 | -1.6311 | 0.6885    | 0.6885   | 0.6885     | 0.5176         | 0.5888            | 0.5093                 | -1.1473     |
+| 0.604         | 0.6122 | 108  | 0.6773          | -94.0584 | -1.7691 | 0.6853    | 0.6853   | 0.6853     | 0.5186         | 0.5888            | 0.5093                 | -1.3961     |
+| 0.5911        | 0.8162 | 144  | 0.6775          | -95.7578 | -1.8165 | 0.6886    | 0.6886   | 0.6886     | 0.5186         | 0.5888            | 0.5093                 | -1.5153     |
+### Framework versions
+- Transformers 4.42.0
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

added_tokens.json ADDED Viewed

	@@ -0,0 +1,24 @@

+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json ADDED Viewed

	@@ -0,0 +1,23 @@

+{
+    "epoch": 0.8162494095418045,
+    "eval_dpo_loss": 0.6925506591796875,
+    "eval_dpo_wo_beta": -0.9551481008529663,
+    "eval_logits": -1.5338705778121948,
+    "eval_logps": -91.41157531738281,
+    "eval_loss": 0.6866393685340881,
+    "eval_objective": 0.6925506591796875,
+    "eval_ranking_idealized": 0.5888429880142212,
+    "eval_ranking_idealized_expo": 0.5092975497245789,
+    "eval_ranking_simple": 0.5051652789115906,
+    "eval_regularize": 0.6925506591796875,
+    "eval_runtime": 210.5773,
+    "eval_samples": 5790,
+    "eval_samples_per_second": 27.496,
+    "eval_steps_per_second": 1.149,
+    "total_flos": 0.0,
+    "train_loss": 0.6411769655015733,
+    "train_runtime": 5149.1432,
+    "train_samples": 50802,
+    "train_samples_per_second": 19.732,
+    "train_steps_per_second": 0.068
+}

config.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "_name_or_path": "hZzy/qwen2.5-0.5b-sft-news-IFT",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151645,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.42.0",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151665
+}

eval_results.json ADDED Viewed

	@@ -0,0 +1,17 @@

+{
+    "epoch": 0.8162494095418045,
+    "eval_dpo_loss": 0.6925506591796875,
+    "eval_dpo_wo_beta": -0.9551481008529663,
+    "eval_logits": -1.5338705778121948,
+    "eval_logps": -91.41157531738281,
+    "eval_loss": 0.6866393685340881,
+    "eval_objective": 0.6925506591796875,
+    "eval_ranking_idealized": 0.5888429880142212,
+    "eval_ranking_idealized_expo": 0.5092975497245789,
+    "eval_ranking_simple": 0.5051652789115906,
+    "eval_regularize": 0.6925506591796875,
+    "eval_runtime": 210.5773,
+    "eval_samples": 5790,
+    "eval_samples_per_second": 27.496,
+    "eval_steps_per_second": 1.149
+}

generation_config.json ADDED Viewed

	@@ -0,0 +1,7 @@

+{
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "max_new_tokens": 2048,
+  "pad_token_id": 151645,
+  "transformers_version": "4.42.0"
+}

merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

model.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:5171a258a6b8d8c41f45bc3540ce7b4ca6f5e1dff5f1e1ae03bec89afe2d1f74
+size 1975192208

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,27 @@

+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,196 @@

+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 0.8162494095418045,
+    "total_flos": 0.0,
+    "train_loss": 0.6411769655015733,
+    "train_runtime": 5149.1432,
+    "train_samples": 50802,
+    "train_samples_per_second": 19.732,
+    "train_steps_per_second": 0.068
+}

trainer_state.json ADDED Viewed

	@@ -0,0 +1,583 @@

+{
+  "best_metric": -0.9551481008529663,
+  "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-TRY/checkpoint-36",
+  "epoch": 0.8162494095418045,
+  "eval_steps": 36,
+  "global_step": 144,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "dpo_loss": 0.6931471824645996,
+      "dpo_wo_beta": -0.6931471824645996,
+      "epoch": 0.005668398677373642,
+      "grad_norm": 13.433596947813236,
+      "learning_rate": 1.3888888888888887e-08,
+      "logits": -1.3874311447143555,
+      "logps": -88.43561553955078,
+      "loss": 0.6931,
+      "objective": 0.6931471824645996,
+      "ranking_idealized": 0.625,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.6931471824645996,
+      "step": 1
+    },
+    {
+      "dpo_loss": 0.6931712031364441,
+      "dpo_wo_beta": -0.6935088634490967,
+      "epoch": 0.02834199338686821,
+      "grad_norm": 13.470599581187907,
+      "learning_rate": 6.944444444444444e-08,
+      "logits": -1.408589482307434,
+      "logps": -84.63459777832031,
+      "loss": 0.6931,
+      "objective": 0.6931712031364441,
+      "ranking_idealized": 0.6197916865348816,
+      "ranking_idealized_expo": 0.546875,
+      "ranking_simple": 0.546875,
+      "regularize": 0.6931712031364441,
+      "step": 5
+    },
+    {
+      "dpo_loss": 0.6912738680839539,
+      "dpo_wo_beta": -0.6787264943122864,
+      "epoch": 0.05668398677373642,
+      "grad_norm": 12.891994338762009,
+      "learning_rate": 1.3888888888888888e-07,
+      "logits": -1.4766913652420044,
+      "logps": -83.30558776855469,
+      "loss": 0.6922,
+      "objective": 0.6912738680839539,
+      "ranking_idealized": 0.6000000238418579,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5083333253860474,
+      "regularize": 0.6912738680839539,
+      "step": 10
+    },
+    {
+      "dpo_loss": 0.6917670369148254,
+      "dpo_wo_beta": -0.6925874352455139,
+      "epoch": 0.08502598016060463,
+      "grad_norm": 12.616091674274701,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits": -1.4627829790115356,
+      "logps": -83.51612854003906,
+      "loss": 0.691,
+      "objective": 0.6917670369148254,
+      "ranking_idealized": 0.5916666388511658,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.6917670369148254,
+      "step": 15
+    },
+    {
+      "dpo_loss": 0.6909351348876953,
+      "dpo_wo_beta": -0.7156455516815186,
+      "epoch": 0.11336797354747284,
+      "grad_norm": 13.131384103632906,
+      "learning_rate": 2.7777777777777776e-07,
+      "logits": -1.4652808904647827,
+      "logps": -83.81787109375,
+      "loss": 0.6882,
+      "objective": 0.6909351348876953,
+      "ranking_idealized": 0.625,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.6909351348876953,
+      "step": 20
+    },
+    {
+      "dpo_loss": 0.6833630800247192,
+      "dpo_wo_beta": -0.6689243316650391,
+      "epoch": 0.14170996693434104,
+      "grad_norm": 12.965314552644642,
+      "learning_rate": 3.472222222222222e-07,
+      "logits": -1.5261986255645752,
+      "logps": -83.36495971679688,
+      "loss": 0.6842,
+      "objective": 0.6833630800247192,
+      "ranking_idealized": 0.5958333611488342,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.5,
+      "regularize": 0.6833630800247192,
+      "step": 25
+    },
+    {
+      "dpo_loss": 0.6835373044013977,
+      "dpo_wo_beta": -0.7437445521354675,
+      "epoch": 0.17005196032120926,
+      "grad_norm": 13.693441774251875,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits": -1.512542724609375,
+      "logps": -83.833984375,
+      "loss": 0.6812,
+      "objective": 0.6835373044013977,
+      "ranking_idealized": 0.6041666865348816,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 0.6835373044013977,
+      "step": 30
+    },
+    {
+      "dpo_loss": 0.6732779145240784,
+      "dpo_wo_beta": -0.7419623136520386,
+      "epoch": 0.19839395370807747,
+      "grad_norm": 13.409062026931796,
+      "learning_rate": 4.861111111111111e-07,
+      "logits": -1.4930462837219238,
+      "logps": -82.56928253173828,
+      "loss": 0.672,
+      "objective": 0.6732779145240784,
+      "ranking_idealized": 0.612500011920929,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.6732779145240784,
+      "step": 35
+    },
+    {
+      "epoch": 0.20406235238545112,
+      "eval_dpo_loss": 0.6925506591796875,
+      "eval_dpo_wo_beta": -0.9551481008529663,
+      "eval_logits": -1.5338705778121948,
+      "eval_logps": -91.41157531738281,
+      "eval_loss": 0.6866393685340881,
+      "eval_objective": 0.6925506591796875,
+      "eval_ranking_idealized": 0.5888429880142212,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5051652789115906,
+      "eval_regularize": 0.6925506591796875,
+      "eval_runtime": 211.8538,
+      "eval_samples_per_second": 27.33,
+      "eval_steps_per_second": 1.142,
+      "step": 36
+    },
+    {
+      "dpo_loss": 0.6632875800132751,
+      "dpo_wo_beta": -0.7245739102363586,
+      "epoch": 0.22673594709494568,
+      "grad_norm": 15.824803800139307,
+      "learning_rate": 4.998023493068254e-07,
+      "logits": -1.4679725170135498,
+      "logps": -83.45999908447266,
+      "loss": 0.6728,
+      "objective": 0.6632875800132751,
+      "ranking_idealized": 0.6000000238418579,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.6632875800132751,
+      "step": 40
+    },
+    {
+      "dpo_loss": 0.6583938002586365,
+      "dpo_wo_beta": -0.7636463642120361,
+      "epoch": 0.25507794048181387,
+      "grad_norm": 13.750505573599494,
+      "learning_rate": 4.989999289644991e-07,
+      "logits": -1.594675898551941,
+      "logps": -84.99964141845703,
+      "loss": 0.6655,
+      "objective": 0.6583938002586365,
+      "ranking_idealized": 0.6208333373069763,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.6583938002586365,
+      "step": 45
+    },
+    {
+      "dpo_loss": 0.6567264795303345,
+      "dpo_wo_beta": -0.863508403301239,
+      "epoch": 0.2834199338686821,
+      "grad_norm": 13.413740152325811,
+      "learning_rate": 4.975823666181255e-07,
+      "logits": -1.5660439729690552,
+      "logps": -82.82308959960938,
+      "loss": 0.6644,
+      "objective": 0.6567264795303345,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.42500001192092896,
+      "ranking_simple": 0.4749999940395355,
+      "regularize": 0.6567264795303345,
+      "step": 50
+    },
+    {
+      "dpo_loss": 0.6495455503463745,
+      "dpo_wo_beta": -0.7752765417098999,
+      "epoch": 0.3117619272555503,
+      "grad_norm": 14.577952034074448,
+      "learning_rate": 4.955531642853403e-07,
+      "logits": -1.5825331211090088,
+      "logps": -84.16141510009766,
+      "loss": 0.653,
+      "objective": 0.6495455503463745,
+      "ranking_idealized": 0.625,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.550000011920929,
+      "regularize": 0.6495455503463745,
+      "step": 55
+    },
+    {
+      "dpo_loss": 0.6498515605926514,
+      "dpo_wo_beta": -0.736571729183197,
+      "epoch": 0.3401039206424185,
+      "grad_norm": 13.788809377020446,
+      "learning_rate": 4.929173350101024e-07,
+      "logits": -1.5849006175994873,
+      "logps": -83.91230773925781,
+      "loss": 0.6514,
+      "objective": 0.6498515605926514,
+      "ranking_idealized": 0.6333333253860474,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.550000011920929,
+      "regularize": 0.6498515605926514,
+      "step": 60
+    },
+    {
+      "dpo_loss": 0.6450857520103455,
+      "dpo_wo_beta": -0.8086111545562744,
+      "epoch": 0.3684459140292867,
+      "grad_norm": 13.799818527893999,
+      "learning_rate": 4.896813904782162e-07,
+      "logits": -1.5256472826004028,
+      "logps": -84.62181091308594,
+      "loss": 0.6495,
+      "objective": 0.6450857520103455,
+      "ranking_idealized": 0.5708333253860474,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 0.6450857520103455,
+      "step": 65
+    },
+    {
+      "dpo_loss": 0.644597053527832,
+      "dpo_wo_beta": -0.891067385673523,
+      "epoch": 0.39678790741615494,
+      "grad_norm": 13.789576838493705,
+      "learning_rate": 4.858533249305336e-07,
+      "logits": -1.6211374998092651,
+      "logps": -86.41259002685547,
+      "loss": 0.6533,
+      "objective": 0.644597053527832,
+      "ranking_idealized": 0.6000000238418579,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.644597053527832,
+      "step": 70
+    },
+    {
+      "epoch": 0.40812470477090224,
+      "eval_dpo_loss": 0.6884610056877136,
+      "eval_dpo_wo_beta": -1.1473097801208496,
+      "eval_logits": -1.6310690641403198,
+      "eval_logps": -92.47576904296875,
+      "eval_loss": 0.6768966913223267,
+      "eval_objective": 0.6884610056877136,
+      "eval_ranking_idealized": 0.5888429880142212,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5175619721412659,
+      "eval_regularize": 0.6884610056877136,
+      "eval_runtime": 210.9564,
+      "eval_samples_per_second": 27.446,
+      "eval_steps_per_second": 1.147,
+      "step": 72
+    },
+    {
+      "dpo_loss": 0.6458336114883423,
+      "dpo_wo_beta": -0.847459614276886,
+      "epoch": 0.42512990080302315,
+      "grad_norm": 13.350435766130868,
+      "learning_rate": 4.814425954135785e-07,
+      "logits": -1.6625837087631226,
+      "logps": -86.4749755859375,
+      "loss": 0.636,
+      "objective": 0.6458336114883423,
+      "ranking_idealized": 0.625,
+      "ranking_idealized_expo": 0.5458333492279053,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 0.6458336114883423,
+      "step": 75
+    },
+    {
+      "dpo_loss": 0.6464484930038452,
+      "dpo_wo_beta": -0.9348036050796509,
+      "epoch": 0.45347189418989137,
+      "grad_norm": 13.762513071445667,
+      "learning_rate": 4.764600984163808e-07,
+      "logits": -1.7147225141525269,
+      "logps": -86.26012420654297,
+      "loss": 0.6326,
+      "objective": 0.6464484930038452,
+      "ranking_idealized": 0.5833333134651184,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.6464484930038452,
+      "step": 80
+    },
+    {
+      "dpo_loss": 0.6432295441627502,
+      "dpo_wo_beta": -1.0359190702438354,
+      "epoch": 0.4818138875767596,
+      "grad_norm": 13.87750405485222,
+      "learning_rate": 4.70918142951239e-07,
+      "logits": -1.6870180368423462,
+      "logps": -85.32923889160156,
+      "loss": 0.6336,
+      "objective": 0.6432295441627502,
+      "ranking_idealized": 0.6000000238418579,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.6432295441627502,
+      "step": 85
+    },
+    {
+      "dpo_loss": 0.6311187744140625,
+      "dpo_wo_beta": -0.9493275284767151,
+      "epoch": 0.5101558809636277,
+      "grad_norm": 14.792020412361724,
+      "learning_rate": 4.6483042014491527e-07,
+      "logits": -1.5992980003356934,
+      "logps": -88.2070083618164,
+      "loss": 0.6216,
+      "objective": 0.6311187744140625,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.4749999940395355,
+      "ranking_simple": 0.5041666626930237,
+      "regularize": 0.6311187744140625,
+      "step": 90
+    },
+    {
+      "dpo_loss": 0.595143735408783,
+      "dpo_wo_beta": -0.8239250183105469,
+      "epoch": 0.538497874350496,
+      "grad_norm": 16.242182565245834,
+      "learning_rate": 4.5821196941538334e-07,
+      "logits": -1.6257656812667847,
+      "logps": -87.32308197021484,
+      "loss": 0.6157,
+      "objective": 0.595143735408783,
+      "ranking_idealized": 0.6291666626930237,
+      "ranking_idealized_expo": 0.5458333492279053,
+      "ranking_simple": 0.6041666865348816,
+      "regularize": 0.595143735408783,
+      "step": 95
+    },
+    {
+      "dpo_loss": 0.6018479466438293,
+      "dpo_wo_beta": -1.0667775869369507,
+      "epoch": 0.5668398677373642,
+      "grad_norm": 16.965716669559736,
+      "learning_rate": 4.510791413176912e-07,
+      "logits": -1.6872950792312622,
+      "logps": -88.73619842529297,
+      "loss": 0.623,
+      "objective": 0.6018479466438293,
+      "ranking_idealized": 0.612500011920929,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5625,
+      "regularize": 0.6018479466438293,
+      "step": 100
+    },
+    {
+      "dpo_loss": 0.5836849808692932,
+      "dpo_wo_beta": -0.7914173007011414,
+      "epoch": 0.5951818611242324,
+      "grad_norm": 14.695711016472089,
+      "learning_rate": 4.434495571507234e-07,
+      "logits": -1.606242299079895,
+      "logps": -87.9868392944336,
+      "loss": 0.604,
+      "objective": 0.5836849808692932,
+      "ranking_idealized": 0.625,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 0.5836849808692932,
+      "step": 105
+    },
+    {
+      "epoch": 0.6121870571563534,
+      "eval_dpo_loss": 0.6853068470954895,
+      "eval_dpo_wo_beta": -1.3960636854171753,
+      "eval_logits": -1.769094467163086,
+      "eval_logps": -94.05838012695312,
+      "eval_loss": 0.677278995513916,
+      "eval_objective": 0.6853068470954895,
+      "eval_ranking_idealized": 0.5888429880142212,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5185950398445129,
+      "eval_regularize": 0.6853068470954895,
+      "eval_runtime": 210.425,
+      "eval_samples_per_second": 27.516,
+      "eval_steps_per_second": 1.15,
+      "step": 108
+    },
+    {
+      "dpo_loss": 0.6007120013237,
+      "dpo_wo_beta": -0.8422455191612244,
+      "epoch": 0.6235238545111006,
+      "grad_norm": 15.750957422421227,
+      "learning_rate": 4.353420654246546e-07,
+      "logits": -1.7943415641784668,
+      "logps": -86.3438491821289,
+      "loss": 0.6054,
+      "objective": 0.6007120013237,
+      "ranking_idealized": 0.5708333253860474,
+      "ranking_idealized_expo": 0.4541666805744171,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 0.6007120013237,
+      "step": 110
+    },
+    {
+      "dpo_loss": 0.6109593510627747,
+      "dpo_wo_beta": -1.0094329118728638,
+      "epoch": 0.6518658478979689,
+      "grad_norm": 16.233880140590145,
+      "learning_rate": 4.2677669529663686e-07,
+      "logits": -1.7633204460144043,
+      "logps": -86.51942443847656,
+      "loss": 0.6046,
+      "objective": 0.6109593510627747,
+      "ranking_idealized": 0.574999988079071,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 0.6109593510627747,
+      "step": 115
+    },
+    {
+      "dpo_loss": 0.5903018712997437,
+      "dpo_wo_beta": -0.9045132994651794,
+      "epoch": 0.680207841284837,
+      "grad_norm": 17.133679937634657,
+      "learning_rate": 4.177746070897592e-07,
+      "logits": -1.7159924507141113,
+      "logps": -87.81356811523438,
+      "loss": 0.5999,
+      "objective": 0.5903018712997437,
+      "ranking_idealized": 0.5874999761581421,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5791666507720947,
+      "regularize": 0.5903018712997437,
+      "step": 120
+    },
+    {
+      "dpo_loss": 0.6066262722015381,
+      "dpo_wo_beta": -1.2728170156478882,
+      "epoch": 0.7085498346717053,
+      "grad_norm": 16.15790530794251,
+      "learning_rate": 4.083580400175153e-07,
+      "logits": -1.698543667793274,
+      "logps": -88.70279693603516,
+      "loss": 0.6139,
+      "objective": 0.6066262722015381,
+      "ranking_idealized": 0.625,
+      "ranking_idealized_expo": 0.550000011920929,
+      "ranking_simple": 0.5874999761581421,
+      "regularize": 0.6066262722015381,
+      "step": 125
+    },
+    {
+      "dpo_loss": 0.6065606474876404,
+      "dpo_wo_beta": -1.1573857069015503,
+      "epoch": 0.7368918280585735,
+      "grad_norm": 16.556049529132082,
+      "learning_rate": 3.9855025724292763e-07,
+      "logits": -1.8358159065246582,
+      "logps": -87.11023712158203,
+      "loss": 0.6035,
+      "objective": 0.6065606474876404,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.44999998807907104,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 0.6065606474876404,
+      "step": 130
+    },
+    {
+      "dpo_loss": 0.6158966422080994,
+      "dpo_wo_beta": -1.1439249515533447,
+      "epoch": 0.7652338214454416,
+      "grad_norm": 14.197353075079507,
+      "learning_rate": 3.883754884080539e-07,
+      "logits": -1.7928814888000488,
+      "logps": -89.14883422851562,
+      "loss": 0.594,
+      "objective": 0.6158966422080994,
+      "ranking_idealized": 0.5874999761581421,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.6158966422080994,
+      "step": 135
+    },
+    {
+      "dpo_loss": 0.61494380235672,
+      "dpo_wo_beta": -1.0414694547653198,
+      "epoch": 0.7935758148323099,
+      "grad_norm": 15.670852477907466,
+      "learning_rate": 3.7785886977585555e-07,
+      "logits": -1.711374282836914,
+      "logps": -88.56974792480469,
+      "loss": 0.5911,
+      "objective": 0.61494380235672,
+      "ranking_idealized": 0.5791666507720947,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 0.61494380235672,
+      "step": 140
+    },
+    {
+      "epoch": 0.8162494095418045,
+      "eval_dpo_loss": 0.6885914206504822,
+      "eval_dpo_wo_beta": -1.5153368711471558,
+      "eval_logits": -1.816495418548584,
+      "eval_logps": -95.75775146484375,
+      "eval_loss": 0.6774819493293762,
+      "eval_objective": 0.6885914206504822,
+      "eval_ranking_idealized": 0.5888429880142212,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5185950398445129,
+      "eval_regularize": 0.6885914206504822,
+      "eval_runtime": 210.8957,
+      "eval_samples_per_second": 27.454,
+      "eval_steps_per_second": 1.147,
+      "step": 144
+    },
+    {
+      "epoch": 0.8162494095418045,
+      "step": 144,
+      "total_flos": 0.0,
+      "train_loss": 0.6411769655015733,
+      "train_runtime": 5149.1432,
+      "train_samples_per_second": 19.732,
+      "train_steps_per_second": 0.068
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 352,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 36,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 3,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:946c5e179ba4793191301d512428a653dfc84a775278f6495b2ab70c8fbf457b
+size 8120

vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff