Model save

README.md

@@ -13,19 +13,10 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/sanqiang/wdpo/runs/i9fm0xk3)
 # zephyr-7b-dpo-full
 
 This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.0513
-- Rewards/chosen: -2.3210
-- Rewards/rejected: -3.0411
-- Rewards/accuracies: 0.7148
-- Rewards/margins: 0.7201
-- Logps/rejected: -561.4609
-- Logps/chosen: -489.1372
-- Logits/rejected: -2.1024
-- Logits/chosen: -2.1271
 
 ## Model description
 
@@ -47,11 +38,11 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 2
+- seed: 42
 - distributed_type: multi-GPU
 - num_devices: 8
-- gradient_accumulation_steps: 4
-- total_train_batch_size: 256
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 128
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
@@ -60,17 +51,11 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0816        | 0.23  | 100  | 0.0900          | -1.3317        | -1.7519          | 0.6602             | 0.4202          | -432.5398      | -390.2077    | -2.3227         | -2.3412       |
-| 0.0436        | 0.45  | 200  | 0.0598          | -2.0588        | -2.6924          | 0.6914             | 0.6335          | -526.5885      | -462.9209    | -2.1205         | -2.1455       |
-| 0.0388        | 0.68  | 300  | 0.0515          | -2.3897        | -3.1201          | 0.6914             | 0.7304          | -569.3627      | -496.0135    | -2.1019         | -2.1254       |
-| 0.0379        | 0.91  | 400  | 0.0513          | -2.3210        | -3.0411          | 0.7148             | 0.7201          | -561.4609      | -489.1372    | -2.1024         | -2.1271       |
 
 
 ### Framework versions
 
-- Transformers 4.35.2
+- Transformers 4.41.0.dev0
 - Pytorch 2.1.2+cu121
 - Datasets 2.14.6
-- Tokenizers 0.14.1
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.07976685402716369,
-    "train_runtime": 7402.7266,
-    "train_samples": 113028,
-    "train_samples_per_second": 15.268,
-    "train_steps_per_second": 0.06
+    "total_flos": 0.0,
+    "train_loss": 0.20092295110225677,
+    "train_runtime": 384.8987,
+    "train_samples": 6750,
+    "train_samples_per_second": 17.537,
+    "train_steps_per_second": 0.138
 }

config.json

@@ -3,6 +3,7 @@
   "architectures": [
     "MistralForCausalLM"
   ],
+  "attention_dropout": 0.0,
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
@@ -19,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.35.2",
+  "transformers_version": "4.41.0.dev0",
   "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.35.2"
+  "transformers_version": "4.41.0.dev0"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e59e1d706e776f4bb6112dc215f13d6a43b67453b5a859c61bce0f8762f42eeb
+oid sha256:a46cc3b5f84d4d99bb5b73eed876f0b5578ddd4737df15a4e469e33c95c08170
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2b30f9389d19c5e0d1de28a31da254c3562caa7388541f928b2398860cd4ef6
+oid sha256:cde9eb125e0d5a3b3305ef0cdb4bea397e66b0d0622e3a57126bb6a5687634fa
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a7668ab91df9a474ad277cfe790a8d47594470c8cb376e6652299336892c440
+oid sha256:3c0430eff547cb6ba5d3819817467987ed432cfa8a3a5fcd748e54a73cb5ddc4
 size 4540516344

tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -34,7 +36,6 @@
   "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": true,
   "model_max_length": 2048,
   "pad_token": "</s>",
   "sp_model_kwargs": {},

train_results.json

@@ -1,8 +1,9 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.07976685402716369,
-    "train_runtime": 7402.7266,
-    "train_samples": 113028,
-    "train_samples_per_second": 15.268,
-    "train_steps_per_second": 0.06
+    "total_flos": 0.0,
+    "train_loss": 0.20092295110225677,
+    "train_runtime": 384.8987,
+    "train_samples": 6750,
+    "train_samples_per_second": 17.537,
+    "train_steps_per_second": 0.138
 }

trainer_state.json

@@ -1,21 +1,25 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.99830220713073,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 441,
+  "global_step": 53,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "learning_rate": 1.111111111111111e-08,
-      "logits/chosen": -2.827775478363037,
-      "logits/rejected": -2.806304931640625,
-      "logps/chosen": -362.28790283203125,
-      "logps/rejected": -205.56106567382812,
-      "loss": 0.2753,
+      "debug/losses": 0.17192834615707397,
+      "debug/policy_weights": 0.24804016947746277,
+      "debug/raw_losses": 0.6931471824645996,
+      "epoch": 0.018867924528301886,
+      "grad_norm": 3.0791833143219045,
+      "learning_rate": 8.333333333333333e-08,
+      "logits/chosen": -2.855412006378174,
+      "logits/rejected": -2.8797199726104736,
+      "logps/chosen": -320.43853759765625,
+      "logps/rejected": -340.07073974609375,
+      "loss": 0.2116,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -23,700 +27,124 @@
       "step": 1
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 1.111111111111111e-07,
-      "logits/chosen": -2.802647829055786,
-      "logits/rejected": -2.7721259593963623,
-      "logps/chosen": -319.9109191894531,
-      "logps/rejected": -176.29066467285156,
-      "loss": 0.2791,
-      "rewards/accuracies": 0.4340277910232544,
-      "rewards/chosen": 0.00017140436102636158,
-      "rewards/margins": -0.00022869586246088147,
-      "rewards/rejected": 0.00040010022348724306,
+      "debug/losses": 0.20367620885372162,
+      "debug/policy_weights": 0.2948996424674988,
+      "debug/raw_losses": 0.6906173229217529,
+      "epoch": 0.18867924528301888,
+      "grad_norm": 3.2025034006962603,
+      "learning_rate": 4.911172937635942e-07,
+      "logits/chosen": -2.8658909797668457,
+      "logits/rejected": -2.8917548656463623,
+      "logps/chosen": -305.7406005859375,
+      "logps/rejected": -332.2490234375,
+      "loss": 0.2033,
+      "rewards/accuracies": 0.4791666567325592,
+      "rewards/chosen": 0.0008342999499291182,
+      "rewards/margins": 0.005153011996299028,
+      "rewards/rejected": -0.004318712279200554,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 2.222222222222222e-07,
-      "logits/chosen": -2.779694080352783,
-      "logits/rejected": -2.74702787399292,
-      "logps/chosen": -311.74652099609375,
-      "logps/rejected": -177.87513732910156,
-      "loss": 0.276,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.008742044679820538,
-      "rewards/margins": 0.014466273598372936,
-      "rewards/rejected": -0.005724229384213686,
+      "debug/losses": 0.20357565581798553,
+      "debug/policy_weights": 0.30120497941970825,
+      "debug/raw_losses": 0.675395131111145,
+      "epoch": 0.37735849056603776,
+      "grad_norm": 3.097721440067098,
+      "learning_rate": 3.982949361823388e-07,
+      "logits/chosen": -2.855691432952881,
+      "logits/rejected": -2.8575425148010254,
+      "logps/chosen": -324.3226623535156,
+      "logps/rejected": -331.01007080078125,
+      "loss": 0.2068,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0029448498971760273,
+      "rewards/margins": 0.03993762657046318,
+      "rewards/rejected": -0.03699277713894844,
       "step": 20
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 3.333333333333333e-07,
-      "logits/chosen": -2.742488384246826,
-      "logits/rejected": -2.722848415374756,
-      "logps/chosen": -313.6725769042969,
-      "logps/rejected": -186.76075744628906,
-      "loss": 0.2771,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": 0.020179977640509605,
-      "rewards/margins": 0.06532613933086395,
-      "rewards/rejected": -0.0451461598277092,
+      "debug/losses": 0.20828036963939667,
+      "debug/policy_weights": 0.3203299641609192,
+      "debug/raw_losses": 0.6491612195968628,
+      "epoch": 0.5660377358490566,
+      "grad_norm": 3.2758930059808353,
+      "learning_rate": 2.416462557480814e-07,
+      "logits/chosen": -2.839444398880005,
+      "logits/rejected": -2.842738628387451,
+      "logps/chosen": -298.2297668457031,
+      "logps/rejected": -313.391845703125,
+      "loss": 0.2007,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.014156119897961617,
+      "rewards/margins": 0.11547034978866577,
+      "rewards/rejected": -0.12962646782398224,
       "step": 30
     },
     {
-      "epoch": 0.09,
-      "learning_rate": 4.444444444444444e-07,
-      "logits/chosen": -2.594104528427124,
-      "logits/rejected": -2.5780692100524902,
-      "logps/chosen": -353.83160400390625,
-      "logps/rejected": -217.767578125,
-      "loss": 0.2573,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": -0.0030881466809660196,
-      "rewards/margins": 0.2199195921421051,
-      "rewards/rejected": -0.22300772368907928,
+      "debug/losses": 0.18270191550254822,
+      "debug/policy_weights": 0.30252760648727417,
+      "debug/raw_losses": 0.584720253944397,
+      "epoch": 0.7547169811320755,
+      "grad_norm": 2.7163808332522805,
+      "learning_rate": 8.859303711029939e-08,
+      "logits/chosen": -2.842470645904541,
+      "logits/rejected": -2.8394596576690674,
+      "logps/chosen": -290.64208984375,
+      "logps/rejected": -320.837890625,
+      "loss": 0.1963,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.011087085120379925,
+      "rewards/margins": 0.30588188767433167,
+      "rewards/rejected": -0.31696897745132446,
       "step": 40
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 4.998033461515242e-07,
-      "logits/chosen": -2.5498433113098145,
-      "logits/rejected": -2.5390689373016357,
-      "logps/chosen": -369.34344482421875,
-      "logps/rejected": -245.7183380126953,
-      "loss": 0.2109,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.1884610652923584,
-      "rewards/margins": 0.33448106050491333,
-      "rewards/rejected": -0.5229421257972717,
+      "debug/losses": 0.23133957386016846,
+      "debug/policy_weights": 0.3536807894706726,
+      "debug/raw_losses": 0.6469973921775818,
+      "epoch": 0.9433962264150944,
+      "grad_norm": 3.004599614335292,
+      "learning_rate": 5.009573740853313e-09,
+      "logits/chosen": -2.8772711753845215,
+      "logits/rejected": -2.864253520965576,
+      "logps/chosen": -282.04022216796875,
+      "logps/rejected": -316.0839538574219,
+      "loss": 0.1939,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.027098428457975388,
+      "rewards/margins": 0.1554117500782013,
+      "rewards/rejected": -0.1825101673603058,
       "step": 50
     },
-    {
-      "epoch": 0.14,
-      "learning_rate": 4.982319711683221e-07,
-      "logits/chosen": -2.5170645713806152,
-      "logits/rejected": -2.4964470863342285,
-      "logps/chosen": -390.71759033203125,
-      "logps/rejected": -284.5617370605469,
-      "loss": 0.1661,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.3339163661003113,
-      "rewards/margins": 0.5272020101547241,
-      "rewards/rejected": -0.8611184358596802,
-      "step": 60
-    },
-    {
-      "epoch": 0.16,
-      "learning_rate": 4.950991058546892e-07,
-      "logits/chosen": -2.5328118801116943,
-      "logits/rejected": -2.4936556816101074,
-      "logps/chosen": -401.39605712890625,
-      "logps/rejected": -309.35333251953125,
-      "loss": 0.1375,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.46392518281936646,
-      "rewards/margins": 0.6156176924705505,
-      "rewards/rejected": -1.079542875289917,
-      "step": 70
-    },
-    {
-      "epoch": 0.18,
-      "learning_rate": 4.904244573372733e-07,
-      "logits/chosen": -2.540886402130127,
-      "logits/rejected": -2.495255708694458,
-      "logps/chosen": -454.4031677246094,
-      "logps/rejected": -335.4547119140625,
-      "loss": 0.1093,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.48765724897384644,
-      "rewards/margins": 0.8079099655151367,
-      "rewards/rejected": -1.2955673933029175,
-      "step": 80
-    },
-    {
-      "epoch": 0.2,
-      "learning_rate": 4.842374312499405e-07,
-      "logits/chosen": -2.4646222591400146,
-      "logits/rejected": -2.417442560195923,
-      "logps/chosen": -409.55828857421875,
-      "logps/rejected": -335.1234130859375,
-      "loss": 0.0864,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": -0.7838326692581177,
-      "rewards/margins": 0.7940716743469238,
-      "rewards/rejected": -1.5779043436050415,
-      "step": 90
-    },
-    {
-      "epoch": 0.23,
-      "learning_rate": 4.7657694675916247e-07,
-      "logits/chosen": -2.3520429134368896,
-      "logits/rejected": -2.3132762908935547,
-      "logps/chosen": -412.2884826660156,
-      "logps/rejected": -369.1390075683594,
-      "loss": 0.0816,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.9074515104293823,
-      "rewards/margins": 0.9162980318069458,
-      "rewards/rejected": -1.8237495422363281,
-      "step": 100
-    },
-    {
-      "epoch": 0.23,
-      "eval_logits/chosen": -2.341198444366455,
-      "eval_logits/rejected": -2.322673797607422,
-      "eval_logps/chosen": -390.20770263671875,
-      "eval_logps/rejected": -432.539794921875,
-      "eval_loss": 0.09001600742340088,
-      "eval_rewards/accuracies": 0.66015625,
-      "eval_rewards/chosen": -1.3316810131072998,
-      "eval_rewards/margins": 0.4201836585998535,
-      "eval_rewards/rejected": -1.7518646717071533,
-      "eval_runtime": 53.5036,
-      "eval_samples_per_second": 37.381,
-      "eval_steps_per_second": 0.598,
-      "step": 100
-    },
-    {
-      "epoch": 0.25,
-      "learning_rate": 4.6749119174501973e-07,
-      "logits/chosen": -2.2996747493743896,
-      "logits/rejected": -2.242867946624756,
-      "logps/chosen": -462.94598388671875,
-      "logps/rejected": -403.91363525390625,
-      "loss": 0.0729,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.8766400218009949,
-      "rewards/margins": 1.1215176582336426,
-      "rewards/rejected": -1.9981575012207031,
-      "step": 110
-    },
-    {
-      "epoch": 0.27,
-      "learning_rate": 4.5703731967784265e-07,
-      "logits/chosen": -2.2242417335510254,
-      "logits/rejected": -2.181910276412964,
-      "logps/chosen": -449.12713623046875,
-      "logps/rejected": -427.40740966796875,
-      "loss": 0.0685,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.2070833444595337,
-      "rewards/margins": 0.9871729612350464,
-      "rewards/rejected": -2.19425630569458,
-      "step": 120
-    },
-    {
-      "epoch": 0.29,
-      "learning_rate": 4.4528109009727333e-07,
-      "logits/chosen": -2.1515939235687256,
-      "logits/rejected": -2.0918548107147217,
-      "logps/chosen": -467.0848693847656,
-      "logps/rejected": -428.3409118652344,
-      "loss": 0.0531,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": -1.561851978302002,
-      "rewards/margins": 0.9053736925125122,
-      "rewards/rejected": -2.4672255516052246,
-      "step": 130
-    },
-    {
-      "epoch": 0.32,
-      "learning_rate": 4.3229645495529427e-07,
-      "logits/chosen": -2.2256131172180176,
-      "logits/rejected": -2.1506736278533936,
-      "logps/chosen": -481.4261169433594,
-      "logps/rejected": -418.9735412597656,
-      "loss": 0.0496,
-      "rewards/accuracies": 0.7281249761581421,
-      "rewards/chosen": -1.3596832752227783,
-      "rewards/margins": 1.1094987392425537,
-      "rewards/rejected": -2.469182014465332,
-      "step": 140
-    },
-    {
-      "epoch": 0.34,
-      "learning_rate": 4.1816509342531317e-07,
-      "logits/chosen": -2.2947065830230713,
-      "logits/rejected": -2.2372803688049316,
-      "logps/chosen": -467.123291015625,
-      "logps/rejected": -395.8599548339844,
-      "loss": 0.0605,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2707955837249756,
-      "rewards/margins": 0.942121148109436,
-      "rewards/rejected": -2.212916851043701,
-      "step": 150
-    },
-    {
-      "epoch": 0.36,
-      "learning_rate": 4.0297589810356166e-07,
-      "logits/chosen": -2.233494281768799,
-      "logits/rejected": -2.1654052734375,
-      "logps/chosen": -485.3228454589844,
-      "logps/rejected": -422.75555419921875,
-      "loss": 0.0551,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": -1.3737380504608154,
-      "rewards/margins": 1.092675805091858,
-      "rewards/rejected": -2.466413974761963,
-      "step": 160
-    },
-    {
-      "epoch": 0.38,
-      "learning_rate": 3.868244158348331e-07,
-      "logits/chosen": -2.2172982692718506,
-      "logits/rejected": -2.1708760261535645,
-      "logps/chosen": -494.2137145996094,
-      "logps/rejected": -449.77587890625,
-      "loss": 0.0577,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -1.3581817150115967,
-      "rewards/margins": 1.175022840499878,
-      "rewards/rejected": -2.5332045555114746,
-      "step": 170
-    },
-    {
-      "epoch": 0.41,
-      "learning_rate": 3.698122466800142e-07,
-      "logits/chosen": -2.196737766265869,
-      "logits/rejected": -2.150078296661377,
-      "logps/chosen": -453.3573303222656,
-      "logps/rejected": -426.41143798828125,
-      "loss": 0.0658,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": -1.144860029220581,
-      "rewards/margins": 1.1531283855438232,
-      "rewards/rejected": -2.2979884147644043,
-      "step": 180
-    },
-    {
-      "epoch": 0.43,
-      "learning_rate": 3.5204640480617574e-07,
-      "logits/chosen": -2.1627323627471924,
-      "logits/rejected": -2.110278367996216,
-      "logps/chosen": -524.0073852539062,
-      "logps/rejected": -460.2293395996094,
-      "loss": 0.0492,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.415473222732544,
-      "rewards/margins": 1.2913919687271118,
-      "rewards/rejected": -2.7068653106689453,
-      "step": 190
-    },
-    {
-      "epoch": 0.45,
-      "learning_rate": 3.336386453195088e-07,
-      "logits/chosen": -2.1464579105377197,
-      "logits/rejected": -2.082240581512451,
-      "logps/chosen": -505.35919189453125,
-      "logps/rejected": -458.5272521972656,
-      "loss": 0.0436,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.7200065851211548,
-      "rewards/margins": 1.1776483058929443,
-      "rewards/rejected": -2.8976550102233887,
-      "step": 200
-    },
-    {
-      "epoch": 0.45,
-      "eval_logits/chosen": -2.1454780101776123,
-      "eval_logits/rejected": -2.1204702854156494,
-      "eval_logps/chosen": -462.9208679199219,
-      "eval_logps/rejected": -526.5885009765625,
-      "eval_loss": 0.059836775064468384,
-      "eval_rewards/accuracies": 0.69140625,
-      "eval_rewards/chosen": -2.058812379837036,
-      "eval_rewards/margins": 0.6335402131080627,
-      "eval_rewards/rejected": -2.692352533340454,
-      "eval_runtime": 53.3575,
-      "eval_samples_per_second": 37.483,
-      "eval_steps_per_second": 0.6,
-      "step": 200
-    },
-    {
-      "epoch": 0.48,
-      "learning_rate": 3.147047612756302e-07,
-      "logits/chosen": -2.164459705352783,
-      "logits/rejected": -2.11089825630188,
-      "logps/chosen": -475.6734924316406,
-      "logps/rejected": -459.15673828125,
-      "loss": 0.0505,
-      "rewards/accuracies": 0.721875011920929,
-      "rewards/chosen": -1.560291051864624,
-      "rewards/margins": 1.194120168685913,
-      "rewards/rejected": -2.754411220550537,
-      "step": 210
-    },
-    {
-      "epoch": 0.5,
-      "learning_rate": 2.9536385528937565e-07,
-      "logits/chosen": -2.1479196548461914,
-      "logits/rejected": -2.0868375301361084,
-      "logps/chosen": -458.2518005371094,
-      "logps/rejected": -436.82305908203125,
-      "loss": 0.0575,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.339440941810608,
-      "rewards/margins": 1.1341434717178345,
-      "rewards/rejected": -2.4735846519470215,
-      "step": 220
-    },
-    {
-      "epoch": 0.52,
-      "learning_rate": 2.7573759032598365e-07,
-      "logits/chosen": -2.1152689456939697,
-      "logits/rejected": -2.074328899383545,
-      "logps/chosen": -492.90380859375,
-      "logps/rejected": -454.63873291015625,
-      "loss": 0.0565,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.4534698724746704,
-      "rewards/margins": 1.1257164478302002,
-      "rewards/rejected": -2.579186201095581,
-      "step": 230
-    },
-    {
-      "epoch": 0.54,
-      "learning_rate": 2.5594942438652685e-07,
-      "logits/chosen": -2.1014912128448486,
-      "logits/rejected": -2.049712657928467,
-      "logps/chosen": -508.5966796875,
-      "logps/rejected": -468.09991455078125,
-      "loss": 0.0471,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.5375745296478271,
-      "rewards/margins": 1.1159954071044922,
-      "rewards/rejected": -2.6535699367523193,
-      "step": 240
-    },
-    {
-      "epoch": 0.57,
-      "learning_rate": 2.36123833901765e-07,
-      "logits/chosen": -2.1314926147460938,
-      "logits/rejected": -2.077198028564453,
-      "logps/chosen": -519.6549072265625,
-      "logps/rejected": -478.80889892578125,
-      "loss": 0.0445,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": -1.766661286354065,
-      "rewards/margins": 1.2140429019927979,
-      "rewards/rejected": -2.9807040691375732,
-      "step": 250
-    },
-    {
-      "epoch": 0.59,
-      "learning_rate": 2.1638553071961704e-07,
-      "logits/chosen": -2.159257650375366,
-      "logits/rejected": -2.0843465328216553,
-      "logps/chosen": -498.5057678222656,
-      "logps/rejected": -481.7154235839844,
-      "loss": 0.0462,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.548032522201538,
-      "rewards/margins": 1.3328226804733276,
-      "rewards/rejected": -2.8808555603027344,
-      "step": 260
-    },
-    {
-      "epoch": 0.61,
-      "learning_rate": 1.968586776117558e-07,
-      "logits/chosen": -2.1956114768981934,
-      "logits/rejected": -2.1366395950317383,
-      "logps/chosen": -484.2972717285156,
-      "logps/rejected": -429.1397399902344,
-      "loss": 0.0537,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.4318220615386963,
-      "rewards/margins": 1.0592107772827148,
-      "rewards/rejected": -2.491032838821411,
-      "step": 270
-    },
-    {
-      "epoch": 0.63,
-      "learning_rate": 1.7766610723413684e-07,
-      "logits/chosen": -2.1730809211730957,
-      "logits/rejected": -2.1212735176086426,
-      "logps/chosen": -490.05908203125,
-      "logps/rejected": -451.37628173828125,
-      "loss": 0.0562,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -1.4677350521087646,
-      "rewards/margins": 1.121753215789795,
-      "rewards/rejected": -2.5894885063171387,
-      "step": 280
-    },
-    {
-      "epoch": 0.66,
-      "learning_rate": 1.589285494545514e-07,
-      "logits/chosen": -2.1455771923065186,
-      "logits/rejected": -2.096673011779785,
-      "logps/chosen": -530.82421875,
-      "logps/rejected": -489.6752014160156,
-      "loss": 0.0405,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": -1.7482846975326538,
-      "rewards/margins": 1.2687255144119263,
-      "rewards/rejected": -3.01701021194458,
-      "step": 290
-    },
-    {
-      "epoch": 0.68,
-      "learning_rate": 1.4076387190766014e-07,
-      "logits/chosen": -2.1248068809509277,
-      "logits/rejected": -2.0664258003234863,
-      "logps/chosen": -537.5523681640625,
-      "logps/rejected": -512.0633544921875,
-      "loss": 0.0388,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": -2.0803182125091553,
-      "rewards/margins": 1.1989731788635254,
-      "rewards/rejected": -3.2792911529541016,
-      "step": 300
-    },
-    {
-      "epoch": 0.68,
-      "eval_logits/chosen": -2.125394821166992,
-      "eval_logits/rejected": -2.1019349098205566,
-      "eval_logps/chosen": -496.0135192871094,
-      "eval_logps/rejected": -569.3626708984375,
-      "eval_loss": 0.051513027399778366,
-      "eval_rewards/accuracies": 0.69140625,
-      "eval_rewards/chosen": -2.3897383213043213,
-      "eval_rewards/margins": 0.7303550243377686,
-      "eval_rewards/rejected": -3.120093822479248,
-      "eval_runtime": 53.3835,
-      "eval_samples_per_second": 37.465,
-      "eval_steps_per_second": 0.599,
-      "step": 300
-    },
-    {
-      "epoch": 0.7,
-      "learning_rate": 1.232863385547543e-07,
-      "logits/chosen": -2.138700246810913,
-      "logits/rejected": -2.0852296352386475,
-      "logps/chosen": -534.0220947265625,
-      "logps/rejected": -511.69708251953125,
-      "loss": 0.0391,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.956750512123108,
-      "rewards/margins": 1.3101288080215454,
-      "rewards/rejected": -3.266878843307495,
-      "step": 310
-    },
-    {
-      "epoch": 0.72,
-      "learning_rate": 1.0660589091223854e-07,
-      "logits/chosen": -2.1349263191223145,
-      "logits/rejected": -2.077669620513916,
-      "logps/chosen": -520.9324340820312,
-      "logps/rejected": -508.90594482421875,
-      "loss": 0.0403,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.970788598060608,
-      "rewards/margins": 1.172662615776062,
-      "rewards/rejected": -3.14345121383667,
-      "step": 320
-    },
-    {
-      "epoch": 0.75,
-      "learning_rate": 9.082745647022797e-08,
-      "logits/chosen": -2.1367201805114746,
-      "logits/rejected": -2.0910043716430664,
-      "logps/chosen": -503.3335876464844,
-      "logps/rejected": -482.1080627441406,
-      "loss": 0.0424,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -1.7588704824447632,
-      "rewards/margins": 1.2970627546310425,
-      "rewards/rejected": -3.0559334754943848,
-      "step": 330
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 7.605028865161809e-08,
-      "logits/chosen": -2.1519930362701416,
-      "logits/rejected": -2.074977397918701,
-      "logps/chosen": -539.51708984375,
-      "logps/rejected": -511.9608459472656,
-      "loss": 0.0443,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.8557418584823608,
-      "rewards/margins": 1.2462117671966553,
-      "rewards/rejected": -3.1019535064697266,
-      "step": 340
-    },
-    {
-      "epoch": 0.79,
-      "learning_rate": 6.236734246357947e-08,
-      "logits/chosen": -2.1703686714172363,
-      "logits/rejected": -2.1259913444519043,
-      "logps/chosen": -537.4775390625,
-      "logps/rejected": -482.252685546875,
-      "loss": 0.0441,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": -1.876011610031128,
-      "rewards/margins": 1.030928611755371,
-      "rewards/rejected": -2.906940221786499,
-      "step": 350
-    },
-    {
-      "epoch": 0.81,
-      "learning_rate": 4.986468976890992e-08,
-      "logits/chosen": -2.1170713901519775,
-      "logits/rejected": -2.0758965015411377,
-      "logps/chosen": -507.7197265625,
-      "logps/rejected": -487.06494140625,
-      "loss": 0.0391,
-      "rewards/accuracies": 0.753125011920929,
-      "rewards/chosen": -1.775130033493042,
-      "rewards/margins": 1.3335883617401123,
-      "rewards/rejected": -3.1087188720703125,
-      "step": 360
-    },
-    {
-      "epoch": 0.84,
-      "learning_rate": 3.8620977855448936e-08,
-      "logits/chosen": -2.1223392486572266,
-      "logits/rejected": -2.07041072845459,
-      "logps/chosen": -520.2659912109375,
-      "logps/rejected": -504.3395080566406,
-      "loss": 0.0399,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.8968217372894287,
-      "rewards/margins": 1.2073428630828857,
-      "rewards/rejected": -3.1041648387908936,
-      "step": 370
-    },
-    {
-      "epoch": 0.86,
-      "learning_rate": 2.8706934709395893e-08,
-      "logits/chosen": -2.122580051422119,
-      "logits/rejected": -2.0831775665283203,
-      "logps/chosen": -529.0001831054688,
-      "logps/rejected": -509.4918518066406,
-      "loss": 0.0402,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": -1.9584871530532837,
-      "rewards/margins": 1.2146377563476562,
-      "rewards/rejected": -3.1731247901916504,
-      "step": 380
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 2.0184924104583612e-08,
-      "logits/chosen": -2.136026620864868,
-      "logits/rejected": -2.085597515106201,
-      "logps/chosen": -521.5689697265625,
-      "logps/rejected": -499.8408203125,
-      "loss": 0.0378,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.883663535118103,
-      "rewards/margins": 1.2730777263641357,
-      "rewards/rejected": -3.156741142272949,
-      "step": 390
-    },
-    {
-      "epoch": 0.91,
-      "learning_rate": 1.3108553306396263e-08,
-      "logits/chosen": -2.1042487621307373,
-      "logits/rejected": -2.0541131496429443,
-      "logps/chosen": -535.3380126953125,
-      "logps/rejected": -510.76416015625,
-      "loss": 0.0379,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.9656769037246704,
-      "rewards/margins": 1.2311311960220337,
-      "rewards/rejected": -3.196807861328125,
-      "step": 400
-    },
-    {
-      "epoch": 0.91,
-      "eval_logits/chosen": -2.1270744800567627,
-      "eval_logits/rejected": -2.102363348007202,
-      "eval_logps/chosen": -489.1372375488281,
-      "eval_logps/rejected": -561.4609375,
-      "eval_loss": 0.05125967413187027,
-      "eval_rewards/accuracies": 0.71484375,
-      "eval_rewards/chosen": -2.3209760189056396,
-      "eval_rewards/margins": 0.7201008796691895,
-      "eval_rewards/rejected": -3.041076898574829,
-      "eval_runtime": 53.3435,
-      "eval_samples_per_second": 37.493,
-      "eval_steps_per_second": 0.6,
-      "step": 400
-    },
-    {
-      "epoch": 0.93,
-      "learning_rate": 7.522335858048705e-09,
-      "logits/chosen": -2.139943838119507,
-      "logits/rejected": -2.093489408493042,
-      "logps/chosen": -541.4684448242188,
-      "logps/rejected": -513.9920654296875,
-      "loss": 0.0401,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.8892101049423218,
-      "rewards/margins": 1.2535432577133179,
-      "rewards/rejected": -3.1427536010742188,
-      "step": 410
-    },
-    {
-      "epoch": 0.95,
-      "learning_rate": 3.4614115704533766e-09,
-      "logits/chosen": -2.1335110664367676,
-      "logits/rejected": -2.0755324363708496,
-      "logps/chosen": -552.4773559570312,
-      "logps/rejected": -511.8482971191406,
-      "loss": 0.0364,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.868373155593872,
-      "rewards/margins": 1.291067123413086,
-      "rewards/rejected": -3.159440279006958,
-      "step": 420
-    },
-    {
-      "epoch": 0.97,
-      "learning_rate": 9.513254770636137e-10,
-      "logits/chosen": -2.1106972694396973,
-      "logits/rejected": -2.043785810470581,
-      "logps/chosen": -515.1985473632812,
-      "logps/rejected": -489.65399169921875,
-      "loss": 0.0408,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.9040443897247314,
-      "rewards/margins": 1.2098912000656128,
-      "rewards/rejected": -3.1139354705810547,
-      "step": 430
-    },
     {
       "epoch": 1.0,
-      "learning_rate": 7.867144166728844e-12,
-      "logits/chosen": -2.0860373973846436,
-      "logits/rejected": -2.0547218322753906,
-      "logps/chosen": -530.7337646484375,
-      "logps/rejected": -510.04888916015625,
-      "loss": 0.0426,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": -1.9042129516601562,
-      "rewards/margins": 1.2389646768569946,
-      "rewards/rejected": -3.1431777477264404,
-      "step": 440
-    },
-    {
-      "epoch": 1.0,
-      "step": 441,
+      "step": 53,
       "total_flos": 0.0,
-      "train_loss": 0.07976685402716369,
-      "train_runtime": 7402.7266,
-      "train_samples_per_second": 15.268,
-      "train_steps_per_second": 0.06
+      "train_loss": 0.20092295110225677,
+      "train_runtime": 384.8987,
+      "train_samples_per_second": 17.537,
+      "train_steps_per_second": 0.138
     }
   ],
   "logging_steps": 10,
-  "max_steps": 441,
+  "max_steps": 53,
+  "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
+  "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8c151d7becb900bed631e41dff74cbdb1243adce5d3b22205a355b75f2b0912
-size 5944
+oid sha256:ead67c9eb9d466ab2cc1bdbe4406c9d56939d72f781464822bd099547b2d5fd8
+size 6456