Hugging Face's logo

sfulay
/
zephyr-7b-dpo-full-gpt_consistent-reward-scale-1-rpo

Safetensors
mistral
trl
dpo
alignment-handbook
Generated from Trainer
Model card Files Community
sfulay commited on
Commit
4d79451
·
verified ·
1 Parent(s): 0a3e251

Model save

Browse files
Files changed (9) hide show
  1. README.md +17 -17
  2. all_results.json +14 -14
  3. eval_results.json +11 -11
  4. model-00001-of-00003.safetensors +1 -1
  5. model-00002-of-00003.safetensors +1 -1
  6. model-00003-of-00003.safetensors +1 -1
  7. train_results.json +3 -3
  8. trainer_state.json +514 -514
  9. training_args.bin +1 -1
README.md CHANGED
@@ -18,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
18
 
19
  This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
20
  It achieves the following results on the evaluation set:
21
- - Loss: 0.1132
22
- - Rewards/chosen: -0.3948
23
- - Rewards/rejected: -0.9598
24
- - Rewards/accuracies: 0.7543
25
- - Rewards/margins: 0.5650
26
- - Logps/rejected: -342.4998
27
- - Logps/chosen: -324.5692
28
- - Logits/rejected: 1.4353
29
- - Logits/chosen: 0.4067
30
 
31
  ## Model description
32
 
@@ -63,14 +63,14 @@ The following hyperparameters were used during training:
63
 
64
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
65
  |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
66
- | 0.1732 | 0.1147 | 50 | 0.1641 | 0.0166 | -0.1092 | 0.7069 | 0.1259 | -257.4446 | -283.4267 | -2.4943 | -2.5737 |
67
- | 0.1412 | 0.2294 | 100 | 0.1347 | -0.4342 | -0.7962 | 0.7284 | 0.3620 | -326.1406 | -328.5106 | -0.1538 | -0.4184 |
68
- | 0.1307 | 0.3440 | 150 | 0.1261 | -0.3553 | -0.8583 | 0.7284 | 0.5030 | -332.3533 | -320.6210 | 0.7144 | 0.0181 |
69
- | 0.1238 | 0.4587 | 200 | 0.1199 | -0.4108 | -0.9476 | 0.7328 | 0.5368 | -341.2862 | -326.1717 | 1.2989 | 0.2969 |
70
- | 0.1185 | 0.5734 | 250 | 0.1166 | -0.3086 | -0.8924 | 0.7543 | 0.5838 | -335.7633 | -315.9550 | 0.8516 | -0.1745 |
71
- | 0.1228 | 0.6881 | 300 | 0.1155 | -0.3695 | -0.9267 | 0.7457 | 0.5571 | -339.1875 | -322.0434 | 0.8574 | -0.1316 |
72
- | 0.1213 | 0.8028 | 350 | 0.1136 | -0.4396 | -1.0157 | 0.7629 | 0.5762 | -348.0973 | -329.0486 | 1.5740 | 0.5152 |
73
- | 0.12 | 0.9174 | 400 | 0.1132 | -0.3948 | -0.9598 | 0.7543 | 0.5650 | -342.4998 | -324.5692 | 1.4353 | 0.4067 |
74
 
75
 
76
  ### Framework versions
 
18
 
19
  This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
20
  It achieves the following results on the evaluation set:
21
+ - Loss: 0.1126
22
+ - Rewards/chosen: -0.4018
23
+ - Rewards/rejected: -0.9642
24
+ - Rewards/accuracies: 0.75
25
+ - Rewards/margins: 0.5623
26
+ - Logps/rejected: -342.9399
27
+ - Logps/chosen: -325.2745
28
+ - Logits/rejected: 0.1904
29
+ - Logits/chosen: -0.6159
30
 
31
  ## Model description
32
 
 
63
 
64
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
65
  |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
66
+ | 0.1732 | 0.1147 | 50 | 0.1640 | 0.0140 | -0.1117 | 0.7069 | 0.1256 | -257.6913 | -283.6945 | -2.4927 | -2.5723 |
67
+ | 0.1412 | 0.2294 | 100 | 0.1392 | -0.5601 | -0.9388 | 0.7198 | 0.3787 | -340.4027 | -341.0970 | -0.8338 | -1.1029 |
68
+ | 0.1327 | 0.3440 | 150 | 0.1257 | -0.3660 | -0.8686 | 0.7241 | 0.5025 | -333.3776 | -321.6920 | -0.4213 | -0.9346 |
69
+ | 0.1242 | 0.4587 | 200 | 0.1193 | -0.3424 | -0.8506 | 0.7457 | 0.5082 | -331.5807 | -319.3252 | -0.1498 | -0.9084 |
70
+ | 0.118 | 0.5734 | 250 | 0.1167 | -0.3734 | -0.9685 | 0.7543 | 0.5950 | -343.3683 | -322.4342 | -0.0251 | -0.8147 |
71
+ | 0.1236 | 0.6881 | 300 | 0.1145 | -0.4780 | -1.0449 | 0.7629 | 0.5669 | -351.0162 | -332.8936 | 0.2891 | -0.5285 |
72
+ | 0.122 | 0.8028 | 350 | 0.1127 | -0.4577 | -1.0376 | 0.7672 | 0.5799 | -350.2830 | -330.8570 | 0.3397 | -0.4806 |
73
+ | 0.12 | 0.9174 | 400 | 0.1126 | -0.4018 | -0.9642 | 0.75 | 0.5623 | -342.9399 | -325.2745 | 0.1904 | -0.6159 |
74
 
75
 
76
  ### Framework versions
all_results.json CHANGED
@@ -1,22 +1,22 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.684370994567871,
4
- "eval_logits/rejected": -1.4736462831497192,
5
- "eval_logps/chosen": -297.6668701171875,
6
- "eval_logps/rejected": -289.2203674316406,
7
- "eval_loss": 0.032715052366256714,
8
  "eval_rewards/accuracies": 0.7543103694915771,
9
- "eval_rewards/chosen": -0.1257677674293518,
10
- "eval_rewards/margins": 0.30121153593063354,
11
- "eval_rewards/rejected": -0.42697927355766296,
12
- "eval_runtime": 94.8006,
13
  "eval_samples": 1818,
14
- "eval_samples_per_second": 19.177,
15
- "eval_steps_per_second": 0.306,
16
  "total_flos": 0.0,
17
- "train_loss": 0.13218348616853767,
18
- "train_runtime": 11343.5766,
19
  "train_samples": 55758,
20
- "train_samples_per_second": 4.915,
21
  "train_steps_per_second": 0.038
22
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": 0.41779446601867676,
4
+ "eval_logits/rejected": 1.454390525817871,
5
+ "eval_logps/chosen": -324.46234130859375,
6
+ "eval_logps/rejected": -342.5584716796875,
7
+ "eval_loss": 0.11311393231153488,
8
  "eval_rewards/accuracies": 0.7543103694915771,
9
+ "eval_rewards/chosen": -0.393722802400589,
10
+ "eval_rewards/margins": 0.566637396812439,
11
+ "eval_rewards/rejected": -0.9603601098060608,
12
+ "eval_runtime": 93.576,
13
  "eval_samples": 1818,
14
+ "eval_samples_per_second": 19.428,
15
+ "eval_steps_per_second": 0.31,
16
  "total_flos": 0.0,
17
+ "train_loss": 0.13197535372108493,
18
+ "train_runtime": 11523.0815,
19
  "train_samples": 55758,
20
+ "train_samples_per_second": 4.839,
21
  "train_steps_per_second": 0.038
22
  }
eval_results.json CHANGED
@@ -1,16 +1,16 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -1.684370994567871,
4
- "eval_logits/rejected": -1.4736462831497192,
5
- "eval_logps/chosen": -297.6668701171875,
6
- "eval_logps/rejected": -289.2203674316406,
7
- "eval_loss": 0.032715052366256714,
8
  "eval_rewards/accuracies": 0.7543103694915771,
9
- "eval_rewards/chosen": -0.1257677674293518,
10
- "eval_rewards/margins": 0.30121153593063354,
11
- "eval_rewards/rejected": -0.42697927355766296,
12
- "eval_runtime": 94.8006,
13
  "eval_samples": 1818,
14
- "eval_samples_per_second": 19.177,
15
- "eval_steps_per_second": 0.306
16
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": 0.41779446601867676,
4
+ "eval_logits/rejected": 1.454390525817871,
5
+ "eval_logps/chosen": -324.46234130859375,
6
+ "eval_logps/rejected": -342.5584716796875,
7
+ "eval_loss": 0.11311393231153488,
8
  "eval_rewards/accuracies": 0.7543103694915771,
9
+ "eval_rewards/chosen": -0.393722802400589,
10
+ "eval_rewards/margins": 0.566637396812439,
11
+ "eval_rewards/rejected": -0.9603601098060608,
12
+ "eval_runtime": 93.576,
13
  "eval_samples": 1818,
14
+ "eval_samples_per_second": 19.428,
15
+ "eval_steps_per_second": 0.31
16
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5905b6fface4e45339623c4d375f9edc78e435e73d598309e1ed43118ed92014
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e0e9f49e5125e802deca218e3a20f75ae9597d6c0bfe6256478d7a39e082ac0
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b04ddbeebeda55ae3848556f3a71f59213ac25a02039521f54874d8cdbff36f1
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e9cabac2e92bfb43293f329c88741c80f6d4c2811430c129984ce94af6a917d
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b01821c5089f266496aec2a04ad6bae11924a4608f9a7968429e02dcfedff8ea
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50e47b05e3ffc16c276de51ad276bb767d2f2baf1107bb39525ad6872def7e26
3
  size 4540516344
train_results.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "epoch": 1.0,
3
  "total_flos": 0.0,
4
- "train_loss": 0.13218348616853767,
5
- "train_runtime": 11343.5766,
6
  "train_samples": 55758,
7
- "train_samples_per_second": 4.915,
8
  "train_steps_per_second": 0.038
9
  }
 
1
  {
2
  "epoch": 1.0,
3
  "total_flos": 0.0,
4
+ "train_loss": 0.13197535372108493,
5
+ "train_runtime": 11523.0815,
6
  "train_samples": 55758,
7
+ "train_samples_per_second": 4.839,
8
  "train_steps_per_second": 0.038
9
  }
trainer_state.json CHANGED
@@ -10,784 +10,784 @@
10
  "log_history": [
11
  {
12
  "epoch": 0.022935779816513763,
13
- "grad_norm": 5.597671583305064,
14
  "learning_rate": 1.1363636363636363e-07,
15
- "logits/chosen": -2.6193342208862305,
16
- "logits/rejected": -2.5525386333465576,
17
- "logps/chosen": -265.3854064941406,
18
- "logps/rejected": -236.1589813232422,
19
- "loss": 0.1941,
20
- "rewards/accuracies": 0.41874998807907104,
21
- "rewards/chosen": 0.0006044252077117562,
22
- "rewards/margins": 0.0008946189773268998,
23
- "rewards/rejected": -0.000290193798718974,
24
  "step": 10
25
  },
26
  {
27
  "epoch": 0.045871559633027525,
28
- "grad_norm": 5.252146327563864,
29
  "learning_rate": 2.2727272727272726e-07,
30
- "logits/chosen": -2.657933473587036,
31
- "logits/rejected": -2.576028347015381,
32
- "logps/chosen": -298.8043518066406,
33
- "logps/rejected": -274.30718994140625,
34
- "loss": 0.1958,
35
- "rewards/accuracies": 0.581250011920929,
36
- "rewards/chosen": -0.00019080772472079843,
37
- "rewards/margins": 0.0017816700274124742,
38
- "rewards/rejected": -0.001972477650269866,
39
  "step": 20
40
  },
41
  {
42
  "epoch": 0.06880733944954129,
43
- "grad_norm": 4.974318928321663,
44
  "learning_rate": 3.4090909090909085e-07,
45
- "logits/chosen": -2.6759729385375977,
46
- "logits/rejected": -2.6023097038269043,
47
- "logps/chosen": -290.3868713378906,
48
- "logps/rejected": -234.3571319580078,
49
  "loss": 0.1923,
50
  "rewards/accuracies": 0.675000011920929,
51
- "rewards/chosen": 0.009458948858082294,
52
- "rewards/margins": 0.013211230747401714,
53
- "rewards/rejected": -0.0037522814236581326,
54
  "step": 30
55
  },
56
  {
57
  "epoch": 0.09174311926605505,
58
- "grad_norm": 5.209557025843711,
59
  "learning_rate": 4.545454545454545e-07,
60
- "logits/chosen": -2.659982204437256,
61
- "logits/rejected": -2.6102194786071777,
62
- "logps/chosen": -280.9975891113281,
63
- "logps/rejected": -267.7278747558594,
64
- "loss": 0.1836,
65
  "rewards/accuracies": 0.675000011920929,
66
- "rewards/chosen": 0.04171619936823845,
67
- "rewards/margins": 0.041064582765102386,
68
- "rewards/rejected": 0.0006516153225675225,
69
  "step": 40
70
  },
71
  {
72
  "epoch": 0.11467889908256881,
73
- "grad_norm": 5.582751836908906,
74
  "learning_rate": 4.997110275491701e-07,
75
- "logits/chosen": -2.6198437213897705,
76
- "logits/rejected": -2.6123862266540527,
77
- "logps/chosen": -293.4254455566406,
78
- "logps/rejected": -303.90386962890625,
79
  "loss": 0.1732,
80
- "rewards/accuracies": 0.6875,
81
- "rewards/chosen": 0.028257910162210464,
82
- "rewards/margins": 0.0754549652338028,
83
- "rewards/rejected": -0.047197047621011734,
84
  "step": 50
85
  },
86
  {
87
  "epoch": 0.11467889908256881,
88
- "eval_logits/chosen": -2.5737149715423584,
89
- "eval_logits/rejected": -2.494300365447998,
90
- "eval_logps/chosen": -283.4266662597656,
91
- "eval_logps/rejected": -257.4446105957031,
92
- "eval_loss": 0.16405636072158813,
93
  "eval_rewards/accuracies": 0.7068965435028076,
94
- "eval_rewards/chosen": 0.016634328290820122,
95
- "eval_rewards/margins": 0.12585583329200745,
96
- "eval_rewards/rejected": -0.10922150313854218,
97
- "eval_runtime": 93.7026,
98
- "eval_samples_per_second": 19.402,
99
- "eval_steps_per_second": 0.309,
100
  "step": 50
101
  },
102
  {
103
  "epoch": 0.13761467889908258,
104
- "grad_norm": 7.241200335566244,
105
  "learning_rate": 4.979475034558115e-07,
106
- "logits/chosen": -2.562969207763672,
107
- "logits/rejected": -2.5053610801696777,
108
- "logps/chosen": -293.1821594238281,
109
- "logps/rejected": -272.9414367675781,
110
- "loss": 0.1593,
111
  "rewards/accuracies": 0.699999988079071,
112
- "rewards/chosen": -0.03697499260306358,
113
- "rewards/margins": 0.11687393486499786,
114
- "rewards/rejected": -0.15384894609451294,
115
  "step": 60
116
  },
117
  {
118
  "epoch": 0.16055045871559634,
119
- "grad_norm": 7.058665086800562,
120
  "learning_rate": 4.945923025551788e-07,
121
- "logits/chosen": -2.4592204093933105,
122
- "logits/rejected": -2.4099252223968506,
123
- "logps/chosen": -327.544189453125,
124
- "logps/rejected": -284.48760986328125,
125
- "loss": 0.1564,
126
  "rewards/accuracies": 0.6937500238418579,
127
- "rewards/chosen": 0.00026793667348101735,
128
- "rewards/margins": 0.23106786608695984,
129
- "rewards/rejected": -0.23079994320869446,
130
  "step": 70
131
  },
132
  {
133
  "epoch": 0.1834862385321101,
134
- "grad_norm": 8.05919017424508,
135
  "learning_rate": 4.896669632591651e-07,
136
- "logits/chosen": -2.3360755443573,
137
- "logits/rejected": -2.2115492820739746,
138
- "logps/chosen": -298.69482421875,
139
- "logps/rejected": -292.111328125,
140
- "loss": 0.1478,
141
- "rewards/accuracies": 0.71875,
142
- "rewards/chosen": -0.128449946641922,
143
- "rewards/margins": 0.26610296964645386,
144
- "rewards/rejected": -0.39455294609069824,
145
  "step": 80
146
  },
147
  {
148
  "epoch": 0.20642201834862386,
149
- "grad_norm": 8.402102216426163,
150
  "learning_rate": 4.832031033425662e-07,
151
- "logits/chosen": -1.9170547723770142,
152
- "logits/rejected": -1.8550583124160767,
153
- "logps/chosen": -299.20281982421875,
154
- "logps/rejected": -289.49163818359375,
155
- "loss": 0.1441,
156
- "rewards/accuracies": 0.7124999761581421,
157
- "rewards/chosen": -0.1160941943526268,
158
- "rewards/margins": 0.3108092248439789,
159
- "rewards/rejected": -0.4269034266471863,
160
  "step": 90
161
  },
162
  {
163
  "epoch": 0.22935779816513763,
164
- "grad_norm": 8.625504630985954,
165
  "learning_rate": 4.752422169756047e-07,
166
- "logits/chosen": -0.48900121450424194,
167
- "logits/rejected": -0.44961875677108765,
168
- "logps/chosen": -311.1794738769531,
169
- "logps/rejected": -339.92254638671875,
170
  "loss": 0.1412,
171
- "rewards/accuracies": 0.6812499761581421,
172
- "rewards/chosen": -0.3875480592250824,
173
- "rewards/margins": 0.2923927903175354,
174
- "rewards/rejected": -0.6799408197402954,
175
  "step": 100
176
  },
177
  {
178
  "epoch": 0.22935779816513763,
179
- "eval_logits/chosen": -0.4184306561946869,
180
- "eval_logits/rejected": -0.1537734419107437,
181
- "eval_logps/chosen": -328.5106201171875,
182
- "eval_logps/rejected": -326.1405944824219,
183
- "eval_loss": 0.13468901813030243,
184
- "eval_rewards/accuracies": 0.7284482717514038,
185
- "eval_rewards/chosen": -0.4342052936553955,
186
- "eval_rewards/margins": 0.3619759678840637,
187
- "eval_rewards/rejected": -0.796181321144104,
188
- "eval_runtime": 92.0893,
189
- "eval_samples_per_second": 19.742,
190
- "eval_steps_per_second": 0.315,
191
  "step": 100
192
  },
193
  {
194
  "epoch": 0.25229357798165136,
195
- "grad_norm": 10.103043851045701,
196
  "learning_rate": 4.658354083558188e-07,
197
- "logits/chosen": -0.40221118927001953,
198
- "logits/rejected": -0.13967064023017883,
199
- "logps/chosen": -312.624267578125,
200
- "logps/rejected": -311.3497009277344,
201
- "loss": 0.1354,
202
- "rewards/accuracies": 0.706250011920929,
203
- "rewards/chosen": -0.4743613302707672,
204
- "rewards/margins": 0.3018040060997009,
205
- "rewards/rejected": -0.776165246963501,
206
  "step": 110
207
  },
208
  {
209
  "epoch": 0.27522935779816515,
210
- "grad_norm": 10.671011000324334,
211
  "learning_rate": 4.550430636492389e-07,
212
- "logits/chosen": -0.04719700291752815,
213
- "logits/rejected": 0.09439365565776825,
214
- "logps/chosen": -324.4268798828125,
215
- "logps/rejected": -335.16424560546875,
216
- "loss": 0.1353,
217
- "rewards/accuracies": 0.699999988079071,
218
- "rewards/chosen": -0.4860754609107971,
219
- "rewards/margins": 0.3509276807308197,
220
- "rewards/rejected": -0.8370031118392944,
221
  "step": 120
222
  },
223
  {
224
  "epoch": 0.2981651376146789,
225
- "grad_norm": 11.348522180816168,
226
  "learning_rate": 4.429344633468004e-07,
227
- "logits/chosen": -0.058751799166202545,
228
- "logits/rejected": 0.29970529675483704,
229
- "logps/chosen": -296.40399169921875,
230
- "logps/rejected": -316.8427429199219,
231
- "loss": 0.1373,
232
- "rewards/accuracies": 0.6875,
233
- "rewards/chosen": -0.4022388458251953,
234
- "rewards/margins": 0.3400627374649048,
235
- "rewards/rejected": -0.7423015832901001,
236
  "step": 130
237
  },
238
  {
239
  "epoch": 0.3211009174311927,
240
- "grad_norm": 25.0836527688945,
241
  "learning_rate": 4.2958733752443187e-07,
242
- "logits/chosen": -0.29172104597091675,
243
- "logits/rejected": 0.11182677745819092,
244
- "logps/chosen": -303.4015197753906,
245
- "logps/rejected": -286.45526123046875,
246
- "loss": 0.1334,
247
- "rewards/accuracies": 0.7124999761581421,
248
- "rewards/chosen": -0.2721634805202484,
249
- "rewards/margins": 0.34942418336868286,
250
- "rewards/rejected": -0.6215876340866089,
251
  "step": 140
252
  },
253
  {
254
  "epoch": 0.3440366972477064,
255
- "grad_norm": 11.640337926409853,
256
  "learning_rate": 4.150873668617898e-07,
257
- "logits/chosen": 0.08257939666509628,
258
- "logits/rejected": 0.5254168510437012,
259
- "logps/chosen": -308.8121337890625,
260
- "logps/rejected": -314.32220458984375,
261
- "loss": 0.1307,
262
- "rewards/accuracies": 0.7124999761581421,
263
- "rewards/chosen": -0.42268529534339905,
264
- "rewards/margins": 0.3886614143848419,
265
- "rewards/rejected": -0.8113466501235962,
266
  "step": 150
267
  },
268
  {
269
  "epoch": 0.3440366972477064,
270
- "eval_logits/chosen": 0.018126631155610085,
271
- "eval_logits/rejected": 0.7143937349319458,
272
- "eval_logps/chosen": -320.6210021972656,
273
- "eval_logps/rejected": -332.353271484375,
274
- "eval_loss": 0.12613436579704285,
275
- "eval_rewards/accuracies": 0.7284482717514038,
276
- "eval_rewards/chosen": -0.3553086221218109,
277
- "eval_rewards/margins": 0.5029994249343872,
278
- "eval_rewards/rejected": -0.8583080768585205,
279
- "eval_runtime": 91.5757,
280
- "eval_samples_per_second": 19.852,
281
- "eval_steps_per_second": 0.317,
282
  "step": 150
283
  },
284
  {
285
  "epoch": 0.3669724770642202,
286
- "grad_norm": 11.56271576551216,
287
  "learning_rate": 3.9952763262280397e-07,
288
- "logits/chosen": 0.02932182513177395,
289
- "logits/rejected": 0.4751170575618744,
290
- "logps/chosen": -337.37237548828125,
291
- "logps/rejected": -373.10382080078125,
292
- "loss": 0.1247,
293
- "rewards/accuracies": 0.731249988079071,
294
- "rewards/chosen": -0.40821242332458496,
295
- "rewards/margins": 0.46313947439193726,
296
- "rewards/rejected": -0.8713518381118774,
297
  "step": 160
298
  },
299
  {
300
  "epoch": 0.38990825688073394,
301
- "grad_norm": 7.985791463305948,
302
  "learning_rate": 3.8300801912883414e-07,
303
- "logits/chosen": 0.06568197906017303,
304
- "logits/rejected": 0.3852883279323578,
305
- "logps/chosen": -311.62481689453125,
306
- "logps/rejected": -353.85443115234375,
307
- "loss": 0.1191,
308
  "rewards/accuracies": 0.699999988079071,
309
- "rewards/chosen": -0.40023931860923767,
310
- "rewards/margins": 0.4376160204410553,
311
- "rewards/rejected": -0.837855339050293,
312
  "step": 170
313
  },
314
  {
315
  "epoch": 0.41284403669724773,
316
- "grad_norm": 9.33389793686436,
317
  "learning_rate": 3.6563457256020884e-07,
318
- "logits/chosen": 0.15779247879981995,
319
- "logits/rejected": 0.6506751775741577,
320
- "logps/chosen": -348.14398193359375,
321
- "logps/rejected": -325.84637451171875,
322
- "loss": 0.1278,
323
- "rewards/accuracies": 0.7124999761581421,
324
- "rewards/chosen": -0.44786080718040466,
325
- "rewards/margins": 0.4256533682346344,
326
- "rewards/rejected": -0.8735141754150391,
327
  "step": 180
328
  },
329
  {
330
  "epoch": 0.43577981651376146,
331
- "grad_norm": 10.313716932222244,
332
  "learning_rate": 3.475188202022617e-07,
333
- "logits/chosen": -0.43816322088241577,
334
- "logits/rejected": 0.023333895951509476,
335
- "logps/chosen": -283.8222351074219,
336
- "logps/rejected": -337.9872741699219,
337
- "loss": 0.1236,
338
  "rewards/accuracies": 0.7749999761581421,
339
- "rewards/chosen": -0.2570773959159851,
340
- "rewards/margins": 0.47935089468955994,
341
- "rewards/rejected": -0.7364283800125122,
342
  "step": 190
343
  },
344
  {
345
  "epoch": 0.45871559633027525,
346
- "grad_norm": 9.248153089741166,
347
  "learning_rate": 3.287770545059052e-07,
348
- "logits/chosen": -0.08292710781097412,
349
- "logits/rejected": 0.5214661359786987,
350
- "logps/chosen": -312.56500244140625,
351
- "logps/rejected": -325.7038269042969,
352
- "loss": 0.1238,
353
- "rewards/accuracies": 0.800000011920929,
354
- "rewards/chosen": -0.34283357858657837,
355
- "rewards/margins": 0.4756447672843933,
356
- "rewards/rejected": -0.8184784054756165,
357
  "step": 200
358
  },
359
  {
360
  "epoch": 0.45871559633027525,
361
- "eval_logits/chosen": 0.2969372272491455,
362
- "eval_logits/rejected": 1.2989132404327393,
363
- "eval_logps/chosen": -326.1716613769531,
364
- "eval_logps/rejected": -341.28619384765625,
365
- "eval_loss": 0.11991516500711441,
366
- "eval_rewards/accuracies": 0.732758641242981,
367
- "eval_rewards/chosen": -0.41081586480140686,
368
- "eval_rewards/margins": 0.5368219614028931,
369
- "eval_rewards/rejected": -0.9476377367973328,
370
- "eval_runtime": 92.1824,
371
- "eval_samples_per_second": 19.722,
372
- "eval_steps_per_second": 0.315,
373
  "step": 200
374
  },
375
  {
376
  "epoch": 0.481651376146789,
377
- "grad_norm": 8.486691996298621,
378
  "learning_rate": 3.0952958655864954e-07,
379
- "logits/chosen": 0.504795253276825,
380
- "logits/rejected": 1.0519931316375732,
381
- "logps/chosen": -317.31756591796875,
382
- "logps/rejected": -346.00689697265625,
383
- "loss": 0.1175,
384
- "rewards/accuracies": 0.75,
385
- "rewards/chosen": -0.44013166427612305,
386
- "rewards/margins": 0.47651535272598267,
387
- "rewards/rejected": -0.9166469573974609,
388
  "step": 210
389
  },
390
  {
391
  "epoch": 0.5045871559633027,
392
- "grad_norm": 9.587760322945808,
393
  "learning_rate": 2.898999737583448e-07,
394
- "logits/chosen": 0.056784339249134064,
395
- "logits/rejected": 1.057108998298645,
396
- "logps/chosen": -358.15313720703125,
397
- "logps/rejected": -375.32452392578125,
398
- "loss": 0.1134,
399
- "rewards/accuracies": 0.7875000238418579,
400
- "rewards/chosen": -0.3547281324863434,
401
- "rewards/margins": 0.548815131187439,
402
- "rewards/rejected": -0.9035432934761047,
403
  "step": 220
404
  },
405
  {
406
  "epoch": 0.5275229357798165,
407
- "grad_norm": 9.093354835969592,
408
  "learning_rate": 2.7001422664752333e-07,
409
- "logits/chosen": -0.12167753279209137,
410
- "logits/rejected": 0.5305444002151489,
411
- "logps/chosen": -298.165283203125,
412
- "logps/rejected": -341.3959045410156,
413
- "loss": 0.1206,
414
- "rewards/accuracies": 0.7749999761581421,
415
- "rewards/chosen": -0.3022175431251526,
416
- "rewards/margins": 0.4555422365665436,
417
- "rewards/rejected": -0.7577598094940186,
418
  "step": 230
419
  },
420
  {
421
  "epoch": 0.5504587155963303,
422
- "grad_norm": 9.439667913770359,
423
  "learning_rate": 2.5e-07,
424
- "logits/chosen": -0.21128520369529724,
425
- "logits/rejected": 0.5969688296318054,
426
- "logps/chosen": -321.5914306640625,
427
- "logps/rejected": -339.0221862792969,
428
- "loss": 0.1218,
429
  "rewards/accuracies": 0.737500011920929,
430
- "rewards/chosen": -0.34059566259384155,
431
- "rewards/margins": 0.43612655997276306,
432
- "rewards/rejected": -0.7767221927642822,
433
  "step": 240
434
  },
435
  {
436
  "epoch": 0.573394495412844,
437
- "grad_norm": 9.292342727311365,
438
  "learning_rate": 2.2998577335247667e-07,
439
- "logits/chosen": -0.020272482186555862,
440
- "logits/rejected": 0.808585524559021,
441
- "logps/chosen": -337.55767822265625,
442
- "logps/rejected": -346.179443359375,
443
- "loss": 0.1185,
444
- "rewards/accuracies": 0.793749988079071,
445
- "rewards/chosen": -0.337638795375824,
446
- "rewards/margins": 0.5197167992591858,
447
- "rewards/rejected": -0.8573554754257202,
448
  "step": 250
449
  },
450
  {
451
  "epoch": 0.573394495412844,
452
- "eval_logits/chosen": -0.1745007336139679,
453
- "eval_logits/rejected": 0.8515735268592834,
454
- "eval_logps/chosen": -315.9549560546875,
455
- "eval_logps/rejected": -335.7632751464844,
456
- "eval_loss": 0.11658623069524765,
457
  "eval_rewards/accuracies": 0.7543103694915771,
458
- "eval_rewards/chosen": -0.3086487948894501,
459
- "eval_rewards/margins": 0.5837592482566833,
460
- "eval_rewards/rejected": -0.8924079537391663,
461
- "eval_runtime": 92.3611,
462
- "eval_samples_per_second": 19.684,
463
- "eval_steps_per_second": 0.314,
464
  "step": 250
465
  },
466
  {
467
  "epoch": 0.5963302752293578,
468
- "grad_norm": 8.186794570934321,
469
  "learning_rate": 2.1010002624165524e-07,
470
- "logits/chosen": -0.006458556745201349,
471
- "logits/rejected": 0.6227847933769226,
472
- "logps/chosen": -308.7506408691406,
473
- "logps/rejected": -372.9380798339844,
474
- "loss": 0.1168,
475
- "rewards/accuracies": 0.800000011920929,
476
- "rewards/chosen": -0.3803611397743225,
477
- "rewards/margins": 0.6050828695297241,
478
- "rewards/rejected": -0.9854438900947571,
479
  "step": 260
480
  },
481
  {
482
  "epoch": 0.6192660550458715,
483
- "grad_norm": 10.337683797583873,
484
  "learning_rate": 1.9047041344135043e-07,
485
- "logits/chosen": 0.44111576676368713,
486
- "logits/rejected": 0.8292443156242371,
487
- "logps/chosen": -313.4148254394531,
488
- "logps/rejected": -354.33258056640625,
489
- "loss": 0.1168,
490
- "rewards/accuracies": 0.793749988079071,
491
- "rewards/chosen": -0.47352510690689087,
492
- "rewards/margins": 0.5513299703598022,
493
- "rewards/rejected": -1.0248548984527588,
494
  "step": 270
495
  },
496
  {
497
  "epoch": 0.6422018348623854,
498
- "grad_norm": 10.116957119119249,
499
  "learning_rate": 1.7122294549409482e-07,
500
- "logits/chosen": 0.1775570809841156,
501
- "logits/rejected": 1.0465366840362549,
502
- "logps/chosen": -317.04180908203125,
503
- "logps/rejected": -370.72479248046875,
504
- "loss": 0.1221,
505
- "rewards/accuracies": 0.762499988079071,
506
- "rewards/chosen": -0.4404297471046448,
507
- "rewards/margins": 0.6288600564002991,
508
- "rewards/rejected": -1.0692898035049438,
509
  "step": 280
510
  },
511
  {
512
  "epoch": 0.6651376146788991,
513
- "grad_norm": 9.609746165977107,
514
  "learning_rate": 1.524811797977383e-07,
515
- "logits/chosen": 0.02145857736468315,
516
- "logits/rejected": 0.823032021522522,
517
- "logps/chosen": -322.24542236328125,
518
- "logps/rejected": -342.22503662109375,
519
- "loss": 0.1156,
520
- "rewards/accuracies": 0.762499988079071,
521
- "rewards/chosen": -0.3451458811759949,
522
- "rewards/margins": 0.5379365086555481,
523
- "rewards/rejected": -0.883082389831543,
524
  "step": 290
525
  },
526
  {
527
  "epoch": 0.6880733944954128,
528
- "grad_norm": 9.08082569283608,
529
  "learning_rate": 1.3436542743979125e-07,
530
- "logits/chosen": -0.18167677521705627,
531
- "logits/rejected": 0.25138911604881287,
532
- "logps/chosen": -341.6298828125,
533
- "logps/rejected": -338.09869384765625,
534
- "loss": 0.1228,
535
- "rewards/accuracies": 0.737500011920929,
536
- "rewards/chosen": -0.3568056523799896,
537
- "rewards/margins": 0.43558257818222046,
538
- "rewards/rejected": -0.7923881411552429,
539
  "step": 300
540
  },
541
  {
542
  "epoch": 0.6880733944954128,
543
- "eval_logits/chosen": -0.13157324492931366,
544
- "eval_logits/rejected": 0.8573818206787109,
545
- "eval_logps/chosen": -322.0434265136719,
546
- "eval_logps/rejected": -339.1875305175781,
547
- "eval_loss": 0.11548212170600891,
548
- "eval_rewards/accuracies": 0.7456896305084229,
549
- "eval_rewards/chosen": -0.3695334494113922,
550
- "eval_rewards/margins": 0.5571174621582031,
551
- "eval_rewards/rejected": -0.9266510009765625,
552
- "eval_runtime": 91.8257,
553
- "eval_samples_per_second": 19.798,
554
- "eval_steps_per_second": 0.316,
555
  "step": 300
556
  },
557
  {
558
  "epoch": 0.7110091743119266,
559
- "grad_norm": 9.32642322141669,
560
  "learning_rate": 1.1699198087116588e-07,
561
- "logits/chosen": -0.07652176916599274,
562
- "logits/rejected": 0.6259859800338745,
563
- "logps/chosen": -321.20538330078125,
564
- "logps/rejected": -355.15618896484375,
565
- "loss": 0.1239,
566
- "rewards/accuracies": 0.737500011920929,
567
- "rewards/chosen": -0.43240228295326233,
568
- "rewards/margins": 0.4546627104282379,
569
- "rewards/rejected": -0.887065052986145,
570
  "step": 310
571
  },
572
  {
573
  "epoch": 0.7339449541284404,
574
- "grad_norm": 9.870138396091912,
575
  "learning_rate": 1.00472367377196e-07,
576
- "logits/chosen": 0.15437743067741394,
577
- "logits/rejected": 0.9939621686935425,
578
- "logps/chosen": -314.3326416015625,
579
- "logps/rejected": -335.50787353515625,
580
- "loss": 0.1193,
581
  "rewards/accuracies": 0.78125,
582
- "rewards/chosen": -0.40100646018981934,
583
- "rewards/margins": 0.6091704964637756,
584
- "rewards/rejected": -1.0101768970489502,
585
  "step": 320
586
  },
587
  {
588
  "epoch": 0.7568807339449541,
589
- "grad_norm": 10.470680354911353,
590
  "learning_rate": 8.49126331382102e-08,
591
- "logits/chosen": 0.41432857513427734,
592
- "logits/rejected": 0.9864773750305176,
593
- "logps/chosen": -319.98114013671875,
594
- "logps/rejected": -336.5364074707031,
595
- "loss": 0.1183,
596
- "rewards/accuracies": 0.731249988079071,
597
- "rewards/chosen": -0.45458975434303284,
598
- "rewards/margins": 0.43977364897727966,
599
- "rewards/rejected": -0.8943634033203125,
600
  "step": 330
601
  },
602
  {
603
  "epoch": 0.7798165137614679,
604
- "grad_norm": 10.41854471188631,
605
  "learning_rate": 7.041266247556812e-08,
606
- "logits/chosen": 0.3841426968574524,
607
- "logits/rejected": 0.9351291656494141,
608
- "logps/chosen": -334.6560974121094,
609
- "logps/rejected": -349.6623229980469,
610
- "loss": 0.1152,
611
- "rewards/accuracies": 0.7875000238418579,
612
- "rewards/chosen": -0.4666348397731781,
613
- "rewards/margins": 0.4301484227180481,
614
- "rewards/rejected": -0.8967832326889038,
615
  "step": 340
616
  },
617
  {
618
  "epoch": 0.8027522935779816,
619
- "grad_norm": 9.996573814918259,
620
  "learning_rate": 5.706553665319955e-08,
621
- "logits/chosen": 0.6401292085647583,
622
- "logits/rejected": 1.3996922969818115,
623
- "logps/chosen": -324.6096496582031,
624
- "logps/rejected": -337.38555908203125,
625
- "loss": 0.1213,
626
- "rewards/accuracies": 0.78125,
627
- "rewards/chosen": -0.5011857151985168,
628
- "rewards/margins": 0.4960555136203766,
629
- "rewards/rejected": -0.997241199016571,
630
  "step": 350
631
  },
632
  {
633
  "epoch": 0.8027522935779816,
634
- "eval_logits/chosen": 0.5151604413986206,
635
- "eval_logits/rejected": 1.5740225315093994,
636
- "eval_logps/chosen": -329.048583984375,
637
- "eval_logps/rejected": -348.0972900390625,
638
- "eval_loss": 0.113578200340271,
639
- "eval_rewards/accuracies": 0.7629310488700867,
640
- "eval_rewards/chosen": -0.4395850598812103,
641
- "eval_rewards/margins": 0.5761635303497314,
642
- "eval_rewards/rejected": -1.0157485008239746,
643
- "eval_runtime": 92.2267,
644
- "eval_samples_per_second": 19.712,
645
- "eval_steps_per_second": 0.314,
646
  "step": 350
647
  },
648
  {
649
  "epoch": 0.8256880733944955,
650
- "grad_norm": 8.23863269387286,
651
  "learning_rate": 4.4956936350761005e-08,
652
- "logits/chosen": 0.49835261702537537,
653
- "logits/rejected": 1.0901247262954712,
654
- "logps/chosen": -292.41424560546875,
655
- "logps/rejected": -345.85540771484375,
656
- "loss": 0.1131,
657
- "rewards/accuracies": 0.6812499761581421,
658
- "rewards/chosen": -0.40094178915023804,
659
- "rewards/margins": 0.4722411632537842,
660
- "rewards/rejected": -0.8731829524040222,
661
  "step": 360
662
  },
663
  {
664
  "epoch": 0.8486238532110092,
665
- "grad_norm": 10.10305933177313,
666
  "learning_rate": 3.416459164418123e-08,
667
- "logits/chosen": 0.16921785473823547,
668
- "logits/rejected": 0.9673768877983093,
669
- "logps/chosen": -340.80059814453125,
670
- "logps/rejected": -353.12274169921875,
671
- "loss": 0.1129,
672
- "rewards/accuracies": 0.7437499761581421,
673
- "rewards/chosen": -0.41216129064559937,
674
- "rewards/margins": 0.4933200776576996,
675
- "rewards/rejected": -0.9054813385009766,
676
  "step": 370
677
  },
678
  {
679
  "epoch": 0.8715596330275229,
680
- "grad_norm": 9.150940276647933,
681
  "learning_rate": 2.475778302439524e-08,
682
- "logits/chosen": 0.3220874071121216,
683
- "logits/rejected": 1.2661678791046143,
684
- "logps/chosen": -332.2058410644531,
685
- "logps/rejected": -351.96856689453125,
686
- "loss": 0.1174,
687
  "rewards/accuracies": 0.7749999761581421,
688
- "rewards/chosen": -0.35827815532684326,
689
- "rewards/margins": 0.5848454236984253,
690
- "rewards/rejected": -0.9431236386299133,
691
  "step": 380
692
  },
693
  {
694
  "epoch": 0.8944954128440367,
695
- "grad_norm": 9.488885903438641,
696
  "learning_rate": 1.6796896657433805e-08,
697
- "logits/chosen": 0.6681944727897644,
698
- "logits/rejected": 1.2935806512832642,
699
- "logps/chosen": -292.4158630371094,
700
- "logps/rejected": -315.7539978027344,
701
- "loss": 0.123,
702
- "rewards/accuracies": 0.75,
703
- "rewards/chosen": -0.40458402037620544,
704
- "rewards/margins": 0.4598938524723053,
705
- "rewards/rejected": -0.8644779324531555,
706
  "step": 390
707
  },
708
  {
709
  "epoch": 0.9174311926605505,
710
- "grad_norm": 9.000204295912624,
711
  "learning_rate": 1.0333036740834855e-08,
712
- "logits/chosen": 0.7832537889480591,
713
- "logits/rejected": 1.2279746532440186,
714
- "logps/chosen": -260.876708984375,
715
- "logps/rejected": -314.25390625,
716
  "loss": 0.12,
717
- "rewards/accuracies": 0.7250000238418579,
718
- "rewards/chosen": -0.3631708323955536,
719
- "rewards/margins": 0.4872121214866638,
720
- "rewards/rejected": -0.8503829836845398,
721
  "step": 400
722
  },
723
  {
724
  "epoch": 0.9174311926605505,
725
- "eval_logits/chosen": 0.40672120451927185,
726
- "eval_logits/rejected": 1.4352502822875977,
727
- "eval_logps/chosen": -324.5692443847656,
728
- "eval_logps/rejected": -342.49981689453125,
729
- "eval_loss": 0.11321888864040375,
730
- "eval_rewards/accuracies": 0.7543103694915771,
731
- "eval_rewards/chosen": -0.39479103684425354,
732
- "eval_rewards/margins": 0.5649827122688293,
733
- "eval_rewards/rejected": -0.9597737789154053,
734
- "eval_runtime": 92.1464,
735
- "eval_samples_per_second": 19.729,
736
- "eval_steps_per_second": 0.315,
737
  "step": 400
738
  },
739
  {
740
  "epoch": 0.9403669724770642,
741
- "grad_norm": 12.797858678996972,
742
  "learning_rate": 5.4076974448211685e-09,
743
- "logits/chosen": 0.733371913433075,
744
- "logits/rejected": 1.6556295156478882,
745
- "logps/chosen": -313.32049560546875,
746
- "logps/rejected": -336.0906677246094,
747
- "loss": 0.1171,
748
- "rewards/accuracies": 0.75,
749
- "rewards/chosen": -0.42368635535240173,
750
- "rewards/margins": 0.5567531585693359,
751
- "rewards/rejected": -0.9804395437240601,
752
  "step": 410
753
  },
754
  {
755
  "epoch": 0.963302752293578,
756
- "grad_norm": 11.330514295442143,
757
  "learning_rate": 2.052496544188487e-09,
758
- "logits/chosen": 0.8590083122253418,
759
- "logits/rejected": 1.5339264869689941,
760
- "logps/chosen": -304.95831298828125,
761
- "logps/rejected": -347.80169677734375,
762
- "loss": 0.1153,
763
- "rewards/accuracies": 0.800000011920929,
764
- "rewards/chosen": -0.4551934599876404,
765
- "rewards/margins": 0.5640500783920288,
766
- "rewards/rejected": -1.019243597984314,
767
  "step": 420
768
  },
769
  {
770
  "epoch": 0.9862385321100917,
771
- "grad_norm": 9.086504274831077,
772
  "learning_rate": 2.889724508297886e-10,
773
- "logits/chosen": 0.5266289710998535,
774
- "logits/rejected": 1.5435163974761963,
775
- "logps/chosen": -344.6377868652344,
776
- "logps/rejected": -336.5421447753906,
777
- "loss": 0.1158,
778
- "rewards/accuracies": 0.762499988079071,
779
- "rewards/chosen": -0.41462403535842896,
780
- "rewards/margins": 0.5104321837425232,
781
- "rewards/rejected": -0.9250561594963074,
782
  "step": 430
783
  },
784
  {
785
  "epoch": 1.0,
786
  "step": 436,
787
  "total_flos": 0.0,
788
- "train_loss": 0.13218348616853767,
789
- "train_runtime": 11343.5766,
790
- "train_samples_per_second": 4.915,
791
  "train_steps_per_second": 0.038
792
  }
793
  ],
 
10
  "log_history": [
11
  {
12
  "epoch": 0.022935779816513763,
13
+ "grad_norm": 5.604829014017037,
14
  "learning_rate": 1.1363636363636363e-07,
15
+ "logits/chosen": -2.619293689727783,
16
+ "logits/rejected": -2.5524563789367676,
17
+ "logps/chosen": -265.4638977050781,
18
+ "logps/rejected": -236.136962890625,
19
+ "loss": 0.1942,
20
+ "rewards/accuracies": 0.4000000059604645,
21
+ "rewards/chosen": -0.00018078314315062016,
22
+ "rewards/margins": -0.00011108322360087186,
23
+ "rewards/rejected": -6.969989772187546e-05,
24
  "step": 10
25
  },
26
  {
27
  "epoch": 0.045871559633027525,
28
+ "grad_norm": 5.250583616242791,
29
  "learning_rate": 2.2727272727272726e-07,
30
+ "logits/chosen": -2.6577353477478027,
31
+ "logits/rejected": -2.575864315032959,
32
+ "logps/chosen": -298.8517150878906,
33
+ "logps/rejected": -274.293212890625,
34
+ "loss": 0.1959,
35
+ "rewards/accuracies": 0.6312500238418579,
36
+ "rewards/chosen": -0.0006643417873419821,
37
+ "rewards/margins": 0.0011681303149089217,
38
+ "rewards/rejected": -0.0018324721604585648,
39
  "step": 20
40
  },
41
  {
42
  "epoch": 0.06880733944954129,
43
+ "grad_norm": 4.980077918182304,
44
  "learning_rate": 3.4090909090909085e-07,
45
+ "logits/chosen": -2.675644636154175,
46
+ "logits/rejected": -2.6021487712860107,
47
+ "logps/chosen": -290.4170227050781,
48
+ "logps/rejected": -234.3779296875,
49
  "loss": 0.1923,
50
  "rewards/accuracies": 0.675000011920929,
51
+ "rewards/chosen": 0.009157607331871986,
52
+ "rewards/margins": 0.013118197210133076,
53
+ "rewards/rejected": -0.003960589878261089,
54
  "step": 30
55
  },
56
  {
57
  "epoch": 0.09174311926605505,
58
+ "grad_norm": 5.215480488397355,
59
  "learning_rate": 4.545454545454545e-07,
60
+ "logits/chosen": -2.660599946975708,
61
+ "logits/rejected": -2.6108055114746094,
62
+ "logps/chosen": -281.02667236328125,
63
+ "logps/rejected": -267.7845764160156,
64
+ "loss": 0.1835,
65
  "rewards/accuracies": 0.675000011920929,
66
+ "rewards/chosen": 0.04142485558986664,
67
+ "rewards/margins": 0.04134037345647812,
68
+ "rewards/rejected": 8.447784784948453e-05,
69
  "step": 40
70
  },
71
  {
72
  "epoch": 0.11467889908256881,
73
+ "grad_norm": 5.6967811731593905,
74
  "learning_rate": 4.997110275491701e-07,
75
+ "logits/chosen": -2.619861602783203,
76
+ "logits/rejected": -2.6124091148376465,
77
+ "logps/chosen": -293.50762939453125,
78
+ "logps/rejected": -303.9714660644531,
79
  "loss": 0.1732,
80
+ "rewards/accuracies": 0.6812499761581421,
81
+ "rewards/chosen": 0.027435380965471268,
82
+ "rewards/margins": 0.0753081664443016,
83
+ "rewards/rejected": -0.04787278175354004,
84
  "step": 50
85
  },
86
  {
87
  "epoch": 0.11467889908256881,
88
+ "eval_logits/chosen": -2.5723471641540527,
89
+ "eval_logits/rejected": -2.49267315864563,
90
+ "eval_logps/chosen": -283.6944580078125,
91
+ "eval_logps/rejected": -257.6913146972656,
92
+ "eval_loss": 0.1639617681503296,
93
  "eval_rewards/accuracies": 0.7068965435028076,
94
+ "eval_rewards/chosen": 0.013956375420093536,
95
+ "eval_rewards/margins": 0.12564486265182495,
96
+ "eval_rewards/rejected": -0.11168847978115082,
97
+ "eval_runtime": 95.9923,
98
+ "eval_samples_per_second": 18.939,
99
+ "eval_steps_per_second": 0.302,
100
  "step": 50
101
  },
102
  {
103
  "epoch": 0.13761467889908258,
104
+ "grad_norm": 7.248589740263705,
105
  "learning_rate": 4.979475034558115e-07,
106
+ "logits/chosen": -2.5621228218078613,
107
+ "logits/rejected": -2.503438949584961,
108
+ "logps/chosen": -293.131103515625,
109
+ "logps/rejected": -272.94793701171875,
110
+ "loss": 0.1592,
111
  "rewards/accuracies": 0.699999988079071,
112
+ "rewards/chosen": -0.03646404668688774,
113
+ "rewards/margins": 0.11744984239339828,
114
+ "rewards/rejected": -0.15391388535499573,
115
  "step": 60
116
  },
117
  {
118
  "epoch": 0.16055045871559634,
119
+ "grad_norm": 7.653772817525038,
120
  "learning_rate": 4.945923025551788e-07,
121
+ "logits/chosen": -2.4572176933288574,
122
+ "logits/rejected": -2.407254219055176,
123
+ "logps/chosen": -327.6614685058594,
124
+ "logps/rejected": -284.7535705566406,
125
+ "loss": 0.1557,
126
  "rewards/accuracies": 0.6937500238418579,
127
+ "rewards/chosen": -0.0009046800551004708,
128
+ "rewards/margins": 0.23255476355552673,
129
+ "rewards/rejected": -0.23345942795276642,
130
  "step": 70
131
  },
132
  {
133
  "epoch": 0.1834862385321101,
134
+ "grad_norm": 8.127224192262021,
135
  "learning_rate": 4.896669632591651e-07,
136
+ "logits/chosen": -2.3246560096740723,
137
+ "logits/rejected": -2.2035281658172607,
138
+ "logps/chosen": -297.11651611328125,
139
+ "logps/rejected": -290.16302490234375,
140
+ "loss": 0.1475,
141
+ "rewards/accuracies": 0.7437499761581421,
142
+ "rewards/chosen": -0.11266676336526871,
143
+ "rewards/margins": 0.2624030113220215,
144
+ "rewards/rejected": -0.3750697076320648,
145
  "step": 80
146
  },
147
  {
148
  "epoch": 0.20642201834862386,
149
+ "grad_norm": 8.777001707426516,
150
  "learning_rate": 4.832031033425662e-07,
151
+ "logits/chosen": -1.9738223552703857,
152
+ "logits/rejected": -1.9115241765975952,
153
+ "logps/chosen": -299.46820068359375,
154
+ "logps/rejected": -291.00994873046875,
155
+ "loss": 0.1449,
156
+ "rewards/accuracies": 0.71875,
157
+ "rewards/chosen": -0.11874761432409286,
158
+ "rewards/margins": 0.3233392536640167,
159
+ "rewards/rejected": -0.4420868456363678,
160
  "step": 90
161
  },
162
  {
163
  "epoch": 0.22935779816513763,
164
+ "grad_norm": 9.678897311244762,
165
  "learning_rate": 4.752422169756047e-07,
166
+ "logits/chosen": -1.0515625476837158,
167
+ "logits/rejected": -1.0063254833221436,
168
+ "logps/chosen": -311.99945068359375,
169
+ "logps/rejected": -342.3149719238281,
170
  "loss": 0.1412,
171
+ "rewards/accuracies": 0.6875,
172
+ "rewards/chosen": -0.39574748277664185,
173
+ "rewards/margins": 0.3081180453300476,
174
+ "rewards/rejected": -0.7038655281066895,
175
  "step": 100
176
  },
177
  {
178
  "epoch": 0.22935779816513763,
179
+ "eval_logits/chosen": -1.1028852462768555,
180
+ "eval_logits/rejected": -0.8337642550468445,
181
+ "eval_logps/chosen": -341.0970458984375,
182
+ "eval_logps/rejected": -340.4026794433594,
183
+ "eval_loss": 0.13921599090099335,
184
+ "eval_rewards/accuracies": 0.7198275923728943,
185
+ "eval_rewards/chosen": -0.5600696802139282,
186
+ "eval_rewards/margins": 0.3787323534488678,
187
+ "eval_rewards/rejected": -0.9388020038604736,
188
+ "eval_runtime": 95.7518,
189
+ "eval_samples_per_second": 18.987,
190
+ "eval_steps_per_second": 0.303,
191
  "step": 100
192
  },
193
  {
194
  "epoch": 0.25229357798165136,
195
+ "grad_norm": 8.075119669591446,
196
  "learning_rate": 4.658354083558188e-07,
197
+ "logits/chosen": -1.309912085533142,
198
+ "logits/rejected": -1.0928936004638672,
199
+ "logps/chosen": -323.1109313964844,
200
+ "logps/rejected": -320.5851135253906,
201
+ "loss": 0.1371,
202
+ "rewards/accuracies": 0.71875,
203
+ "rewards/chosen": -0.5792280435562134,
204
+ "rewards/margins": 0.2892912030220032,
205
+ "rewards/rejected": -0.868519127368927,
206
  "step": 110
207
  },
208
  {
209
  "epoch": 0.27522935779816515,
210
+ "grad_norm": 11.527192184915142,
211
  "learning_rate": 4.550430636492389e-07,
212
+ "logits/chosen": -1.2554824352264404,
213
+ "logits/rejected": -1.2111155986785889,
214
+ "logps/chosen": -315.65582275390625,
215
+ "logps/rejected": -323.9941101074219,
216
+ "loss": 0.1347,
217
+ "rewards/accuracies": 0.6812499761581421,
218
+ "rewards/chosen": -0.39836472272872925,
219
+ "rewards/margins": 0.3269372582435608,
220
+ "rewards/rejected": -0.7253020405769348,
221
  "step": 120
222
  },
223
  {
224
  "epoch": 0.2981651376146789,
225
+ "grad_norm": 10.253713539855786,
226
  "learning_rate": 4.429344633468004e-07,
227
+ "logits/chosen": -0.9448979496955872,
228
+ "logits/rejected": -0.6800575256347656,
229
+ "logps/chosen": -294.2066345214844,
230
+ "logps/rejected": -319.7179870605469,
231
+ "loss": 0.1367,
232
+ "rewards/accuracies": 0.699999988079071,
233
+ "rewards/chosen": -0.38026565313339233,
234
+ "rewards/margins": 0.39078813791275024,
235
+ "rewards/rejected": -0.7710537910461426,
236
  "step": 130
237
  },
238
  {
239
  "epoch": 0.3211009174311927,
240
+ "grad_norm": 11.166916308293251,
241
  "learning_rate": 4.2958733752443187e-07,
242
+ "logits/chosen": -0.8638193011283875,
243
+ "logits/rejected": -0.5810004472732544,
244
+ "logps/chosen": -307.26165771484375,
245
+ "logps/rejected": -287.42547607421875,
246
+ "loss": 0.134,
247
+ "rewards/accuracies": 0.706250011920929,
248
+ "rewards/chosen": -0.3107650578022003,
249
+ "rewards/margins": 0.3205246925354004,
250
+ "rewards/rejected": -0.6312897801399231,
251
  "step": 140
252
  },
253
  {
254
  "epoch": 0.3440366972477064,
255
+ "grad_norm": 10.491465778765697,
256
  "learning_rate": 4.150873668617898e-07,
257
+ "logits/chosen": -0.785676121711731,
258
+ "logits/rejected": -0.4156356453895569,
259
+ "logps/chosen": -306.7540588378906,
260
+ "logps/rejected": -312.4991149902344,
261
+ "loss": 0.1327,
262
+ "rewards/accuracies": 0.71875,
263
+ "rewards/chosen": -0.40210476517677307,
264
+ "rewards/margins": 0.39101067185401917,
265
+ "rewards/rejected": -0.793115496635437,
266
  "step": 150
267
  },
268
  {
269
  "epoch": 0.3440366972477064,
270
+ "eval_logits/chosen": -0.934603214263916,
271
+ "eval_logits/rejected": -0.4213140308856964,
272
+ "eval_logps/chosen": -321.6920166015625,
273
+ "eval_logps/rejected": -333.3775939941406,
274
+ "eval_loss": 0.1256996989250183,
275
+ "eval_rewards/accuracies": 0.7241379022598267,
276
+ "eval_rewards/chosen": -0.36601918935775757,
277
+ "eval_rewards/margins": 0.5025321841239929,
278
+ "eval_rewards/rejected": -0.8685513734817505,
279
+ "eval_runtime": 95.7427,
280
+ "eval_samples_per_second": 18.988,
281
+ "eval_steps_per_second": 0.303,
282
  "step": 150
283
  },
284
  {
285
  "epoch": 0.3669724770642202,
286
+ "grad_norm": 10.606572226202415,
287
  "learning_rate": 3.9952763262280397e-07,
288
+ "logits/chosen": -0.6937485933303833,
289
+ "logits/rejected": -0.39156150817871094,
290
+ "logps/chosen": -341.2958068847656,
291
+ "logps/rejected": -376.64788818359375,
292
+ "loss": 0.1226,
293
+ "rewards/accuracies": 0.737500011920929,
294
+ "rewards/chosen": -0.4474472105503082,
295
+ "rewards/margins": 0.4593452513217926,
296
+ "rewards/rejected": -0.9067924618721008,
297
  "step": 160
298
  },
299
  {
300
  "epoch": 0.38990825688073394,
301
+ "grad_norm": 7.909933445936232,
302
  "learning_rate": 3.8300801912883414e-07,
303
+ "logits/chosen": -0.5848881602287292,
304
+ "logits/rejected": -0.295288622379303,
305
+ "logps/chosen": -314.4745178222656,
306
+ "logps/rejected": -355.83209228515625,
307
+ "loss": 0.1194,
308
  "rewards/accuracies": 0.699999988079071,
309
+ "rewards/chosen": -0.42873620986938477,
310
+ "rewards/margins": 0.4288958013057709,
311
+ "rewards/rejected": -0.857632040977478,
312
  "step": 170
313
  },
314
  {
315
  "epoch": 0.41284403669724773,
316
+ "grad_norm": 9.296204512324422,
317
  "learning_rate": 3.6563457256020884e-07,
318
+ "logits/chosen": -0.4405382573604584,
319
+ "logits/rejected": 0.17199954390525818,
320
+ "logps/chosen": -351.5772399902344,
321
+ "logps/rejected": -332.4178771972656,
322
+ "loss": 0.1256,
323
+ "rewards/accuracies": 0.7250000238418579,
324
+ "rewards/chosen": -0.48219355940818787,
325
+ "rewards/margins": 0.4570356011390686,
326
+ "rewards/rejected": -0.9392291903495789,
327
  "step": 180
328
  },
329
  {
330
  "epoch": 0.43577981651376146,
331
+ "grad_norm": 9.788859469537575,
332
  "learning_rate": 3.475188202022617e-07,
333
+ "logits/chosen": -0.9616500735282898,
334
+ "logits/rejected": -0.5780771970748901,
335
+ "logps/chosen": -284.89788818359375,
336
+ "logps/rejected": -339.87139892578125,
337
+ "loss": 0.124,
338
  "rewards/accuracies": 0.7749999761581421,
339
+ "rewards/chosen": -0.2678338885307312,
340
+ "rewards/margins": 0.487435519695282,
341
+ "rewards/rejected": -0.7552694082260132,
342
  "step": 190
343
  },
344
  {
345
  "epoch": 0.45871559633027525,
346
+ "grad_norm": 8.31792118291438,
347
  "learning_rate": 3.287770545059052e-07,
348
+ "logits/chosen": -0.9191502332687378,
349
+ "logits/rejected": -0.3312894105911255,
350
+ "logps/chosen": -313.3896179199219,
351
+ "logps/rejected": -324.57000732421875,
352
+ "loss": 0.1242,
353
+ "rewards/accuracies": 0.768750011920929,
354
+ "rewards/chosen": -0.3510800302028656,
355
+ "rewards/margins": 0.4560603201389313,
356
+ "rewards/rejected": -0.8071403503417969,
357
  "step": 200
358
  },
359
  {
360
  "epoch": 0.45871559633027525,
361
+ "eval_logits/chosen": -0.9084411859512329,
362
+ "eval_logits/rejected": -0.1497526615858078,
363
+ "eval_logps/chosen": -319.3252258300781,
364
+ "eval_logps/rejected": -331.5806884765625,
365
+ "eval_loss": 0.11925708502531052,
366
+ "eval_rewards/accuracies": 0.7456896305084229,
367
+ "eval_rewards/chosen": -0.34235113859176636,
368
+ "eval_rewards/margins": 0.5082312226295471,
369
+ "eval_rewards/rejected": -0.8505823612213135,
370
+ "eval_runtime": 93.7494,
371
+ "eval_samples_per_second": 19.392,
372
+ "eval_steps_per_second": 0.309,
373
  "step": 200
374
  },
375
  {
376
  "epoch": 0.481651376146789,
377
+ "grad_norm": 7.584339731807936,
378
  "learning_rate": 3.0952958655864954e-07,
379
+ "logits/chosen": -0.6695399284362793,
380
+ "logits/rejected": -0.2294241189956665,
381
+ "logps/chosen": -311.20343017578125,
382
+ "logps/rejected": -338.2936096191406,
383
+ "loss": 0.1173,
384
+ "rewards/accuracies": 0.762499988079071,
385
+ "rewards/chosen": -0.3789902925491333,
386
+ "rewards/margins": 0.4605233073234558,
387
+ "rewards/rejected": -0.8395135998725891,
388
  "step": 210
389
  },
390
  {
391
  "epoch": 0.5045871559633027,
392
+ "grad_norm": 10.1245037946263,
393
  "learning_rate": 2.898999737583448e-07,
394
+ "logits/chosen": -0.6778780221939087,
395
+ "logits/rejected": 0.15124213695526123,
396
+ "logps/chosen": -365.30010986328125,
397
+ "logps/rejected": -385.3759460449219,
398
+ "loss": 0.1146,
399
+ "rewards/accuracies": 0.793749988079071,
400
+ "rewards/chosen": -0.42619770765304565,
401
+ "rewards/margins": 0.5778595209121704,
402
+ "rewards/rejected": -1.0040571689605713,
403
  "step": 220
404
  },
405
  {
406
  "epoch": 0.5275229357798165,
407
+ "grad_norm": 8.611834221562093,
408
  "learning_rate": 2.7001422664752333e-07,
409
+ "logits/chosen": -0.5495241284370422,
410
+ "logits/rejected": -0.10778895765542984,
411
+ "logps/chosen": -308.42864990234375,
412
+ "logps/rejected": -350.29937744140625,
413
+ "loss": 0.1211,
414
+ "rewards/accuracies": 0.737500011920929,
415
+ "rewards/chosen": -0.4048517346382141,
416
+ "rewards/margins": 0.4419426918029785,
417
+ "rewards/rejected": -0.8467944264411926,
418
  "step": 230
419
  },
420
  {
421
  "epoch": 0.5504587155963303,
422
+ "grad_norm": 8.84744756928296,
423
  "learning_rate": 2.5e-07,
424
+ "logits/chosen": -0.8606957197189331,
425
+ "logits/rejected": -0.25019243359565735,
426
+ "logps/chosen": -323.67840576171875,
427
+ "logps/rejected": -341.2455749511719,
428
+ "loss": 0.1198,
429
  "rewards/accuracies": 0.737500011920929,
430
+ "rewards/chosen": -0.3614657521247864,
431
+ "rewards/margins": 0.4374900758266449,
432
+ "rewards/rejected": -0.7989557981491089,
433
  "step": 240
434
  },
435
  {
436
  "epoch": 0.573394495412844,
437
+ "grad_norm": 8.762072273295908,
438
  "learning_rate": 2.2998577335247667e-07,
439
+ "logits/chosen": -0.8679742813110352,
440
+ "logits/rejected": -0.18818725645542145,
441
+ "logps/chosen": -340.2317810058594,
442
+ "logps/rejected": -349.81414794921875,
443
+ "loss": 0.118,
444
+ "rewards/accuracies": 0.78125,
445
+ "rewards/chosen": -0.3643796145915985,
446
+ "rewards/margins": 0.5293234586715698,
447
+ "rewards/rejected": -0.8937031030654907,
448
  "step": 250
449
  },
450
  {
451
  "epoch": 0.573394495412844,
452
+ "eval_logits/chosen": -0.8147245645523071,
453
+ "eval_logits/rejected": -0.025059914216399193,
454
+ "eval_logps/chosen": -322.4342041015625,
455
+ "eval_logps/rejected": -343.3682556152344,
456
+ "eval_loss": 0.11674168705940247,
457
  "eval_rewards/accuracies": 0.7543103694915771,
458
+ "eval_rewards/chosen": -0.3734411895275116,
459
+ "eval_rewards/margins": 0.5950165390968323,
460
+ "eval_rewards/rejected": -0.9684576988220215,
461
+ "eval_runtime": 95.6284,
462
+ "eval_samples_per_second": 19.011,
463
+ "eval_steps_per_second": 0.303,
464
  "step": 250
465
  },
466
  {
467
  "epoch": 0.5963302752293578,
468
+ "grad_norm": 8.055173398595795,
469
  "learning_rate": 2.1010002624165524e-07,
470
+ "logits/chosen": -0.5792064666748047,
471
+ "logits/rejected": -0.028709029778838158,
472
+ "logps/chosen": -319.5377197265625,
473
+ "logps/rejected": -383.94610595703125,
474
+ "loss": 0.1186,
475
+ "rewards/accuracies": 0.78125,
476
+ "rewards/chosen": -0.4882320463657379,
477
+ "rewards/margins": 0.6072918176651001,
478
+ "rewards/rejected": -1.0955239534378052,
479
  "step": 260
480
  },
481
  {
482
  "epoch": 0.6192660550458715,
483
+ "grad_norm": 9.235774055264796,
484
  "learning_rate": 1.9047041344135043e-07,
485
+ "logits/chosen": -0.37728264927864075,
486
+ "logits/rejected": 0.017757948487997055,
487
+ "logps/chosen": -318.92266845703125,
488
+ "logps/rejected": -360.00250244140625,
489
+ "loss": 0.1169,
490
+ "rewards/accuracies": 0.8062499761581421,
491
+ "rewards/chosen": -0.5286036729812622,
492
+ "rewards/margins": 0.5529505014419556,
493
+ "rewards/rejected": -1.0815541744232178,
494
  "step": 270
495
  },
496
  {
497
  "epoch": 0.6422018348623854,
498
+ "grad_norm": 9.286061693337006,
499
  "learning_rate": 1.7122294549409482e-07,
500
+ "logits/chosen": -0.7753764986991882,
501
+ "logits/rejected": -0.038079630583524704,
502
+ "logps/chosen": -314.3120422363281,
503
+ "logps/rejected": -365.4776611328125,
504
+ "loss": 0.1216,
505
+ "rewards/accuracies": 0.78125,
506
+ "rewards/chosen": -0.4131316542625427,
507
+ "rewards/margins": 0.6036869883537292,
508
+ "rewards/rejected": -1.016818642616272,
509
  "step": 280
510
  },
511
  {
512
  "epoch": 0.6651376146788991,
513
+ "grad_norm": 10.391059671359317,
514
  "learning_rate": 1.524811797977383e-07,
515
+ "logits/chosen": -0.6743412017822266,
516
+ "logits/rejected": -0.05127597600221634,
517
+ "logps/chosen": -326.2575378417969,
518
+ "logps/rejected": -345.9309387207031,
519
+ "loss": 0.1144,
520
+ "rewards/accuracies": 0.737500011920929,
521
+ "rewards/chosen": -0.3852667212486267,
522
+ "rewards/margins": 0.5348750352859497,
523
+ "rewards/rejected": -0.9201416969299316,
524
  "step": 290
525
  },
526
  {
527
  "epoch": 0.6880733944954128,
528
+ "grad_norm": 11.142166437672913,
529
  "learning_rate": 1.3436542743979125e-07,
530
+ "logits/chosen": -0.6289922595024109,
531
+ "logits/rejected": -0.2742362916469574,
532
+ "logps/chosen": -350.96099853515625,
533
+ "logps/rejected": -345.9538879394531,
534
+ "loss": 0.1236,
535
+ "rewards/accuracies": 0.731249988079071,
536
+ "rewards/chosen": -0.45011693239212036,
537
+ "rewards/margins": 0.42082324624061584,
538
+ "rewards/rejected": -0.8709400296211243,
539
  "step": 300
540
  },
541
  {
542
  "epoch": 0.6880733944954128,
543
+ "eval_logits/chosen": -0.5284780263900757,
544
+ "eval_logits/rejected": 0.2890760600566864,
545
+ "eval_logps/chosen": -332.8935546875,
546
+ "eval_logps/rejected": -351.01617431640625,
547
+ "eval_loss": 0.11446455121040344,
548
+ "eval_rewards/accuracies": 0.7629310488700867,
549
+ "eval_rewards/chosen": -0.4780345857143402,
550
+ "eval_rewards/margins": 0.5669029355049133,
551
+ "eval_rewards/rejected": -1.0449376106262207,
552
+ "eval_runtime": 94.2008,
553
+ "eval_samples_per_second": 19.299,
554
+ "eval_steps_per_second": 0.308,
555
  "step": 300
556
  },
557
  {
558
  "epoch": 0.7110091743119266,
559
+ "grad_norm": 9.249175988688211,
560
  "learning_rate": 1.1699198087116588e-07,
561
+ "logits/chosen": -0.5085445642471313,
562
+ "logits/rejected": 0.12936155498027802,
563
+ "logps/chosen": -332.48468017578125,
564
+ "logps/rejected": -366.32489013671875,
565
+ "loss": 0.1229,
566
+ "rewards/accuracies": 0.75,
567
+ "rewards/chosen": -0.5451949238777161,
568
+ "rewards/margins": 0.4535568654537201,
569
+ "rewards/rejected": -0.9987518191337585,
570
  "step": 310
571
  },
572
  {
573
  "epoch": 0.7339449541284404,
574
+ "grad_norm": 9.319929398035963,
575
  "learning_rate": 1.00472367377196e-07,
576
+ "logits/chosen": -0.37469834089279175,
577
+ "logits/rejected": 0.34990087151527405,
578
+ "logps/chosen": -323.309326171875,
579
+ "logps/rejected": -343.5829772949219,
580
+ "loss": 0.1184,
581
  "rewards/accuracies": 0.78125,
582
+ "rewards/chosen": -0.49077337980270386,
583
+ "rewards/margins": 0.6001543402671814,
584
+ "rewards/rejected": -1.0909278392791748,
585
  "step": 320
586
  },
587
  {
588
  "epoch": 0.7568807339449541,
589
+ "grad_norm": 10.87259656283017,
590
  "learning_rate": 8.49126331382102e-08,
591
+ "logits/chosen": -0.20644304156303406,
592
+ "logits/rejected": 0.2470076084136963,
593
+ "logps/chosen": -329.26171875,
594
+ "logps/rejected": -346.26666259765625,
595
+ "loss": 0.1182,
596
+ "rewards/accuracies": 0.7250000238418579,
597
+ "rewards/chosen": -0.5473949909210205,
598
+ "rewards/margins": 0.44427090883255005,
599
+ "rewards/rejected": -0.9916658401489258,
600
  "step": 330
601
  },
602
  {
603
  "epoch": 0.7798165137614679,
604
+ "grad_norm": 11.502524720737265,
605
  "learning_rate": 7.041266247556812e-08,
606
+ "logits/chosen": -0.511077344417572,
607
+ "logits/rejected": -0.07181791216135025,
608
+ "logps/chosen": -338.0530700683594,
609
+ "logps/rejected": -354.2392578125,
610
+ "loss": 0.1129,
611
+ "rewards/accuracies": 0.78125,
612
+ "rewards/chosen": -0.5006048679351807,
613
+ "rewards/margins": 0.4419478476047516,
614
+ "rewards/rejected": -0.9425527453422546,
615
  "step": 340
616
  },
617
  {
618
  "epoch": 0.8027522935779816,
619
+ "grad_norm": 10.417951906388708,
620
  "learning_rate": 5.706553665319955e-08,
621
+ "logits/chosen": -0.4384083151817322,
622
+ "logits/rejected": 0.22936472296714783,
623
+ "logps/chosen": -327.1330261230469,
624
+ "logps/rejected": -339.122314453125,
625
+ "loss": 0.122,
626
+ "rewards/accuracies": 0.762499988079071,
627
+ "rewards/chosen": -0.526419460773468,
628
+ "rewards/margins": 0.48818930983543396,
629
+ "rewards/rejected": -1.0146087408065796,
630
  "step": 350
631
  },
632
  {
633
  "epoch": 0.8027522935779816,
634
+ "eval_logits/chosen": -0.48056575655937195,
635
+ "eval_logits/rejected": 0.33971619606018066,
636
+ "eval_logps/chosen": -330.8570251464844,
637
+ "eval_logps/rejected": -350.28302001953125,
638
+ "eval_loss": 0.11267489194869995,
639
+ "eval_rewards/accuracies": 0.767241358757019,
640
+ "eval_rewards/chosen": -0.45766934752464294,
641
+ "eval_rewards/margins": 0.5799364447593689,
642
+ "eval_rewards/rejected": -1.0376057624816895,
643
+ "eval_runtime": 94.5966,
644
+ "eval_samples_per_second": 19.218,
645
+ "eval_steps_per_second": 0.307,
646
  "step": 350
647
  },
648
  {
649
  "epoch": 0.8256880733944955,
650
+ "grad_norm": 8.120253834632708,
651
  "learning_rate": 4.4956936350761005e-08,
652
+ "logits/chosen": -0.412276029586792,
653
+ "logits/rejected": 0.03293975815176964,
654
+ "logps/chosen": -295.88397216796875,
655
+ "logps/rejected": -349.67828369140625,
656
+ "loss": 0.1126,
657
+ "rewards/accuracies": 0.706250011920929,
658
+ "rewards/chosen": -0.43563956022262573,
659
+ "rewards/margins": 0.47577181458473206,
660
+ "rewards/rejected": -0.9114114046096802,
661
  "step": 360
662
  },
663
  {
664
  "epoch": 0.8486238532110092,
665
+ "grad_norm": 8.937083414679394,
666
  "learning_rate": 3.416459164418123e-08,
667
+ "logits/chosen": -0.7006546258926392,
668
+ "logits/rejected": -0.1087837815284729,
669
+ "logps/chosen": -344.0120544433594,
670
+ "logps/rejected": -357.1898498535156,
671
+ "loss": 0.1124,
672
+ "rewards/accuracies": 0.737500011920929,
673
+ "rewards/chosen": -0.4442760944366455,
674
+ "rewards/margins": 0.5018762350082397,
675
+ "rewards/rejected": -0.9461522102355957,
676
  "step": 370
677
  },
678
  {
679
  "epoch": 0.8715596330275229,
680
+ "grad_norm": 9.250415689561605,
681
  "learning_rate": 2.475778302439524e-08,
682
+ "logits/chosen": -0.6538180112838745,
683
+ "logits/rejected": 0.11514680087566376,
684
+ "logps/chosen": -334.7962951660156,
685
+ "logps/rejected": -353.5735778808594,
686
+ "loss": 0.1165,
687
  "rewards/accuracies": 0.7749999761581421,
688
+ "rewards/chosen": -0.3841826617717743,
689
+ "rewards/margins": 0.5749909281730652,
690
+ "rewards/rejected": -0.9591735601425171,
691
  "step": 380
692
  },
693
  {
694
  "epoch": 0.8944954128440367,
695
+ "grad_norm": 9.532136069213038,
696
  "learning_rate": 1.6796896657433805e-08,
697
+ "logits/chosen": -0.4097678065299988,
698
+ "logits/rejected": 0.13332059979438782,
699
+ "logps/chosen": -294.68280029296875,
700
+ "logps/rejected": -319.02825927734375,
701
+ "loss": 0.122,
702
+ "rewards/accuracies": 0.7437499761581421,
703
+ "rewards/chosen": -0.4272535443305969,
704
+ "rewards/margins": 0.4699665606021881,
705
+ "rewards/rejected": -0.8972201347351074,
706
  "step": 390
707
  },
708
  {
709
  "epoch": 0.9174311926605505,
710
+ "grad_norm": 9.826819983657527,
711
  "learning_rate": 1.0333036740834855e-08,
712
+ "logits/chosen": -0.3362836241722107,
713
+ "logits/rejected": 0.09532846510410309,
714
+ "logps/chosen": -262.51324462890625,
715
+ "logps/rejected": -315.52349853515625,
716
  "loss": 0.12,
717
+ "rewards/accuracies": 0.737500011920929,
718
+ "rewards/chosen": -0.37953639030456543,
719
+ "rewards/margins": 0.48354291915893555,
720
+ "rewards/rejected": -0.8630792498588562,
721
  "step": 400
722
  },
723
  {
724
  "epoch": 0.9174311926605505,
725
+ "eval_logits/chosen": -0.6159119606018066,
726
+ "eval_logits/rejected": 0.19038529694080353,
727
+ "eval_logps/chosen": -325.2745361328125,
728
+ "eval_logps/rejected": -342.9399108886719,
729
+ "eval_loss": 0.11256735771894455,
730
+ "eval_rewards/accuracies": 0.75,
731
+ "eval_rewards/chosen": -0.40184491872787476,
732
+ "eval_rewards/margins": 0.5623298287391663,
733
+ "eval_rewards/rejected": -0.9641746282577515,
734
+ "eval_runtime": 93.497,
735
+ "eval_samples_per_second": 19.444,
736
+ "eval_steps_per_second": 0.31,
737
  "step": 400
738
  },
739
  {
740
  "epoch": 0.9403669724770642,
741
+ "grad_norm": 9.708164546519836,
742
  "learning_rate": 5.4076974448211685e-09,
743
+ "logits/chosen": -0.34120887517929077,
744
+ "logits/rejected": 0.403463214635849,
745
+ "logps/chosen": -314.7442321777344,
746
+ "logps/rejected": -337.3968811035156,
747
+ "loss": 0.1168,
748
+ "rewards/accuracies": 0.7437499761581421,
749
+ "rewards/chosen": -0.43792352080345154,
750
+ "rewards/margins": 0.5555781126022339,
751
+ "rewards/rejected": -0.9935015439987183,
752
  "step": 410
753
  },
754
  {
755
  "epoch": 0.963302752293578,
756
+ "grad_norm": 10.889878191306106,
757
  "learning_rate": 2.052496544188487e-09,
758
+ "logits/chosen": -0.31919345259666443,
759
+ "logits/rejected": 0.3534119725227356,
760
+ "logps/chosen": -306.3693542480469,
761
+ "logps/rejected": -349.92462158203125,
762
+ "loss": 0.1147,
763
+ "rewards/accuracies": 0.768750011920929,
764
+ "rewards/chosen": -0.4693039059638977,
765
+ "rewards/margins": 0.5711686015129089,
766
+ "rewards/rejected": -1.040472388267517,
767
  "step": 420
768
  },
769
  {
770
  "epoch": 0.9862385321100917,
771
+ "grad_norm": 9.153719920278654,
772
  "learning_rate": 2.889724508297886e-10,
773
+ "logits/chosen": -0.556784451007843,
774
+ "logits/rejected": 0.2818067669868469,
775
+ "logps/chosen": -345.05450439453125,
776
+ "logps/rejected": -335.64605712890625,
777
+ "loss": 0.1155,
778
+ "rewards/accuracies": 0.7749999761581421,
779
+ "rewards/chosen": -0.41879063844680786,
780
+ "rewards/margins": 0.49730420112609863,
781
+ "rewards/rejected": -0.9160947799682617,
782
  "step": 430
783
  },
784
  {
785
  "epoch": 1.0,
786
  "step": 436,
787
  "total_flos": 0.0,
788
+ "train_loss": 0.13197535372108493,
789
+ "train_runtime": 11523.0815,
790
+ "train_samples_per_second": 4.839,
791
  "train_steps_per_second": 0.038
792
  }
793
  ],
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5b97c68988ee761051dc175f846465ecfe6423242be1592fe1880198a31410e6
3
  size 7608
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d6fef0b25cff78175dd3905d73113b24ac1239ca17c5ec75144eef77b24b3ac
3
  size 7608