Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wzhouad commited on
Commit
ba49b4e
•
1 Parent(s): d282f28

Model save

Browse files
Files changed (9) hide show
  1. README.md +18 -18
  2. all_results.json +4 -4
  3. model-00001-of-00004.safetensors +1 -1
  4. model-00002-of-00004.safetensors +1 -1
  5. model-00003-of-00004.safetensors +1 -1
  6. model-00004-of-00004.safetensors +1 -1
  7. train_results.json +4 -4
  8. trainer_state.json +964 -964
  9. training_args.bin +1 -1
README.md CHANGED
@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
- - Loss: 0.0216
19
- - Rewards/chosen: -2.0018
20
- - Rewards/rejected: -3.2655
21
- - Rewards/accuracies: 0.7344
22
- - Rewards/margins: 1.2637
23
- - Logps/rejected: -638.1763
24
- - Logps/chosen: -503.7670
25
- - Logits/rejected: 0.7139
26
- - Logits/chosen: 0.6104
27
 
28
  ## Model description
29
 
@@ -60,15 +60,15 @@ The following hyperparameters were used during training:
60
 
61
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
62
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
63
- | 0.1135 | 0.21 | 100 | 0.1060 | -0.4112 | -0.7633 | 0.7070 | 0.3521 | -387.9543 | -344.7048 | 0.5669 | 0.5124 |
64
- | 0.0648 | 0.42 | 200 | 0.0664 | -0.6849 | -1.3101 | 0.7188 | 0.6252 | -442.6332 | -372.0718 | 0.3867 | 0.3105 |
65
- | 0.0565 | 0.63 | 300 | 0.0585 | -0.7598 | -1.4962 | 0.7305 | 0.7364 | -461.2486 | -379.5646 | 0.3625 | 0.2834 |
66
- | 0.0643 | 0.84 | 400 | 0.0623 | -0.6660 | -1.4002 | 0.7305 | 0.7341 | -451.6428 | -370.1878 | 0.2607 | 0.1862 |
67
- | 0.0214 | 1.05 | 500 | 0.0310 | -1.5688 | -2.7542 | 0.7422 | 1.1854 | -587.0469 | -460.4668 | 0.5000 | 0.4214 |
68
- | 0.0145 | 1.26 | 600 | 0.0245 | -1.7918 | -3.0398 | 0.7344 | 1.2481 | -615.6083 | -482.7609 | 0.6354 | 0.5416 |
69
- | 0.0098 | 1.47 | 700 | 0.0210 | -2.0363 | -3.3488 | 0.7305 | 1.3125 | -646.5033 | -507.2101 | 0.6871 | 0.5851 |
70
- | 0.0101 | 1.67 | 800 | 0.0218 | -1.9924 | -3.2641 | 0.7344 | 1.2718 | -638.0398 | -502.8234 | 0.6981 | 0.5937 |
71
- | 0.0103 | 1.88 | 900 | 0.0216 | -2.0018 | -3.2655 | 0.7344 | 1.2637 | -638.1763 | -503.7670 | 0.7139 | 0.6104 |
72
 
73
 
74
  ### Framework versions
 
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
+ - Loss: 0.5261
19
+ - Rewards/chosen: -2.4591
20
+ - Rewards/rejected: -3.9221
21
+ - Rewards/accuracies: 0.7773
22
+ - Rewards/margins: 1.4631
23
+ - Logps/rejected: -703.8400
24
+ - Logps/chosen: -549.4910
25
+ - Logits/rejected: 0.0289
26
+ - Logits/chosen: 0.0663
27
 
28
  ## Model description
29
 
 
60
 
61
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
62
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
63
+ | 0.6201 | 0.21 | 100 | 0.6253 | -0.2753 | -0.6662 | 0.7031 | 0.3909 | -378.2405 | -331.1124 | 0.4172 | 0.3706 |
64
+ | 0.5547 | 0.42 | 200 | 0.5549 | -0.6988 | -1.4726 | 0.7656 | 0.7738 | -458.8863 | -373.4661 | 0.4261 | 0.3909 |
65
+ | 0.5343 | 0.63 | 300 | 0.5316 | -0.8044 | -1.6474 | 0.7656 | 0.8430 | -476.3628 | -384.0199 | 0.2851 | 0.2449 |
66
+ | 0.5323 | 0.84 | 400 | 0.5211 | -0.9068 | -1.8283 | 0.7812 | 0.9216 | -494.4600 | -394.2621 | 0.2834 | 0.2514 |
67
+ | 0.352 | 1.05 | 500 | 0.5258 | -1.9533 | -3.4166 | 0.7969 | 1.4634 | -653.2899 | -498.9117 | -0.0846 | -0.0654 |
68
+ | 0.3342 | 1.26 | 600 | 0.5268 | -2.3123 | -3.7246 | 0.7930 | 1.4124 | -684.0857 | -534.8101 | 0.1128 | 0.1344 |
69
+ | 0.337 | 1.47 | 700 | 0.5290 | -2.3753 | -3.8837 | 0.7773 | 1.5084 | -699.9910 | -541.1116 | 0.0099 | 0.0414 |
70
+ | 0.3398 | 1.67 | 800 | 0.5297 | -2.5097 | -4.0133 | 0.7734 | 1.5036 | -712.9506 | -554.5546 | 0.0381 | 0.0750 |
71
+ | 0.307 | 1.88 | 900 | 0.5261 | -2.4591 | -3.9221 | 0.7773 | 1.4631 | -703.8400 | -549.4910 | 0.0289 | 0.0663 |
72
 
73
 
74
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 2.0,
3
- "train_loss": 0.048426837674039463,
4
- "train_runtime": 8860.0868,
5
  "train_samples": 61134,
6
- "train_samples_per_second": 13.8,
7
- "train_steps_per_second": 0.108
8
  }
 
1
  {
2
  "epoch": 2.0,
3
+ "train_loss": 0.44779854000739333,
4
+ "train_runtime": 8782.9823,
5
  "train_samples": 61134,
6
+ "train_samples_per_second": 13.921,
7
+ "train_steps_per_second": 0.109
8
  }
model-00001-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1bbf762dfc95b30382f7d12aa322403420f2fedeb1f40de707e07a41b4b65a74
3
  size 4976698672
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac6928598b87aa9e7ace3c1f94d7df4eaa7164f92df0fda9603ca217542e9949
3
  size 4976698672
model-00002-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8b360ad3bfe2c13ef13d8ec3499feb010e9e772efbcd73344a4fa33c853c08df
3
  size 4999802720
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1832fd3b8ecfcb01bbcfbb4425fba1dbd3c57767b6fe8fe806acc6e01654a97b
3
  size 4999802720
model-00003-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:611e09fdf80d346d40d4790f93b01096529390ec9de8882089baa707b39f1a31
3
  size 4915916176
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a888c6fcef84f2cb517c57068441aa0a6ba00c8bfc4bd4c7155e79556f8133b9
3
  size 4915916176
model-00004-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c34cb68b36e33fe3b029cc36b78c1729aa55f897d89fb8a7def6c059f706b100
3
  size 1168138808
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c2d194b9549d875e2494134696a27103429d2e21501246a202a7415a40b5de72
3
  size 1168138808
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 2.0,
3
- "train_loss": 0.048426837674039463,
4
- "train_runtime": 8860.0868,
5
  "train_samples": 61134,
6
- "train_samples_per_second": 13.8,
7
- "train_steps_per_second": 0.108
8
  }
 
1
  {
2
  "epoch": 2.0,
3
+ "train_loss": 0.44779854000739333,
4
+ "train_runtime": 8782.9823,
5
  "train_samples": 61134,
6
+ "train_samples_per_second": 13.921,
7
+ "train_steps_per_second": 0.109
8
  }
trainer_state.json CHANGED
@@ -11,1485 +11,1485 @@
11
  {
12
  "epoch": 0.02,
13
  "learning_rate": 1.0416666666666667e-07,
14
- "logits/chosen": 0.294111967086792,
15
- "logits/rejected": 0.3186231553554535,
16
- "logps/chosen": -295.23431396484375,
17
- "logps/rejected": -290.9120178222656,
18
- "loss": 0.1859,
19
- "rewards/accuracies": 0.44999998807907104,
20
- "rewards/chosen": 6.947966903680936e-05,
21
- "rewards/margins": 0.0006955973221920431,
22
- "rewards/rejected": -0.0006261176313273609,
23
  "step": 10
24
  },
25
  {
26
  "epoch": 0.04,
27
  "learning_rate": 2.0833333333333333e-07,
28
- "logits/chosen": 0.2527770400047302,
29
- "logits/rejected": 0.3252316415309906,
30
- "logps/chosen": -318.26434326171875,
31
- "logps/rejected": -289.712158203125,
32
- "loss": 0.1867,
33
  "rewards/accuracies": 0.53125,
34
- "rewards/chosen": -0.0002512992941774428,
35
- "rewards/margins": 0.0006638198392465711,
36
- "rewards/rejected": -0.0009151192498393357,
37
  "step": 20
38
  },
39
  {
40
  "epoch": 0.06,
41
  "learning_rate": 3.1249999999999997e-07,
42
- "logits/chosen": 0.33200520277023315,
43
- "logits/rejected": 0.386562705039978,
44
- "logps/chosen": -296.428466796875,
45
- "logps/rejected": -283.0581970214844,
46
- "loss": 0.1751,
47
- "rewards/accuracies": 0.606249988079071,
48
- "rewards/chosen": -0.0013968495186418295,
49
- "rewards/margins": 0.002019281964749098,
50
- "rewards/rejected": -0.003416131716221571,
51
  "step": 30
52
  },
53
  {
54
  "epoch": 0.08,
55
  "learning_rate": 4.1666666666666667e-07,
56
- "logits/chosen": 0.22856274247169495,
57
- "logits/rejected": 0.32988241314888,
58
- "logps/chosen": -305.99603271484375,
59
- "logps/rejected": -293.2794189453125,
60
- "loss": 0.1853,
61
- "rewards/accuracies": 0.59375,
62
- "rewards/chosen": -0.0037044710479676723,
63
- "rewards/margins": 0.006597139872610569,
64
- "rewards/rejected": -0.010301610454916954,
65
  "step": 40
66
  },
67
  {
68
  "epoch": 0.1,
69
  "learning_rate": 5.208333333333334e-07,
70
- "logits/chosen": 0.2711636424064636,
71
- "logits/rejected": 0.3201292157173157,
72
- "logps/chosen": -314.5457458496094,
73
- "logps/rejected": -306.9486083984375,
74
- "loss": 0.1804,
75
- "rewards/accuracies": 0.643750011920929,
76
- "rewards/chosen": -0.008643127046525478,
77
- "rewards/margins": 0.0160285122692585,
78
- "rewards/rejected": -0.024671640247106552,
79
  "step": 50
80
  },
81
  {
82
  "epoch": 0.13,
83
  "learning_rate": 6.249999999999999e-07,
84
- "logits/chosen": 0.318474680185318,
85
- "logits/rejected": 0.4364282488822937,
86
- "logps/chosen": -294.63751220703125,
87
- "logps/rejected": -272.2162780761719,
88
- "loss": 0.1829,
89
  "rewards/accuracies": 0.6312500238418579,
90
- "rewards/chosen": -0.02497897669672966,
91
- "rewards/margins": 0.03536849468946457,
92
- "rewards/rejected": -0.06034746766090393,
93
  "step": 60
94
  },
95
  {
96
  "epoch": 0.15,
97
  "learning_rate": 7.291666666666666e-07,
98
- "logits/chosen": 0.3239240050315857,
99
- "logits/rejected": 0.3422989249229431,
100
- "logps/chosen": -304.72418212890625,
101
- "logps/rejected": -310.6500549316406,
102
- "loss": 0.1733,
103
- "rewards/accuracies": 0.6187499761581421,
104
- "rewards/chosen": -0.05714759975671768,
105
- "rewards/margins": 0.04348129406571388,
106
- "rewards/rejected": -0.10062889009714127,
107
  "step": 70
108
  },
109
  {
110
  "epoch": 0.17,
111
  "learning_rate": 8.333333333333333e-07,
112
- "logits/chosen": 0.35590413212776184,
113
- "logits/rejected": 0.4268670678138733,
114
- "logps/chosen": -306.149169921875,
115
- "logps/rejected": -279.54949951171875,
116
- "loss": 0.1565,
117
- "rewards/accuracies": 0.65625,
118
- "rewards/chosen": -0.10662279278039932,
119
- "rewards/margins": 0.10379795730113983,
120
- "rewards/rejected": -0.21042077243328094,
121
  "step": 80
122
  },
123
  {
124
  "epoch": 0.19,
125
  "learning_rate": 9.374999999999999e-07,
126
- "logits/chosen": 0.35440078377723694,
127
- "logits/rejected": 0.46889105439186096,
128
- "logps/chosen": -340.5415954589844,
129
- "logps/rejected": -296.3750915527344,
130
- "loss": 0.1275,
131
- "rewards/accuracies": 0.606249988079071,
132
- "rewards/chosen": -0.2491822987794876,
133
- "rewards/margins": 0.09365006536245346,
134
- "rewards/rejected": -0.34283238649368286,
135
  "step": 90
136
  },
137
  {
138
  "epoch": 0.21,
139
  "learning_rate": 9.999463737538052e-07,
140
- "logits/chosen": 0.4465979039669037,
141
- "logits/rejected": 0.4796988070011139,
142
- "logps/chosen": -326.7769470214844,
143
- "logps/rejected": -353.57916259765625,
144
- "loss": 0.1135,
145
- "rewards/accuracies": 0.606249988079071,
146
- "rewards/chosen": -0.39248064160346985,
147
- "rewards/margins": 0.1683400571346283,
148
- "rewards/rejected": -0.5608206987380981,
149
  "step": 100
150
  },
151
  {
152
  "epoch": 0.21,
153
- "eval_logits/chosen": 0.5124487280845642,
154
- "eval_logits/rejected": 0.5669243931770325,
155
- "eval_logps/chosen": -344.704833984375,
156
- "eval_logps/rejected": -387.95428466796875,
157
- "eval_loss": 0.10603953152894974,
158
- "eval_rewards/accuracies": 0.70703125,
159
- "eval_rewards/chosen": -0.4112018942832947,
160
- "eval_rewards/margins": 0.35208994150161743,
161
- "eval_rewards/rejected": -0.7632918357849121,
162
- "eval_runtime": 72.7637,
163
- "eval_samples_per_second": 27.486,
164
- "eval_steps_per_second": 0.44,
165
  "step": 100
166
  },
167
  {
168
  "epoch": 0.23,
169
  "learning_rate": 9.993432105822034e-07,
170
- "logits/chosen": 0.46603718400001526,
171
- "logits/rejected": 0.5153345465660095,
172
- "logps/chosen": -368.1249694824219,
173
- "logps/rejected": -372.76104736328125,
174
- "loss": 0.0948,
175
- "rewards/accuracies": 0.643750011920929,
176
- "rewards/chosen": -0.546645998954773,
177
- "rewards/margins": 0.3340264856815338,
178
- "rewards/rejected": -0.8806723356246948,
179
  "step": 110
180
  },
181
  {
182
  "epoch": 0.25,
183
  "learning_rate": 9.980706626858607e-07,
184
- "logits/chosen": 0.3831257224082947,
185
- "logits/rejected": 0.4766886830329895,
186
- "logps/chosen": -413.6620178222656,
187
- "logps/rejected": -443.69097900390625,
188
- "loss": 0.0872,
189
- "rewards/accuracies": 0.65625,
190
- "rewards/chosen": -0.6011692881584167,
191
- "rewards/margins": 0.3687607944011688,
192
- "rewards/rejected": -0.9699299931526184,
193
  "step": 120
194
  },
195
  {
196
  "epoch": 0.27,
197
  "learning_rate": 9.961304359538434e-07,
198
- "logits/chosen": 0.2602660059928894,
199
- "logits/rejected": 0.38597235083580017,
200
- "logps/chosen": -382.05316162109375,
201
- "logps/rejected": -372.43804931640625,
202
- "loss": 0.088,
203
- "rewards/accuracies": 0.637499988079071,
204
- "rewards/chosen": -0.5223852396011353,
205
- "rewards/margins": 0.23098063468933105,
206
- "rewards/rejected": -0.7533659338951111,
207
  "step": 130
208
  },
209
  {
210
  "epoch": 0.29,
211
  "learning_rate": 9.935251313189563e-07,
212
- "logits/chosen": 0.30170053243637085,
213
- "logits/rejected": 0.34471625089645386,
214
- "logps/chosen": -358.23101806640625,
215
- "logps/rejected": -367.53643798828125,
216
- "loss": 0.0912,
217
- "rewards/accuracies": 0.706250011920929,
218
- "rewards/chosen": -0.44500336050987244,
219
- "rewards/margins": 0.29020196199417114,
220
- "rewards/rejected": -0.7352052927017212,
221
  "step": 140
222
  },
223
  {
224
  "epoch": 0.31,
225
  "learning_rate": 9.902582412711118e-07,
226
- "logits/chosen": 0.2502862811088562,
227
- "logits/rejected": 0.41286569833755493,
228
- "logps/chosen": -331.702392578125,
229
- "logps/rejected": -343.1724548339844,
230
- "loss": 0.0805,
231
- "rewards/accuracies": 0.643750011920929,
232
- "rewards/chosen": -0.5062417984008789,
233
- "rewards/margins": 0.3571576774120331,
234
- "rewards/rejected": -0.8633993864059448,
235
  "step": 150
236
  },
237
  {
238
  "epoch": 0.33,
239
  "learning_rate": 9.86334145175542e-07,
240
- "logits/chosen": 0.2876195013523102,
241
- "logits/rejected": 0.40448102355003357,
242
- "logps/chosen": -360.23065185546875,
243
- "logps/rejected": -405.818603515625,
244
- "loss": 0.0739,
245
- "rewards/accuracies": 0.7250000238418579,
246
- "rewards/chosen": -0.6673613786697388,
247
- "rewards/margins": 0.4857359826564789,
248
- "rewards/rejected": -1.15309739112854,
249
  "step": 160
250
  },
251
  {
252
  "epoch": 0.36,
253
  "learning_rate": 9.817581034021272e-07,
254
- "logits/chosen": 0.2106766402721405,
255
- "logits/rejected": 0.3229225277900696,
256
- "logps/chosen": -428.03643798828125,
257
- "logps/rejected": -429.11322021484375,
258
- "loss": 0.0617,
259
- "rewards/accuracies": 0.668749988079071,
260
- "rewards/chosen": -0.6919075846672058,
261
- "rewards/margins": 0.339050829410553,
262
- "rewards/rejected": -1.0309584140777588,
263
  "step": 170
264
  },
265
  {
266
  "epoch": 0.38,
267
  "learning_rate": 9.765362502737097e-07,
268
- "logits/chosen": 0.2592887878417969,
269
- "logits/rejected": 0.32125118374824524,
270
- "logps/chosen": -373.13519287109375,
271
- "logps/rejected": -411.79132080078125,
272
- "loss": 0.0567,
273
- "rewards/accuracies": 0.6812499761581421,
274
- "rewards/chosen": -0.795512855052948,
275
- "rewards/margins": 0.5069063901901245,
276
- "rewards/rejected": -1.3024193048477173,
277
  "step": 180
278
  },
279
  {
280
  "epoch": 0.4,
281
  "learning_rate": 9.706755858428485e-07,
282
- "logits/chosen": 0.3405285179615021,
283
- "logits/rejected": 0.4521329998970032,
284
- "logps/chosen": -348.65020751953125,
285
- "logps/rejected": -362.8314208984375,
286
- "loss": 0.0642,
287
- "rewards/accuracies": 0.6499999761581421,
288
- "rewards/chosen": -0.8077024221420288,
289
- "rewards/margins": 0.4007204473018646,
290
- "rewards/rejected": -1.2084228992462158,
291
  "step": 190
292
  },
293
  {
294
  "epoch": 0.42,
295
  "learning_rate": 9.641839665080363e-07,
296
- "logits/chosen": 0.2780831456184387,
297
- "logits/rejected": 0.38184481859207153,
298
- "logps/chosen": -348.57135009765625,
299
- "logps/rejected": -405.402099609375,
300
- "loss": 0.0648,
301
- "rewards/accuracies": 0.6875,
302
- "rewards/chosen": -0.730074405670166,
303
- "rewards/margins": 0.5448073148727417,
304
- "rewards/rejected": -1.2748818397521973,
305
  "step": 200
306
  },
307
  {
308
  "epoch": 0.42,
309
- "eval_logits/chosen": 0.31048065423965454,
310
- "eval_logits/rejected": 0.3866545557975769,
311
- "eval_logps/chosen": -372.07177734375,
312
- "eval_logps/rejected": -442.6331787109375,
313
- "eval_loss": 0.06636561453342438,
314
- "eval_rewards/accuracies": 0.71875,
315
- "eval_rewards/chosen": -0.6848713755607605,
316
- "eval_rewards/margins": 0.6252094507217407,
317
- "eval_rewards/rejected": -1.3100807666778564,
318
- "eval_runtime": 74.8918,
319
- "eval_samples_per_second": 26.705,
320
- "eval_steps_per_second": 0.427,
321
  "step": 200
322
  },
323
  {
324
  "epoch": 0.44,
325
  "learning_rate": 9.570700944819582e-07,
326
- "logits/chosen": 0.23630718886852264,
327
- "logits/rejected": 0.31974127888679504,
328
- "logps/chosen": -394.2319641113281,
329
- "logps/rejected": -433.3797912597656,
330
- "loss": 0.0565,
331
- "rewards/accuracies": 0.731249988079071,
332
- "rewards/chosen": -0.7185484170913696,
333
- "rewards/margins": 0.5530284643173218,
334
- "rewards/rejected": -1.271577000617981,
335
  "step": 210
336
  },
337
  {
338
  "epoch": 0.46,
339
  "learning_rate": 9.493435061259129e-07,
340
- "logits/chosen": 0.13368266820907593,
341
- "logits/rejected": 0.32444554567337036,
342
- "logps/chosen": -409.3749084472656,
343
- "logps/rejected": -413.44097900390625,
344
- "loss": 0.0558,
345
- "rewards/accuracies": 0.6812499761581421,
346
- "rewards/chosen": -0.8843868374824524,
347
- "rewards/margins": 0.42033252120018005,
348
- "rewards/rejected": -1.3047194480895996,
349
  "step": 220
350
  },
351
  {
352
  "epoch": 0.48,
353
  "learning_rate": 9.4101455916603e-07,
354
- "logits/chosen": 0.23291189968585968,
355
- "logits/rejected": 0.3359973430633545,
356
- "logps/chosen": -385.23681640625,
357
- "logps/rejected": -439.33294677734375,
358
- "loss": 0.0557,
359
- "rewards/accuracies": 0.6937500238418579,
360
- "rewards/chosen": -1.0321977138519287,
361
- "rewards/margins": 0.5443316102027893,
362
- "rewards/rejected": -1.5765292644500732,
363
  "step": 230
364
  },
365
  {
366
  "epoch": 0.5,
367
  "learning_rate": 9.320944188084241e-07,
368
- "logits/chosen": 0.1577446311712265,
369
- "logits/rejected": 0.2382678985595703,
370
- "logps/chosen": -437.01275634765625,
371
- "logps/rejected": -492.4737243652344,
372
- "loss": 0.0618,
373
- "rewards/accuracies": 0.7250000238418579,
374
- "rewards/chosen": -0.8377536535263062,
375
- "rewards/margins": 0.61639404296875,
376
- "rewards/rejected": -1.4541476964950562,
377
  "step": 240
378
  },
379
  {
380
  "epoch": 0.52,
381
  "learning_rate": 9.225950427718974e-07,
382
- "logits/chosen": 0.19211766123771667,
383
- "logits/rejected": 0.27316194772720337,
384
- "logps/chosen": -396.7606201171875,
385
- "logps/rejected": -429.4735412597656,
386
- "loss": 0.0599,
387
- "rewards/accuracies": 0.65625,
388
- "rewards/chosen": -0.8097313046455383,
389
- "rewards/margins": 0.5714440941810608,
390
- "rewards/rejected": -1.3811752796173096,
391
  "step": 250
392
  },
393
  {
394
  "epoch": 0.54,
395
  "learning_rate": 9.125291652582547e-07,
396
- "logits/chosen": 0.2013740837574005,
397
- "logits/rejected": 0.3595736622810364,
398
- "logps/chosen": -440.74432373046875,
399
- "logps/rejected": -463.736572265625,
400
- "loss": 0.0633,
401
- "rewards/accuracies": 0.675000011920929,
402
- "rewards/chosen": -0.9895216822624207,
403
- "rewards/margins": 0.7094276547431946,
404
- "rewards/rejected": -1.6989490985870361,
405
  "step": 260
406
  },
407
  {
408
  "epoch": 0.57,
409
  "learning_rate": 9.019102798817195e-07,
410
- "logits/chosen": 0.2775947153568268,
411
- "logits/rejected": 0.3985646963119507,
412
- "logps/chosen": -395.5704345703125,
413
- "logps/rejected": -408.7200622558594,
414
- "loss": 0.0775,
415
  "rewards/accuracies": 0.71875,
416
- "rewards/chosen": -0.6699169874191284,
417
- "rewards/margins": 0.4529454708099365,
418
- "rewards/rejected": -1.122862458229065,
419
  "step": 270
420
  },
421
  {
422
  "epoch": 0.59,
423
  "learning_rate": 8.90752621580335e-07,
424
- "logits/chosen": 0.25832900404930115,
425
- "logits/rejected": 0.3815799355506897,
426
- "logps/chosen": -392.74462890625,
427
- "logps/rejected": -462.65399169921875,
428
- "loss": 0.0621,
429
- "rewards/accuracies": 0.706250011920929,
430
- "rewards/chosen": -0.8548792004585266,
431
- "rewards/margins": 0.5883964896202087,
432
- "rewards/rejected": -1.4432756900787354,
433
  "step": 280
434
  },
435
  {
436
  "epoch": 0.61,
437
  "learning_rate": 8.79071147533597e-07,
438
- "logits/chosen": 0.27000856399536133,
439
- "logits/rejected": 0.3684462904930115,
440
- "logps/chosen": -373.5545654296875,
441
- "logps/rejected": -402.52203369140625,
442
- "loss": 0.0565,
443
- "rewards/accuracies": 0.65625,
444
- "rewards/chosen": -0.7661363482475281,
445
- "rewards/margins": 0.4899563789367676,
446
- "rewards/rejected": -1.2560927867889404,
447
  "step": 290
448
  },
449
  {
450
  "epoch": 0.63,
451
  "learning_rate": 8.668815171119019e-07,
452
- "logits/chosen": 0.20499548316001892,
453
- "logits/rejected": 0.4121769070625305,
454
- "logps/chosen": -405.9000549316406,
455
- "logps/rejected": -392.7608337402344,
456
- "loss": 0.0565,
457
- "rewards/accuracies": 0.6625000238418579,
458
- "rewards/chosen": -0.9424744844436646,
459
- "rewards/margins": 0.392583429813385,
460
- "rewards/rejected": -1.3350577354431152,
461
  "step": 300
462
  },
463
  {
464
  "epoch": 0.63,
465
- "eval_logits/chosen": 0.28342685103416443,
466
- "eval_logits/rejected": 0.3624545931816101,
467
- "eval_logps/chosen": -379.5645751953125,
468
- "eval_logps/rejected": -461.2485656738281,
469
- "eval_loss": 0.05852030962705612,
470
- "eval_rewards/accuracies": 0.73046875,
471
- "eval_rewards/chosen": -0.7597990036010742,
472
- "eval_rewards/margins": 0.7364358305931091,
473
- "eval_rewards/rejected": -1.4962347745895386,
474
- "eval_runtime": 74.348,
475
- "eval_samples_per_second": 26.901,
476
- "eval_steps_per_second": 0.43,
477
  "step": 300
478
  },
479
  {
480
  "epoch": 0.65,
481
  "learning_rate": 8.54200070884685e-07,
482
- "logits/chosen": 0.2647777199745178,
483
- "logits/rejected": 0.28958767652511597,
484
- "logps/chosen": -424.25604248046875,
485
- "logps/rejected": -472.11639404296875,
486
- "loss": 0.0603,
487
- "rewards/accuracies": 0.6625000238418579,
488
- "rewards/chosen": -1.0271676778793335,
489
- "rewards/margins": 0.4712137281894684,
490
- "rewards/rejected": -1.4983813762664795,
491
  "step": 310
492
  },
493
  {
494
  "epoch": 0.67,
495
  "learning_rate": 8.410438087153911e-07,
496
- "logits/chosen": 0.18836882710456848,
497
- "logits/rejected": 0.3168850541114807,
498
- "logps/chosen": -439.59716796875,
499
- "logps/rejected": -433.8974609375,
500
- "loss": 0.0608,
501
- "rewards/accuracies": 0.625,
502
- "rewards/chosen": -0.8473286628723145,
503
- "rewards/margins": 0.38924068212509155,
504
- "rewards/rejected": -1.2365691661834717,
505
  "step": 320
506
  },
507
  {
508
  "epoch": 0.69,
509
  "learning_rate": 8.274303669726426e-07,
510
- "logits/chosen": 0.23949918150901794,
511
- "logits/rejected": 0.25133609771728516,
512
- "logps/chosen": -380.25909423828125,
513
- "logps/rejected": -497.5677185058594,
514
- "loss": 0.0576,
515
- "rewards/accuracies": 0.737500011920929,
516
- "rewards/chosen": -0.7643330097198486,
517
- "rewards/margins": 0.8182552456855774,
518
- "rewards/rejected": -1.5825883150100708,
519
  "step": 330
520
  },
521
  {
522
  "epoch": 0.71,
523
  "learning_rate": 8.133779948881513e-07,
524
- "logits/chosen": 0.2764629125595093,
525
- "logits/rejected": 0.2793533205986023,
526
- "logps/chosen": -400.74652099609375,
527
- "logps/rejected": -499.9100036621094,
528
- "loss": 0.0488,
529
- "rewards/accuracies": 0.643750011920929,
530
- "rewards/chosen": -1.11314058303833,
531
- "rewards/margins": 0.7436734437942505,
532
- "rewards/rejected": -1.8568140268325806,
533
  "step": 340
534
  },
535
  {
536
  "epoch": 0.73,
537
  "learning_rate": 7.989055300930704e-07,
538
- "logits/chosen": 0.321544349193573,
539
- "logits/rejected": 0.34274429082870483,
540
- "logps/chosen": -420.0064392089844,
541
- "logps/rejected": -518.2902221679688,
542
- "loss": 0.047,
543
- "rewards/accuracies": 0.731249988079071,
544
- "rewards/chosen": -1.1256664991378784,
545
- "rewards/margins": 0.7915542125701904,
546
- "rewards/rejected": -1.9172207117080688,
547
  "step": 350
548
  },
549
  {
550
  "epoch": 0.75,
551
  "learning_rate": 7.840323733655778e-07,
552
- "logits/chosen": 0.25519177317619324,
553
- "logits/rejected": 0.36787793040275574,
554
- "logps/chosen": -438.910400390625,
555
- "logps/rejected": -482.37774658203125,
556
- "loss": 0.0508,
557
- "rewards/accuracies": 0.737500011920929,
558
- "rewards/chosen": -0.9125250577926636,
559
- "rewards/margins": 0.8073097467422485,
560
- "rewards/rejected": -1.7198346853256226,
561
  "step": 360
562
  },
563
  {
564
  "epoch": 0.77,
565
  "learning_rate": 7.687784626235447e-07,
566
- "logits/chosen": 0.24404089152812958,
567
- "logits/rejected": 0.3457106649875641,
568
- "logps/chosen": -447.19036865234375,
569
- "logps/rejected": -464.69091796875,
570
- "loss": 0.0524,
571
- "rewards/accuracies": 0.706250011920929,
572
- "rewards/chosen": -0.9422460794448853,
573
- "rewards/margins": 0.6395829916000366,
574
- "rewards/rejected": -1.5818291902542114,
575
  "step": 370
576
  },
577
  {
578
  "epoch": 0.8,
579
  "learning_rate": 7.531642461971514e-07,
580
- "logits/chosen": 0.13880091905593872,
581
- "logits/rejected": 0.28412070870399475,
582
- "logps/chosen": -437.6341247558594,
583
- "logps/rejected": -500.48712158203125,
584
- "loss": 0.0526,
585
- "rewards/accuracies": 0.7124999761581421,
586
- "rewards/chosen": -1.0558099746704102,
587
- "rewards/margins": 0.8256821632385254,
588
- "rewards/rejected": -1.881492018699646,
589
  "step": 380
590
  },
591
  {
592
  "epoch": 0.82,
593
  "learning_rate": 7.372106554172801e-07,
594
- "logits/chosen": 0.19525602459907532,
595
- "logits/rejected": 0.29236602783203125,
596
- "logps/chosen": -405.10064697265625,
597
- "logps/rejected": -449.4366149902344,
598
- "loss": 0.06,
599
- "rewards/accuracies": 0.706250011920929,
600
- "rewards/chosen": -0.8169919848442078,
601
- "rewards/margins": 0.6195858120918274,
602
- "rewards/rejected": -1.4365779161453247,
603
  "step": 390
604
  },
605
  {
606
  "epoch": 0.84,
607
  "learning_rate": 7.209390765564318e-07,
608
- "logits/chosen": 0.157417893409729,
609
- "logits/rejected": 0.24023254215717316,
610
- "logps/chosen": -378.41204833984375,
611
- "logps/rejected": -445.98272705078125,
612
- "loss": 0.0643,
613
- "rewards/accuracies": 0.6499999761581421,
614
- "rewards/chosen": -0.813235878944397,
615
- "rewards/margins": 0.5227171182632446,
616
- "rewards/rejected": -1.3359529972076416,
617
  "step": 400
618
  },
619
  {
620
  "epoch": 0.84,
621
- "eval_logits/chosen": 0.18623091280460358,
622
- "eval_logits/rejected": 0.2607361972332001,
623
- "eval_logps/chosen": -370.18780517578125,
624
- "eval_logps/rejected": -451.6427917480469,
625
- "eval_loss": 0.06227795034646988,
626
- "eval_rewards/accuracies": 0.73046875,
627
- "eval_rewards/chosen": -0.6660312414169312,
628
- "eval_rewards/margins": 0.7341459393501282,
629
- "eval_rewards/rejected": -1.400177240371704,
630
- "eval_runtime": 75.0562,
631
- "eval_samples_per_second": 26.647,
632
- "eval_steps_per_second": 0.426,
633
  "step": 400
634
  },
635
  {
636
  "epoch": 0.86,
637
  "learning_rate": 7.043713221597773e-07,
638
- "logits/chosen": 0.13670489192008972,
639
- "logits/rejected": 0.21374602615833282,
640
- "logps/chosen": -379.88201904296875,
641
- "logps/rejected": -425.9203186035156,
642
- "loss": 0.0629,
643
- "rewards/accuracies": 0.7437499761581421,
644
- "rewards/chosen": -0.6690796613693237,
645
- "rewards/margins": 0.6432248950004578,
646
- "rewards/rejected": -1.3123046159744263,
647
  "step": 410
648
  },
649
  {
650
  "epoch": 0.88,
651
  "learning_rate": 6.875296018047809e-07,
652
- "logits/chosen": 0.15494689345359802,
653
- "logits/rejected": 0.22296158969402313,
654
- "logps/chosen": -377.009765625,
655
- "logps/rejected": -412.5980529785156,
656
- "loss": 0.0605,
657
- "rewards/accuracies": 0.7437499761581421,
658
- "rewards/chosen": -0.6854912042617798,
659
- "rewards/margins": 0.6449334025382996,
660
- "rewards/rejected": -1.3304245471954346,
661
  "step": 420
662
  },
663
  {
664
  "epoch": 0.9,
665
  "learning_rate": 6.704364923285857e-07,
666
- "logits/chosen": 0.10000725835561752,
667
- "logits/rejected": 0.20355455577373505,
668
- "logps/chosen": -442.444580078125,
669
- "logps/rejected": -475.17901611328125,
670
- "loss": 0.0633,
671
- "rewards/accuracies": 0.699999988079071,
672
- "rewards/chosen": -0.920007050037384,
673
- "rewards/margins": 0.6362106204032898,
674
- "rewards/rejected": -1.5562175512313843,
675
  "step": 430
676
  },
677
  {
678
  "epoch": 0.92,
679
  "learning_rate": 6.531149075630796e-07,
680
- "logits/chosen": 0.18681207299232483,
681
- "logits/rejected": 0.24837341904640198,
682
- "logps/chosen": -394.078857421875,
683
- "logps/rejected": -446.2527770996094,
684
- "loss": 0.0585,
685
- "rewards/accuracies": 0.6499999761581421,
686
- "rewards/chosen": -0.9974417686462402,
687
- "rewards/margins": 0.7502747774124146,
688
- "rewards/rejected": -1.7477165460586548,
689
  "step": 440
690
  },
691
  {
692
  "epoch": 0.94,
693
  "learning_rate": 6.355880676182085e-07,
694
- "logits/chosen": 0.1681515872478485,
695
- "logits/rejected": 0.34068578481674194,
696
- "logps/chosen": -438.5791015625,
697
- "logps/rejected": -468.8702087402344,
698
- "loss": 0.056,
699
- "rewards/accuracies": 0.65625,
700
- "rewards/chosen": -0.959572434425354,
701
- "rewards/margins": 0.6984410881996155,
702
- "rewards/rejected": -1.6580133438110352,
703
  "step": 450
704
  },
705
  {
706
  "epoch": 0.96,
707
  "learning_rate": 6.178794677547137e-07,
708
- "logits/chosen": 0.22829516232013702,
709
- "logits/rejected": 0.33127135038375854,
710
- "logps/chosen": -417.236572265625,
711
- "logps/rejected": -454.47930908203125,
712
- "loss": 0.0488,
713
- "rewards/accuracies": 0.6812499761581421,
714
- "rewards/chosen": -1.1504933834075928,
715
- "rewards/margins": 0.6518447399139404,
716
- "rewards/rejected": -1.8023380041122437,
717
  "step": 460
718
  },
719
  {
720
  "epoch": 0.98,
721
  "learning_rate": 6.000128468880222e-07,
722
- "logits/chosen": 0.1735696941614151,
723
- "logits/rejected": 0.23917797207832336,
724
- "logps/chosen": -441.33526611328125,
725
- "logps/rejected": -507.3721618652344,
726
- "loss": 0.0469,
727
- "rewards/accuracies": 0.699999988079071,
728
- "rewards/chosen": -1.0348496437072754,
729
- "rewards/margins": 0.6962344646453857,
730
- "rewards/rejected": -1.7310841083526611,
731
  "step": 470
732
  },
733
  {
734
  "epoch": 1.0,
735
  "learning_rate": 5.820121557655108e-07,
736
- "logits/chosen": 0.16151778399944305,
737
- "logits/rejected": 0.30070850253105164,
738
- "logps/chosen": -389.2116394042969,
739
- "logps/rejected": -456.37554931640625,
740
- "loss": 0.0468,
741
- "rewards/accuracies": 0.731249988079071,
742
- "rewards/chosen": -0.9426639676094055,
743
- "rewards/margins": 0.6416986584663391,
744
- "rewards/rejected": -1.5843626260757446,
745
  "step": 480
746
  },
747
  {
748
  "epoch": 1.03,
749
  "learning_rate": 5.639015248598023e-07,
750
- "logits/chosen": 0.2655644118785858,
751
- "logits/rejected": 0.3393844962120056,
752
- "logps/chosen": -394.0737609863281,
753
- "logps/rejected": -481.3250427246094,
754
- "loss": 0.0334,
755
- "rewards/accuracies": 0.7749999761581421,
756
- "rewards/chosen": -0.886431097984314,
757
- "rewards/margins": 1.0793970823287964,
758
- "rewards/rejected": -1.9658282995224,
759
  "step": 490
760
  },
761
  {
762
  "epoch": 1.05,
763
  "learning_rate": 5.457052320211339e-07,
764
- "logits/chosen": 0.30765271186828613,
765
- "logits/rejected": 0.33542585372924805,
766
- "logps/chosen": -416.736083984375,
767
- "logps/rejected": -526.0009765625,
768
- "loss": 0.0214,
769
- "rewards/accuracies": 0.75,
770
- "rewards/chosen": -1.1540133953094482,
771
- "rewards/margins": 1.152156114578247,
772
- "rewards/rejected": -2.3061697483062744,
773
  "step": 500
774
  },
775
  {
776
  "epoch": 1.05,
777
- "eval_logits/chosen": 0.4214042127132416,
778
- "eval_logits/rejected": 0.4999687075614929,
779
- "eval_logps/chosen": -460.466796875,
780
- "eval_logps/rejected": -587.046875,
781
- "eval_loss": 0.030972089618444443,
782
- "eval_rewards/accuracies": 0.7421875,
783
- "eval_rewards/chosen": -1.5688213109970093,
784
- "eval_rewards/margins": 1.185396432876587,
785
- "eval_rewards/rejected": -2.7542178630828857,
786
- "eval_runtime": 74.4978,
787
- "eval_samples_per_second": 26.846,
788
- "eval_steps_per_second": 0.43,
789
  "step": 500
790
  },
791
  {
792
  "epoch": 1.07,
793
  "learning_rate": 5.274476699321637e-07,
794
- "logits/chosen": 0.35821476578712463,
795
- "logits/rejected": 0.46395426988601685,
796
- "logps/chosen": -462.271484375,
797
- "logps/rejected": -581.0950927734375,
798
- "loss": 0.0148,
799
- "rewards/accuracies": 0.8062499761581421,
800
- "rewards/chosen": -1.6695778369903564,
801
- "rewards/margins": 1.4044713973999023,
802
- "rewards/rejected": -3.074049472808838,
803
  "step": 510
804
  },
805
  {
806
  "epoch": 1.09,
807
  "learning_rate": 5.091533134088387e-07,
808
- "logits/chosen": 0.3840331435203552,
809
- "logits/rejected": 0.5413318872451782,
810
- "logps/chosen": -466.8184509277344,
811
- "logps/rejected": -561.7572631835938,
812
- "loss": 0.0145,
813
- "rewards/accuracies": 0.78125,
814
- "rewards/chosen": -1.635196328163147,
815
- "rewards/margins": 1.1625127792358398,
816
- "rewards/rejected": -2.7977089881896973,
817
  "step": 520
818
  },
819
  {
820
  "epoch": 1.11,
821
  "learning_rate": 4.908466865911614e-07,
822
- "logits/chosen": 0.3811998963356018,
823
- "logits/rejected": 0.5391031503677368,
824
- "logps/chosen": -462.8243103027344,
825
- "logps/rejected": -599.1219482421875,
826
- "loss": 0.0153,
827
- "rewards/accuracies": 0.7749999761581421,
828
- "rewards/chosen": -1.5375772714614868,
829
- "rewards/margins": 1.5291885137557983,
830
- "rewards/rejected": -3.066765785217285,
831
  "step": 530
832
  },
833
  {
834
  "epoch": 1.13,
835
  "learning_rate": 4.7255233006783624e-07,
836
- "logits/chosen": 0.29220065474510193,
837
- "logits/rejected": 0.5385439395904541,
838
- "logps/chosen": -494.8135681152344,
839
- "logps/rejected": -553.2826538085938,
840
- "loss": 0.0152,
841
- "rewards/accuracies": 0.7437499761581421,
842
- "rewards/chosen": -1.4295642375946045,
843
- "rewards/margins": 1.265754222869873,
844
- "rewards/rejected": -2.6953186988830566,
845
  "step": 540
846
  },
847
  {
848
  "epoch": 1.15,
849
  "learning_rate": 4.5429476797886617e-07,
850
- "logits/chosen": 0.36708998680114746,
851
- "logits/rejected": 0.4897306561470032,
852
- "logps/chosen": -459.0501403808594,
853
- "logps/rejected": -559.3533935546875,
854
- "loss": 0.0156,
855
- "rewards/accuracies": 0.706250011920929,
856
- "rewards/chosen": -1.6044361591339111,
857
- "rewards/margins": 1.0187304019927979,
858
- "rewards/rejected": -2.623166561126709,
859
  "step": 550
860
  },
861
  {
862
  "epoch": 1.17,
863
  "learning_rate": 4.3609847514019763e-07,
864
- "logits/chosen": 0.401399701833725,
865
- "logits/rejected": 0.5879624485969543,
866
- "logps/chosen": -483.4132385253906,
867
- "logps/rejected": -575.8525390625,
868
- "loss": 0.0143,
869
- "rewards/accuracies": 0.800000011920929,
870
- "rewards/chosen": -1.5152244567871094,
871
- "rewards/margins": 1.3872954845428467,
872
- "rewards/rejected": -2.902519702911377,
873
  "step": 560
874
  },
875
  {
876
  "epoch": 1.19,
877
  "learning_rate": 4.179878442344892e-07,
878
- "logits/chosen": 0.3999864161014557,
879
- "logits/rejected": 0.5597594976425171,
880
- "logps/chosen": -466.9307556152344,
881
- "logps/rejected": -579.4730834960938,
882
- "loss": 0.0142,
883
- "rewards/accuracies": 0.7562500238418579,
884
- "rewards/chosen": -1.7734342813491821,
885
- "rewards/margins": 1.3825814723968506,
886
- "rewards/rejected": -3.1560158729553223,
887
  "step": 570
888
  },
889
  {
890
  "epoch": 1.21,
891
  "learning_rate": 3.9998715311197783e-07,
892
- "logits/chosen": 0.43183040618896484,
893
- "logits/rejected": 0.49680033326148987,
894
- "logps/chosen": -505.7025451660156,
895
- "logps/rejected": -659.0694580078125,
896
- "loss": 0.0109,
897
- "rewards/accuracies": 0.7875000238418579,
898
- "rewards/chosen": -2.024803638458252,
899
- "rewards/margins": 1.5267921686172485,
900
- "rewards/rejected": -3.5515952110290527,
901
  "step": 580
902
  },
903
  {
904
  "epoch": 1.23,
905
  "learning_rate": 3.821205322452863e-07,
906
- "logits/chosen": 0.45015770196914673,
907
- "logits/rejected": 0.6397759318351746,
908
- "logps/chosen": -516.25830078125,
909
- "logps/rejected": -631.279052734375,
910
- "loss": 0.0115,
911
- "rewards/accuracies": 0.7437499761581421,
912
- "rewards/chosen": -1.9833446741104126,
913
- "rewards/margins": 1.3147289752960205,
914
- "rewards/rejected": -3.2980740070343018,
915
  "step": 590
916
  },
917
  {
918
  "epoch": 1.26,
919
  "learning_rate": 3.6441193238179146e-07,
920
- "logits/chosen": 0.46848899126052856,
921
- "logits/rejected": 0.5270673036575317,
922
- "logps/chosen": -509.9579162597656,
923
- "logps/rejected": -689.2035522460938,
924
- "loss": 0.0145,
925
- "rewards/accuracies": 0.824999988079071,
926
- "rewards/chosen": -1.8605167865753174,
927
- "rewards/margins": 1.5842502117156982,
928
- "rewards/rejected": -3.4447669982910156,
929
  "step": 600
930
  },
931
  {
932
  "epoch": 1.26,
933
- "eval_logits/chosen": 0.5416067838668823,
934
- "eval_logits/rejected": 0.63544762134552,
935
- "eval_logps/chosen": -482.76092529296875,
936
- "eval_logps/rejected": -615.6082763671875,
937
- "eval_loss": 0.024462632834911346,
938
- "eval_rewards/accuracies": 0.734375,
939
- "eval_rewards/chosen": -1.7917625904083252,
940
- "eval_rewards/margins": 1.2480695247650146,
941
- "eval_rewards/rejected": -3.039832353591919,
942
- "eval_runtime": 75.4471,
943
- "eval_samples_per_second": 26.509,
944
- "eval_steps_per_second": 0.424,
945
  "step": 600
946
  },
947
  {
948
  "epoch": 1.28,
949
  "learning_rate": 3.4688509243692034e-07,
950
- "logits/chosen": 0.47085338830947876,
951
- "logits/rejected": 0.5862728357315063,
952
- "logps/chosen": -508.70098876953125,
953
- "logps/rejected": -643.3338623046875,
954
- "loss": 0.0121,
955
- "rewards/accuracies": 0.831250011920929,
956
- "rewards/chosen": -1.7418628931045532,
957
- "rewards/margins": 1.5565435886383057,
958
- "rewards/rejected": -3.2984061241149902,
959
  "step": 610
960
  },
961
  {
962
  "epoch": 1.3,
963
  "learning_rate": 3.295635076714144e-07,
964
- "logits/chosen": 0.3978613018989563,
965
- "logits/rejected": 0.5926721096038818,
966
- "logps/chosen": -494.2681579589844,
967
- "logps/rejected": -604.7338256835938,
968
- "loss": 0.0114,
969
- "rewards/accuracies": 0.7124999761581421,
970
- "rewards/chosen": -1.8917449712753296,
971
- "rewards/margins": 1.2589935064315796,
972
- "rewards/rejected": -3.1507387161254883,
973
  "step": 620
974
  },
975
  {
976
  "epoch": 1.32,
977
  "learning_rate": 3.12470398195219e-07,
978
- "logits/chosen": 0.4904108941555023,
979
- "logits/rejected": 0.6586324572563171,
980
- "logps/chosen": -502.24322509765625,
981
- "logps/rejected": -618.8930053710938,
982
- "loss": 0.0112,
983
- "rewards/accuracies": 0.737500011920929,
984
- "rewards/chosen": -1.9003562927246094,
985
- "rewards/margins": 1.4167784452438354,
986
- "rewards/rejected": -3.3171348571777344,
987
  "step": 630
988
  },
989
  {
990
  "epoch": 1.34,
991
  "learning_rate": 2.956286778402226e-07,
992
- "logits/chosen": 0.4602645933628082,
993
- "logits/rejected": 0.5689934492111206,
994
- "logps/chosen": -449.55718994140625,
995
- "logps/rejected": -578.3916625976562,
996
- "loss": 0.0118,
997
- "rewards/accuracies": 0.7749999761581421,
998
- "rewards/chosen": -1.755163550376892,
999
- "rewards/margins": 1.4266632795333862,
1000
- "rewards/rejected": -3.1818270683288574,
1001
  "step": 640
1002
  },
1003
  {
1004
  "epoch": 1.36,
1005
  "learning_rate": 2.7906092344356826e-07,
1006
- "logits/chosen": 0.4582684636116028,
1007
- "logits/rejected": 0.5123009085655212,
1008
- "logps/chosen": -464.2190856933594,
1009
- "logps/rejected": -599.2928466796875,
1010
- "loss": 0.0127,
1011
- "rewards/accuracies": 0.731249988079071,
1012
- "rewards/chosen": -1.6637153625488281,
1013
- "rewards/margins": 1.4032952785491943,
1014
- "rewards/rejected": -3.0670104026794434,
1015
  "step": 650
1016
  },
1017
  {
1018
  "epoch": 1.38,
1019
  "learning_rate": 2.6278934458271996e-07,
1020
- "logits/chosen": 0.3585265576839447,
1021
- "logits/rejected": 0.5152703523635864,
1022
- "logps/chosen": -539.8528442382812,
1023
- "logps/rejected": -647.3905029296875,
1024
- "loss": 0.0106,
1025
- "rewards/accuracies": 0.78125,
1026
- "rewards/chosen": -1.8331562280654907,
1027
- "rewards/margins": 1.5139731168746948,
1028
- "rewards/rejected": -3.3471293449401855,
1029
  "step": 660
1030
  },
1031
  {
1032
  "epoch": 1.4,
1033
  "learning_rate": 2.468357538028487e-07,
1034
- "logits/chosen": 0.4794502854347229,
1035
- "logits/rejected": 0.6796830892562866,
1036
- "logps/chosen": -531.4798583984375,
1037
- "logps/rejected": -636.687255859375,
1038
- "loss": 0.0106,
1039
- "rewards/accuracies": 0.7250000238418579,
1040
- "rewards/chosen": -2.1033949851989746,
1041
- "rewards/margins": 1.3440332412719727,
1042
- "rewards/rejected": -3.4474282264709473,
1043
  "step": 670
1044
  },
1045
  {
1046
  "epoch": 1.42,
1047
  "learning_rate": 2.312215373764551e-07,
1048
- "logits/chosen": 0.48690152168273926,
1049
- "logits/rejected": 0.6909142136573792,
1050
- "logps/chosen": -482.9384765625,
1051
- "logps/rejected": -627.9971313476562,
1052
- "loss": 0.0105,
1053
- "rewards/accuracies": 0.800000011920929,
1054
- "rewards/chosen": -1.8502315282821655,
1055
- "rewards/margins": 1.5311414003372192,
1056
- "rewards/rejected": -3.381373167037964,
1057
  "step": 680
1058
  },
1059
  {
1060
  "epoch": 1.44,
1061
  "learning_rate": 2.1596762663442213e-07,
1062
- "logits/chosen": 0.40491142868995667,
1063
- "logits/rejected": 0.6269996166229248,
1064
- "logps/chosen": -517.865234375,
1065
- "logps/rejected": -638.0469970703125,
1066
- "loss": 0.0118,
1067
- "rewards/accuracies": 0.800000011920929,
1068
- "rewards/chosen": -1.894121766090393,
1069
- "rewards/margins": 1.5707839727401733,
1070
- "rewards/rejected": -3.4649059772491455,
1071
  "step": 690
1072
  },
1073
  {
1074
  "epoch": 1.47,
1075
  "learning_rate": 2.0109446990692963e-07,
1076
- "logits/chosen": 0.4644289016723633,
1077
- "logits/rejected": 0.5091214179992676,
1078
- "logps/chosen": -506.99554443359375,
1079
- "logps/rejected": -711.6946411132812,
1080
- "loss": 0.0098,
1081
- "rewards/accuracies": 0.793749988079071,
1082
- "rewards/chosen": -1.9687172174453735,
1083
- "rewards/margins": 1.7158300876617432,
1084
- "rewards/rejected": -3.684546709060669,
1085
  "step": 700
1086
  },
1087
  {
1088
  "epoch": 1.47,
1089
- "eval_logits/chosen": 0.5851482152938843,
1090
- "eval_logits/rejected": 0.6870884299278259,
1091
- "eval_logps/chosen": -507.2100830078125,
1092
- "eval_logps/rejected": -646.5032958984375,
1093
- "eval_loss": 0.02104853093624115,
1094
- "eval_rewards/accuracies": 0.73046875,
1095
- "eval_rewards/chosen": -2.0362541675567627,
1096
- "eval_rewards/margins": 1.3125278949737549,
1097
- "eval_rewards/rejected": -3.3487820625305176,
1098
- "eval_runtime": 74.9024,
1099
- "eval_samples_per_second": 26.701,
1100
- "eval_steps_per_second": 0.427,
1101
  "step": 700
1102
  },
1103
  {
1104
  "epoch": 1.49,
1105
  "learning_rate": 1.8662200511184872e-07,
1106
- "logits/chosen": 0.5449516773223877,
1107
- "logits/rejected": 0.7054388523101807,
1108
- "logps/chosen": -489.23712158203125,
1109
- "logps/rejected": -618.7713623046875,
1110
- "loss": 0.0106,
1111
- "rewards/accuracies": 0.8187500238418579,
1112
- "rewards/chosen": -1.8367626667022705,
1113
- "rewards/margins": 1.529603362083435,
1114
- "rewards/rejected": -3.366366147994995,
1115
  "step": 710
1116
  },
1117
  {
1118
  "epoch": 1.51,
1119
  "learning_rate": 1.725696330273575e-07,
1120
- "logits/chosen": 0.4658689498901367,
1121
- "logits/rejected": 0.5483459234237671,
1122
- "logps/chosen": -499.9456481933594,
1123
- "logps/rejected": -656.5256958007812,
1124
- "loss": 0.0115,
1125
- "rewards/accuracies": 0.8062499761581421,
1126
- "rewards/chosen": -1.7960010766983032,
1127
- "rewards/margins": 1.747243881225586,
1128
- "rewards/rejected": -3.5432448387145996,
1129
  "step": 720
1130
  },
1131
  {
1132
  "epoch": 1.53,
1133
  "learning_rate": 1.589561912846089e-07,
1134
- "logits/chosen": 0.5471647381782532,
1135
- "logits/rejected": 0.7508308291435242,
1136
- "logps/chosen": -486.8019104003906,
1137
- "logps/rejected": -648.6491088867188,
1138
- "loss": 0.012,
1139
- "rewards/accuracies": 0.7875000238418579,
1140
- "rewards/chosen": -1.7438154220581055,
1141
- "rewards/margins": 1.5669682025909424,
1142
- "rewards/rejected": -3.310783863067627,
1143
  "step": 730
1144
  },
1145
  {
1146
  "epoch": 1.55,
1147
  "learning_rate": 1.4579992911531496e-07,
1148
- "logits/chosen": 0.43005961179733276,
1149
- "logits/rejected": 0.5700550079345703,
1150
- "logps/chosen": -507.7278747558594,
1151
- "logps/rejected": -604.3984375,
1152
- "loss": 0.0111,
1153
- "rewards/accuracies": 0.78125,
1154
- "rewards/chosen": -1.9238866567611694,
1155
- "rewards/margins": 1.2898902893066406,
1156
- "rewards/rejected": -3.2137768268585205,
1157
  "step": 740
1158
  },
1159
  {
1160
  "epoch": 1.57,
1161
  "learning_rate": 1.3311848288809813e-07,
1162
- "logits/chosen": 0.5181199312210083,
1163
- "logits/rejected": 0.7587286233901978,
1164
- "logps/chosen": -517.6824951171875,
1165
- "logps/rejected": -604.0447387695312,
1166
- "loss": 0.0114,
1167
- "rewards/accuracies": 0.75,
1168
- "rewards/chosen": -1.8500522375106812,
1169
- "rewards/margins": 1.3126447200775146,
1170
- "rewards/rejected": -3.1626968383789062,
1171
  "step": 750
1172
  },
1173
  {
1174
  "epoch": 1.59,
1175
  "learning_rate": 1.209288524664029e-07,
1176
- "logits/chosen": 0.38817617297172546,
1177
- "logits/rejected": 0.6967793703079224,
1178
- "logps/chosen": -534.35888671875,
1179
- "logps/rejected": -617.2965087890625,
1180
- "loss": 0.0116,
1181
- "rewards/accuracies": 0.7250000238418579,
1182
- "rewards/chosen": -1.964895248413086,
1183
- "rewards/margins": 1.3300648927688599,
1184
- "rewards/rejected": -3.2949604988098145,
1185
  "step": 760
1186
  },
1187
  {
1188
  "epoch": 1.61,
1189
  "learning_rate": 1.0924737841966497e-07,
1190
- "logits/chosen": 0.657262921333313,
1191
- "logits/rejected": 0.8421579599380493,
1192
- "logps/chosen": -479.6363220214844,
1193
- "logps/rejected": -551.2875366210938,
1194
- "loss": 0.0107,
1195
- "rewards/accuracies": 0.6625000238418579,
1196
- "rewards/chosen": -1.988033652305603,
1197
- "rewards/margins": 1.0586892366409302,
1198
- "rewards/rejected": -3.046722888946533,
1199
  "step": 770
1200
  },
1201
  {
1202
  "epoch": 1.63,
1203
  "learning_rate": 9.808972011828054e-08,
1204
- "logits/chosen": 0.534065842628479,
1205
- "logits/rejected": 0.7022881507873535,
1206
- "logps/chosen": -485.6368713378906,
1207
- "logps/rejected": -590.6656494140625,
1208
- "loss": 0.0112,
1209
- "rewards/accuracies": 0.6875,
1210
- "rewards/chosen": -1.9203450679779053,
1211
- "rewards/margins": 1.1289684772491455,
1212
- "rewards/rejected": -3.04931378364563,
1213
  "step": 780
1214
  },
1215
  {
1216
  "epoch": 1.65,
1217
  "learning_rate": 8.747083474174527e-08,
1218
- "logits/chosen": 0.6039692163467407,
1219
- "logits/rejected": 0.7411171197891235,
1220
- "logps/chosen": -460.4100646972656,
1221
- "logps/rejected": -635.5348510742188,
1222
- "loss": 0.01,
1223
- "rewards/accuracies": 0.8187500238418579,
1224
- "rewards/chosen": -1.8810646533966064,
1225
- "rewards/margins": 1.626961350440979,
1226
- "rewards/rejected": -3.508026123046875,
1227
  "step": 790
1228
  },
1229
  {
1230
  "epoch": 1.67,
1231
  "learning_rate": 7.740495722810269e-08,
1232
- "logits/chosen": 0.35127943754196167,
1233
- "logits/rejected": 0.535417914390564,
1234
- "logps/chosen": -558.7343139648438,
1235
- "logps/rejected": -669.262451171875,
1236
- "loss": 0.0101,
1237
- "rewards/accuracies": 0.78125,
1238
- "rewards/chosen": -2.1135106086730957,
1239
- "rewards/margins": 1.4616328477859497,
1240
- "rewards/rejected": -3.575143337249756,
1241
  "step": 800
1242
  },
1243
  {
1244
  "epoch": 1.67,
1245
- "eval_logits/chosen": 0.5936872959136963,
1246
- "eval_logits/rejected": 0.6981104016304016,
1247
- "eval_logps/chosen": -502.8233642578125,
1248
- "eval_logps/rejected": -638.039794921875,
1249
- "eval_loss": 0.021807128563523293,
1250
- "eval_rewards/accuracies": 0.734375,
1251
- "eval_rewards/chosen": -1.992387056350708,
1252
- "eval_rewards/margins": 1.2717599868774414,
1253
- "eval_rewards/rejected": -3.2641468048095703,
1254
- "eval_runtime": 73.4364,
1255
- "eval_samples_per_second": 27.234,
1256
- "eval_steps_per_second": 0.436,
1257
  "step": 800
1258
  },
1259
  {
1260
  "epoch": 1.7,
1261
  "learning_rate": 6.790558119157597e-08,
1262
- "logits/chosen": 0.514393150806427,
1263
- "logits/rejected": 0.7291263341903687,
1264
- "logps/chosen": -525.2340698242188,
1265
- "logps/rejected": -616.1982421875,
1266
- "loss": 0.0106,
1267
- "rewards/accuracies": 0.699999988079071,
1268
- "rewards/chosen": -1.9875078201293945,
1269
- "rewards/margins": 1.2109524011611938,
1270
- "rewards/rejected": -3.198460102081299,
1271
  "step": 810
1272
  },
1273
  {
1274
  "epoch": 1.72,
1275
  "learning_rate": 5.898544083397e-08,
1276
- "logits/chosen": 0.5488702654838562,
1277
- "logits/rejected": 0.6949459910392761,
1278
- "logps/chosen": -483.0582580566406,
1279
- "logps/rejected": -598.3394775390625,
1280
- "loss": 0.0115,
1281
- "rewards/accuracies": 0.71875,
1282
- "rewards/chosen": -1.958407998085022,
1283
- "rewards/margins": 1.3388407230377197,
1284
- "rewards/rejected": -3.2972488403320312,
1285
  "step": 820
1286
  },
1287
  {
1288
  "epoch": 1.74,
1289
  "learning_rate": 5.065649387408705e-08,
1290
- "logits/chosen": 0.5013612508773804,
1291
- "logits/rejected": 0.5702673196792603,
1292
- "logps/chosen": -528.1060180664062,
1293
- "logps/rejected": -625.3968505859375,
1294
- "loss": 0.012,
1295
- "rewards/accuracies": 0.768750011920929,
1296
- "rewards/chosen": -2.1246533393859863,
1297
- "rewards/margins": 1.261916995048523,
1298
- "rewards/rejected": -3.3865699768066406,
1299
  "step": 830
1300
  },
1301
  {
1302
  "epoch": 1.76,
1303
  "learning_rate": 4.292990551804171e-08,
1304
- "logits/chosen": 0.42700353264808655,
1305
- "logits/rejected": 0.6631879806518555,
1306
- "logps/chosen": -502.4617614746094,
1307
- "logps/rejected": -618.699462890625,
1308
- "loss": 0.0114,
1309
- "rewards/accuracies": 0.731249988079071,
1310
- "rewards/chosen": -1.8777034282684326,
1311
- "rewards/margins": 1.4012324810028076,
1312
- "rewards/rejected": -3.2789359092712402,
1313
  "step": 840
1314
  },
1315
  {
1316
  "epoch": 1.78,
1317
  "learning_rate": 3.581603349196371e-08,
1318
- "logits/chosen": 0.5486747026443481,
1319
- "logits/rejected": 0.7163097262382507,
1320
- "logps/chosen": -501.1570739746094,
1321
- "logps/rejected": -640.0488891601562,
1322
- "loss": 0.0102,
1323
- "rewards/accuracies": 0.8062499761581421,
1324
- "rewards/chosen": -1.7478796243667603,
1325
- "rewards/margins": 1.5438367128372192,
1326
- "rewards/rejected": -3.2917160987854004,
1327
  "step": 850
1328
  },
1329
  {
1330
  "epoch": 1.8,
1331
  "learning_rate": 2.9324414157151367e-08,
1332
- "logits/chosen": 0.4644288420677185,
1333
- "logits/rejected": 0.6574305891990662,
1334
- "logps/chosen": -473.5594787597656,
1335
- "logps/rejected": -598.8511962890625,
1336
- "loss": 0.0108,
1337
- "rewards/accuracies": 0.768750011920929,
1338
- "rewards/chosen": -1.788891077041626,
1339
- "rewards/margins": 1.2945648431777954,
1340
- "rewards/rejected": -3.083456039428711,
1341
  "step": 860
1342
  },
1343
  {
1344
  "epoch": 1.82,
1345
  "learning_rate": 2.3463749726290284e-08,
1346
- "logits/chosen": 0.49835458397865295,
1347
- "logits/rejected": 0.6767226457595825,
1348
- "logps/chosen": -532.7442016601562,
1349
- "logps/rejected": -652.2874755859375,
1350
- "loss": 0.0114,
1351
- "rewards/accuracies": 0.75,
1352
- "rewards/chosen": -1.964927315711975,
1353
- "rewards/margins": 1.482194185256958,
1354
- "rewards/rejected": -3.4471213817596436,
1355
  "step": 870
1356
  },
1357
  {
1358
  "epoch": 1.84,
1359
  "learning_rate": 1.824189659787284e-08,
1360
- "logits/chosen": 0.432788610458374,
1361
- "logits/rejected": 0.6490308046340942,
1362
- "logps/chosen": -524.7098388671875,
1363
- "logps/rejected": -685.0862426757812,
1364
- "loss": 0.0119,
1365
- "rewards/accuracies": 0.7875000238418579,
1366
- "rewards/chosen": -1.886285424232483,
1367
- "rewards/margins": 1.6881020069122314,
1368
- "rewards/rejected": -3.5743870735168457,
1369
  "step": 880
1370
  },
1371
  {
1372
  "epoch": 1.86,
1373
  "learning_rate": 1.3665854824458035e-08,
1374
- "logits/chosen": 0.45631036162376404,
1375
- "logits/rejected": 0.6264600157737732,
1376
- "logps/chosen": -514.7445068359375,
1377
- "logps/rejected": -619.1950073242188,
1378
- "loss": 0.011,
1379
- "rewards/accuracies": 0.737500011920929,
1380
- "rewards/chosen": -1.918135643005371,
1381
- "rewards/margins": 1.308524250984192,
1382
- "rewards/rejected": -3.2266597747802734,
1383
  "step": 890
1384
  },
1385
  {
1386
  "epoch": 1.88,
1387
  "learning_rate": 9.741758728888217e-09,
1388
- "logits/chosen": 0.4965108036994934,
1389
- "logits/rejected": 0.7444375157356262,
1390
- "logps/chosen": -533.3218994140625,
1391
- "logps/rejected": -607.4019165039062,
1392
- "loss": 0.0103,
1393
- "rewards/accuracies": 0.768750011920929,
1394
- "rewards/chosen": -2.0954043865203857,
1395
- "rewards/margins": 1.2425765991210938,
1396
- "rewards/rejected": -3.3379809856414795,
1397
  "step": 900
1398
  },
1399
  {
1400
  "epoch": 1.88,
1401
- "eval_logits/chosen": 0.610427975654602,
1402
- "eval_logits/rejected": 0.7138903141021729,
1403
- "eval_logps/chosen": -503.7669677734375,
1404
- "eval_logps/rejected": -638.1763305664062,
1405
- "eval_loss": 0.021610137075185776,
1406
- "eval_rewards/accuracies": 0.734375,
1407
- "eval_rewards/chosen": -2.0018229484558105,
1408
- "eval_rewards/margins": 1.2636897563934326,
1409
- "eval_rewards/rejected": -3.2655129432678223,
1410
- "eval_runtime": 74.571,
1411
- "eval_samples_per_second": 26.82,
1412
- "eval_steps_per_second": 0.429,
1413
  "step": 900
1414
  },
1415
  {
1416
  "epoch": 1.9,
1417
  "learning_rate": 6.474868681043577e-09,
1418
- "logits/chosen": 0.5612128973007202,
1419
- "logits/rejected": 0.8626712560653687,
1420
- "logps/chosen": -493.08551025390625,
1421
- "logps/rejected": -595.2066650390625,
1422
- "loss": 0.0104,
1423
- "rewards/accuracies": 0.7562500238418579,
1424
- "rewards/chosen": -1.881819486618042,
1425
- "rewards/margins": 1.3230464458465576,
1426
- "rewards/rejected": -3.2048659324645996,
1427
  "step": 910
1428
  },
1429
  {
1430
  "epoch": 1.93,
1431
  "learning_rate": 3.869564046156459e-09,
1432
- "logits/chosen": 0.33559730648994446,
1433
- "logits/rejected": 0.5526424646377563,
1434
- "logps/chosen": -545.949951171875,
1435
- "logps/rejected": -630.3507080078125,
1436
- "loss": 0.0106,
1437
- "rewards/accuracies": 0.737500011920929,
1438
- "rewards/chosen": -2.0992321968078613,
1439
- "rewards/margins": 1.3220652341842651,
1440
- "rewards/rejected": -3.421297550201416,
1441
  "step": 920
1442
  },
1443
  {
1444
  "epoch": 1.95,
1445
  "learning_rate": 1.929337314139412e-09,
1446
- "logits/chosen": 0.3952362537384033,
1447
- "logits/rejected": 0.6363921165466309,
1448
- "logps/chosen": -528.9135131835938,
1449
- "logps/rejected": -657.9598388671875,
1450
- "loss": 0.0099,
1451
- "rewards/accuracies": 0.762499988079071,
1452
- "rewards/chosen": -1.9300628900527954,
1453
- "rewards/margins": 1.5338274240493774,
1454
- "rewards/rejected": -3.4638900756835938,
1455
  "step": 930
1456
  },
1457
  {
1458
  "epoch": 1.97,
1459
  "learning_rate": 6.567894177967325e-10,
1460
- "logits/chosen": 0.44664573669433594,
1461
- "logits/rejected": 0.6714277267456055,
1462
- "logps/chosen": -544.5101928710938,
1463
- "logps/rejected": -673.3298950195312,
1464
- "loss": 0.0113,
1465
- "rewards/accuracies": 0.731249988079071,
1466
- "rewards/chosen": -2.0046563148498535,
1467
- "rewards/margins": 1.3531763553619385,
1468
- "rewards/rejected": -3.35783314704895,
1469
  "step": 940
1470
  },
1471
  {
1472
  "epoch": 1.99,
1473
  "learning_rate": 5.3626246194704575e-11,
1474
- "logits/chosen": 0.5509423017501831,
1475
- "logits/rejected": 0.6711171865463257,
1476
- "logps/chosen": -456.3231506347656,
1477
- "logps/rejected": -598.7819213867188,
1478
- "loss": 0.0095,
1479
- "rewards/accuracies": 0.78125,
1480
- "rewards/chosen": -1.7993762493133545,
1481
- "rewards/margins": 1.4040424823760986,
1482
- "rewards/rejected": -3.203418731689453,
1483
  "step": 950
1484
  },
1485
  {
1486
  "epoch": 2.0,
1487
  "step": 954,
1488
  "total_flos": 0.0,
1489
- "train_loss": 0.048426837674039463,
1490
- "train_runtime": 8860.0868,
1491
- "train_samples_per_second": 13.8,
1492
- "train_steps_per_second": 0.108
1493
  }
1494
  ],
1495
  "logging_steps": 10,
 
11
  {
12
  "epoch": 0.02,
13
  "learning_rate": 1.0416666666666667e-07,
14
+ "logits/chosen": 0.29368966817855835,
15
+ "logits/rejected": 0.3178113102912903,
16
+ "logps/chosen": -295.21783447265625,
17
+ "logps/rejected": -290.84619140625,
18
+ "loss": 0.6933,
19
+ "rewards/accuracies": 0.4000000059604645,
20
+ "rewards/chosen": 0.00023447822604794055,
21
+ "rewards/margins": 0.00020264319027774036,
22
+ "rewards/rejected": 3.183506123605184e-05,
23
  "step": 10
24
  },
25
  {
26
  "epoch": 0.04,
27
  "learning_rate": 2.0833333333333333e-07,
28
+ "logits/chosen": 0.25306791067123413,
29
+ "logits/rejected": 0.3252382278442383,
30
+ "logps/chosen": -318.19073486328125,
31
+ "logps/rejected": -289.6706237792969,
32
+ "loss": 0.6929,
33
  "rewards/accuracies": 0.53125,
34
+ "rewards/chosen": 0.00048495858209207654,
35
+ "rewards/margins": 0.0009848512709140778,
36
+ "rewards/rejected": -0.0004998926888220012,
37
  "step": 20
38
  },
39
  {
40
  "epoch": 0.06,
41
  "learning_rate": 3.1249999999999997e-07,
42
+ "logits/chosen": 0.33226653933525085,
43
+ "logits/rejected": 0.3872108459472656,
44
+ "logps/chosen": -296.3697204589844,
45
+ "logps/rejected": -283.0611877441406,
46
+ "loss": 0.692,
47
+ "rewards/accuracies": 0.550000011920929,
48
+ "rewards/chosen": -0.0008094090735539794,
49
+ "rewards/margins": 0.0026363185606896877,
50
+ "rewards/rejected": -0.003445727750658989,
51
  "step": 30
52
  },
53
  {
54
  "epoch": 0.08,
55
  "learning_rate": 4.1666666666666667e-07,
56
+ "logits/chosen": 0.22801101207733154,
57
+ "logits/rejected": 0.32900214195251465,
58
+ "logps/chosen": -305.9015197753906,
59
+ "logps/rejected": -293.1842346191406,
60
+ "loss": 0.6883,
61
+ "rewards/accuracies": 0.643750011920929,
62
+ "rewards/chosen": -0.0027595984283834696,
63
+ "rewards/margins": 0.006589935161173344,
64
+ "rewards/rejected": -0.00934953335672617,
65
  "step": 40
66
  },
67
  {
68
  "epoch": 0.1,
69
  "learning_rate": 5.208333333333334e-07,
70
+ "logits/chosen": 0.27091675996780396,
71
+ "logits/rejected": 0.31866759061813354,
72
+ "logps/chosen": -314.2833557128906,
73
+ "logps/rejected": -307.02532958984375,
74
+ "loss": 0.683,
75
+ "rewards/accuracies": 0.6312500238418579,
76
+ "rewards/chosen": -0.0060192132368683815,
77
+ "rewards/margins": 0.019419629126787186,
78
+ "rewards/rejected": -0.025438839569687843,
79
  "step": 50
80
  },
81
  {
82
  "epoch": 0.13,
83
  "learning_rate": 6.249999999999999e-07,
84
+ "logits/chosen": 0.31704145669937134,
85
+ "logits/rejected": 0.4334793984889984,
86
+ "logps/chosen": -294.429931640625,
87
+ "logps/rejected": -272.87994384765625,
88
+ "loss": 0.6735,
89
  "rewards/accuracies": 0.6312500238418579,
90
+ "rewards/chosen": -0.022902976721525192,
91
+ "rewards/margins": 0.04408121109008789,
92
+ "rewards/rejected": -0.06698418408632278,
93
  "step": 60
94
  },
95
  {
96
  "epoch": 0.15,
97
  "learning_rate": 7.291666666666666e-07,
98
+ "logits/chosen": 0.31964099407196045,
99
+ "logits/rejected": 0.3377896547317505,
100
+ "logps/chosen": -304.6803894042969,
101
+ "logps/rejected": -311.32794189453125,
102
+ "loss": 0.6602,
103
+ "rewards/accuracies": 0.6312500238418579,
104
+ "rewards/chosen": -0.05670947954058647,
105
+ "rewards/margins": 0.05069820210337639,
106
+ "rewards/rejected": -0.10740767419338226,
107
  "step": 70
108
  },
109
  {
110
  "epoch": 0.17,
111
  "learning_rate": 8.333333333333333e-07,
112
+ "logits/chosen": 0.35048729181289673,
113
+ "logits/rejected": 0.4193252921104431,
114
+ "logps/chosen": -306.3404541015625,
115
+ "logps/rejected": -282.4783020019531,
116
+ "loss": 0.6474,
117
+ "rewards/accuracies": 0.643750011920929,
118
+ "rewards/chosen": -0.10853584855794907,
119
+ "rewards/margins": 0.13117292523384094,
120
+ "rewards/rejected": -0.2397087812423706,
121
  "step": 80
122
  },
123
  {
124
  "epoch": 0.19,
125
  "learning_rate": 9.374999999999999e-07,
126
+ "logits/chosen": 0.32813602685928345,
127
+ "logits/rejected": 0.4464220404624939,
128
+ "logps/chosen": -341.1703186035156,
129
+ "logps/rejected": -299.92340087890625,
130
+ "loss": 0.6357,
131
+ "rewards/accuracies": 0.6499999761581421,
132
+ "rewards/chosen": -0.2554694712162018,
133
+ "rewards/margins": 0.12284588813781738,
134
+ "rewards/rejected": -0.37831538915634155,
135
  "step": 90
136
  },
137
  {
138
  "epoch": 0.21,
139
  "learning_rate": 9.999463737538052e-07,
140
+ "logits/chosen": 0.35799938440322876,
141
+ "logits/rejected": 0.3899138271808624,
142
+ "logps/chosen": -318.7712097167969,
143
+ "logps/rejected": -348.5688781738281,
144
+ "loss": 0.6201,
145
+ "rewards/accuracies": 0.6312500238418579,
146
+ "rewards/chosen": -0.3124231994152069,
147
+ "rewards/margins": 0.19829413294792175,
148
+ "rewards/rejected": -0.5107173323631287,
149
  "step": 100
150
  },
151
  {
152
  "epoch": 0.21,
153
+ "eval_logits/chosen": 0.3705582916736603,
154
+ "eval_logits/rejected": 0.4172414541244507,
155
+ "eval_logps/chosen": -331.11236572265625,
156
+ "eval_logps/rejected": -378.240478515625,
157
+ "eval_loss": 0.6252639293670654,
158
+ "eval_rewards/accuracies": 0.703125,
159
+ "eval_rewards/chosen": -0.27527713775634766,
160
+ "eval_rewards/margins": 0.3908771872520447,
161
+ "eval_rewards/rejected": -0.6661543846130371,
162
+ "eval_runtime": 64.993,
163
+ "eval_samples_per_second": 30.773,
164
+ "eval_steps_per_second": 0.492,
165
  "step": 100
166
  },
167
  {
168
  "epoch": 0.23,
169
  "learning_rate": 9.993432105822034e-07,
170
+ "logits/chosen": 0.3002661168575287,
171
+ "logits/rejected": 0.3483879864215851,
172
+ "logps/chosen": -350.9095458984375,
173
+ "logps/rejected": -360.30963134765625,
174
+ "loss": 0.6093,
175
+ "rewards/accuracies": 0.706250011920929,
176
+ "rewards/chosen": -0.37449145317077637,
177
+ "rewards/margins": 0.3816668689250946,
178
+ "rewards/rejected": -0.7561584115028381,
179
  "step": 110
180
  },
181
  {
182
  "epoch": 0.25,
183
  "learning_rate": 9.980706626858607e-07,
184
+ "logits/chosen": 0.2088731825351715,
185
+ "logits/rejected": 0.2899537980556488,
186
+ "logps/chosen": -395.3984069824219,
187
+ "logps/rejected": -433.7286682128906,
188
+ "loss": 0.5905,
189
+ "rewards/accuracies": 0.6875,
190
+ "rewards/chosen": -0.4185329079627991,
191
+ "rewards/margins": 0.4517739713191986,
192
+ "rewards/rejected": -0.8703069686889648,
193
  "step": 120
194
  },
195
  {
196
  "epoch": 0.27,
197
  "learning_rate": 9.961304359538434e-07,
198
+ "logits/chosen": 0.09751267731189728,
199
+ "logits/rejected": 0.22797170281410217,
200
+ "logps/chosen": -374.6502380371094,
201
+ "logps/rejected": -373.14263916015625,
202
+ "loss": 0.5998,
203
+ "rewards/accuracies": 0.6312500238418579,
204
+ "rewards/chosen": -0.4483565390110016,
205
+ "rewards/margins": 0.3120550811290741,
206
+ "rewards/rejected": -0.7604116201400757,
207
  "step": 130
208
  },
209
  {
210
  "epoch": 0.29,
211
  "learning_rate": 9.935251313189563e-07,
212
+ "logits/chosen": 0.23218217492103577,
213
+ "logits/rejected": 0.28826406598091125,
214
+ "logps/chosen": -363.15338134765625,
215
+ "logps/rejected": -381.83795166015625,
216
+ "loss": 0.5847,
217
+ "rewards/accuracies": 0.699999988079071,
218
+ "rewards/chosen": -0.49422675371170044,
219
+ "rewards/margins": 0.38399404287338257,
220
+ "rewards/rejected": -0.8782208561897278,
221
  "step": 140
222
  },
223
  {
224
  "epoch": 0.31,
225
  "learning_rate": 9.902582412711118e-07,
226
+ "logits/chosen": 0.29499703645706177,
227
+ "logits/rejected": 0.49714046716690063,
228
+ "logps/chosen": -342.9471130371094,
229
+ "logps/rejected": -372.1763610839844,
230
+ "loss": 0.5681,
231
+ "rewards/accuracies": 0.699999988079071,
232
+ "rewards/chosen": -0.6186890602111816,
233
+ "rewards/margins": 0.5347500443458557,
234
+ "rewards/rejected": -1.1534390449523926,
235
  "step": 150
236
  },
237
  {
238
  "epoch": 0.33,
239
  "learning_rate": 9.86334145175542e-07,
240
+ "logits/chosen": 0.35773637890815735,
241
+ "logits/rejected": 0.4941268861293793,
242
+ "logps/chosen": -370.3203125,
243
+ "logps/rejected": -428.2557067871094,
244
+ "loss": 0.5792,
245
+ "rewards/accuracies": 0.7437499761581421,
246
+ "rewards/chosen": -0.7682583928108215,
247
+ "rewards/margins": 0.6092099547386169,
248
+ "rewards/rejected": -1.377468228340149,
249
  "step": 160
250
  },
251
  {
252
  "epoch": 0.36,
253
  "learning_rate": 9.817581034021272e-07,
254
+ "logits/chosen": 0.19401590526103973,
255
+ "logits/rejected": 0.3156794607639313,
256
+ "logps/chosen": -415.65313720703125,
257
+ "logps/rejected": -425.68963623046875,
258
+ "loss": 0.5747,
259
+ "rewards/accuracies": 0.731249988079071,
260
+ "rewards/chosen": -0.5680743455886841,
261
+ "rewards/margins": 0.4286484122276306,
262
+ "rewards/rejected": -0.9967228174209595,
263
  "step": 170
264
  },
265
  {
266
  "epoch": 0.38,
267
  "learning_rate": 9.765362502737097e-07,
268
+ "logits/chosen": 0.2572034001350403,
269
+ "logits/rejected": 0.27328386902809143,
270
+ "logps/chosen": -358.0533447265625,
271
+ "logps/rejected": -398.5332946777344,
272
+ "loss": 0.5634,
273
+ "rewards/accuracies": 0.699999988079071,
274
+ "rewards/chosen": -0.644694447517395,
275
+ "rewards/margins": 0.5251447558403015,
276
+ "rewards/rejected": -1.1698391437530518,
277
  "step": 180
278
  },
279
  {
280
  "epoch": 0.4,
281
  "learning_rate": 9.706755858428485e-07,
282
+ "logits/chosen": 0.3962785303592682,
283
+ "logits/rejected": 0.4539657235145569,
284
+ "logps/chosen": -347.5164794921875,
285
+ "logps/rejected": -378.83184814453125,
286
+ "loss": 0.5423,
287
+ "rewards/accuracies": 0.71875,
288
+ "rewards/chosen": -0.7963652610778809,
289
+ "rewards/margins": 0.5720622539520264,
290
+ "rewards/rejected": -1.3684275150299072,
291
  "step": 190
292
  },
293
  {
294
  "epoch": 0.42,
295
  "learning_rate": 9.641839665080363e-07,
296
+ "logits/chosen": 0.3198946714401245,
297
+ "logits/rejected": 0.4063253402709961,
298
+ "logps/chosen": -352.0743713378906,
299
+ "logps/rejected": -419.7388610839844,
300
+ "loss": 0.5547,
301
+ "rewards/accuracies": 0.7250000238418579,
302
+ "rewards/chosen": -0.7651049494743347,
303
+ "rewards/margins": 0.6531444787979126,
304
+ "rewards/rejected": -1.418249487876892,
305
  "step": 200
306
  },
307
  {
308
  "epoch": 0.42,
309
+ "eval_logits/chosen": 0.3908616304397583,
310
+ "eval_logits/rejected": 0.4261176884174347,
311
+ "eval_logps/chosen": -373.46612548828125,
312
+ "eval_logps/rejected": -458.88629150390625,
313
+ "eval_loss": 0.5549300312995911,
314
+ "eval_rewards/accuracies": 0.765625,
315
+ "eval_rewards/chosen": -0.6988146305084229,
316
+ "eval_rewards/margins": 0.7737974524497986,
317
+ "eval_rewards/rejected": -1.4726121425628662,
318
+ "eval_runtime": 65.2313,
319
+ "eval_samples_per_second": 30.66,
320
+ "eval_steps_per_second": 0.491,
321
  "step": 200
322
  },
323
  {
324
  "epoch": 0.44,
325
  "learning_rate": 9.570700944819582e-07,
326
+ "logits/chosen": 0.3505176901817322,
327
+ "logits/rejected": 0.42375579476356506,
328
+ "logps/chosen": -397.61199951171875,
329
+ "logps/rejected": -454.89776611328125,
330
+ "loss": 0.5265,
331
+ "rewards/accuracies": 0.762499988079071,
332
+ "rewards/chosen": -0.7523492574691772,
333
+ "rewards/margins": 0.7344074845314026,
334
+ "rewards/rejected": -1.486756682395935,
335
  "step": 210
336
  },
337
  {
338
  "epoch": 0.46,
339
  "learning_rate": 9.493435061259129e-07,
340
+ "logits/chosen": 0.2726442813873291,
341
+ "logits/rejected": 0.4434526860713959,
342
+ "logps/chosen": -410.77667236328125,
343
+ "logps/rejected": -431.207275390625,
344
+ "loss": 0.555,
345
+ "rewards/accuracies": 0.731249988079071,
346
+ "rewards/chosen": -0.8984044194221497,
347
+ "rewards/margins": 0.5839776992797852,
348
+ "rewards/rejected": -1.48238205909729,
349
  "step": 220
350
  },
351
  {
352
  "epoch": 0.48,
353
  "learning_rate": 9.4101455916603e-07,
354
+ "logits/chosen": 0.28837597370147705,
355
+ "logits/rejected": 0.35526323318481445,
356
+ "logps/chosen": -363.0335998535156,
357
+ "logps/rejected": -436.46612548828125,
358
+ "loss": 0.5498,
359
+ "rewards/accuracies": 0.7124999761581421,
360
+ "rewards/chosen": -0.8101651072502136,
361
+ "rewards/margins": 0.737695574760437,
362
+ "rewards/rejected": -1.5478605031967163,
363
  "step": 230
364
  },
365
  {
366
  "epoch": 0.5,
367
  "learning_rate": 9.320944188084241e-07,
368
+ "logits/chosen": 0.23826150596141815,
369
+ "logits/rejected": 0.285171240568161,
370
+ "logps/chosen": -432.6297912597656,
371
+ "logps/rejected": -497.56341552734375,
372
+ "loss": 0.5392,
373
+ "rewards/accuracies": 0.7437499761581421,
374
+ "rewards/chosen": -0.7939236164093018,
375
+ "rewards/margins": 0.7111212611198425,
376
+ "rewards/rejected": -1.505044937133789,
377
  "step": 240
378
  },
379
  {
380
  "epoch": 0.52,
381
  "learning_rate": 9.225950427718974e-07,
382
+ "logits/chosen": 0.2762988209724426,
383
+ "logits/rejected": 0.31130319833755493,
384
+ "logps/chosen": -399.399169921875,
385
+ "logps/rejected": -445.702880859375,
386
+ "loss": 0.5465,
387
+ "rewards/accuracies": 0.6875,
388
+ "rewards/chosen": -0.83611661195755,
389
+ "rewards/margins": 0.7073522806167603,
390
+ "rewards/rejected": -1.5434690713882446,
391
  "step": 250
392
  },
393
  {
394
  "epoch": 0.54,
395
  "learning_rate": 9.125291652582547e-07,
396
+ "logits/chosen": 0.1327328383922577,
397
+ "logits/rejected": 0.3085227310657501,
398
+ "logps/chosen": -436.08135986328125,
399
+ "logps/rejected": -457.634765625,
400
+ "loss": 0.5194,
401
+ "rewards/accuracies": 0.699999988079071,
402
+ "rewards/chosen": -0.9428914785385132,
403
+ "rewards/margins": 0.6950392723083496,
404
+ "rewards/rejected": -1.6379308700561523,
405
  "step": 260
406
  },
407
  {
408
  "epoch": 0.57,
409
  "learning_rate": 9.019102798817195e-07,
410
+ "logits/chosen": 0.23745720088481903,
411
+ "logits/rejected": 0.34172096848487854,
412
+ "logps/chosen": -421.8299865722656,
413
+ "logps/rejected": -466.4856872558594,
414
+ "loss": 0.5496,
415
  "rewards/accuracies": 0.71875,
416
+ "rewards/chosen": -0.9325121641159058,
417
+ "rewards/margins": 0.7680062651634216,
418
+ "rewards/rejected": -1.7005186080932617,
419
  "step": 270
420
  },
421
  {
422
  "epoch": 0.59,
423
  "learning_rate": 8.90752621580335e-07,
424
+ "logits/chosen": 0.16251161694526672,
425
+ "logits/rejected": 0.2581509053707123,
426
+ "logps/chosen": -418.5828552246094,
427
+ "logps/rejected": -507.22412109375,
428
+ "loss": 0.5168,
429
+ "rewards/accuracies": 0.7250000238418579,
430
+ "rewards/chosen": -1.113261342048645,
431
+ "rewards/margins": 0.7757157683372498,
432
+ "rewards/rejected": -1.88897705078125,
433
  "step": 280
434
  },
435
  {
436
  "epoch": 0.61,
437
  "learning_rate": 8.79071147533597e-07,
438
+ "logits/chosen": 0.18345972895622253,
439
+ "logits/rejected": 0.24752414226531982,
440
+ "logps/chosen": -374.2388916015625,
441
+ "logps/rejected": -421.7548828125,
442
+ "loss": 0.5452,
443
+ "rewards/accuracies": 0.699999988079071,
444
+ "rewards/chosen": -0.7729798555374146,
445
+ "rewards/margins": 0.6754422187805176,
446
+ "rewards/rejected": -1.448421835899353,
447
  "step": 290
448
  },
449
  {
450
  "epoch": 0.63,
451
  "learning_rate": 8.668815171119019e-07,
452
+ "logits/chosen": 0.11917382478713989,
453
+ "logits/rejected": 0.2862890362739563,
454
+ "logps/chosen": -396.6165466308594,
455
+ "logps/rejected": -400.9344787597656,
456
+ "loss": 0.5343,
457
+ "rewards/accuracies": 0.75,
458
+ "rewards/chosen": -0.8496394157409668,
459
+ "rewards/margins": 0.5671547055244446,
460
+ "rewards/rejected": -1.4167941808700562,
461
  "step": 300
462
  },
463
  {
464
  "epoch": 0.63,
465
+ "eval_logits/chosen": 0.24486932158470154,
466
+ "eval_logits/rejected": 0.2851215898990631,
467
+ "eval_logps/chosen": -384.0199279785156,
468
+ "eval_logps/rejected": -476.3627624511719,
469
+ "eval_loss": 0.5315821766853333,
470
+ "eval_rewards/accuracies": 0.765625,
471
+ "eval_rewards/chosen": -0.8043524026870728,
472
+ "eval_rewards/margins": 0.8430246114730835,
473
+ "eval_rewards/rejected": -1.6473771333694458,
474
+ "eval_runtime": 64.8898,
475
+ "eval_samples_per_second": 30.822,
476
+ "eval_steps_per_second": 0.493,
477
  "step": 300
478
  },
479
  {
480
  "epoch": 0.65,
481
  "learning_rate": 8.54200070884685e-07,
482
+ "logits/chosen": 0.21563191711902618,
483
+ "logits/rejected": 0.1952591836452484,
484
+ "logps/chosen": -438.91552734375,
485
+ "logps/rejected": -499.61395263671875,
486
+ "loss": 0.5567,
487
+ "rewards/accuracies": 0.699999988079071,
488
+ "rewards/chosen": -1.173762559890747,
489
+ "rewards/margins": 0.5995947122573853,
490
+ "rewards/rejected": -1.7733571529388428,
491
  "step": 310
492
  },
493
  {
494
  "epoch": 0.67,
495
  "learning_rate": 8.410438087153911e-07,
496
+ "logits/chosen": 0.031750187277793884,
497
+ "logits/rejected": 0.14312420785427094,
498
+ "logps/chosen": -457.0936584472656,
499
+ "logps/rejected": -459.15411376953125,
500
+ "loss": 0.5366,
501
+ "rewards/accuracies": 0.612500011920929,
502
+ "rewards/chosen": -1.0222933292388916,
503
+ "rewards/margins": 0.4668423533439636,
504
+ "rewards/rejected": -1.4891356229782104,
505
  "step": 320
506
  },
507
  {
508
  "epoch": 0.69,
509
  "learning_rate": 8.274303669726426e-07,
510
+ "logits/chosen": 0.07745673507452011,
511
+ "logits/rejected": 0.07082104682922363,
512
+ "logps/chosen": -399.32464599609375,
513
+ "logps/rejected": -523.4277954101562,
514
+ "loss": 0.532,
515
+ "rewards/accuracies": 0.7749999761581421,
516
+ "rewards/chosen": -0.9549884796142578,
517
+ "rewards/margins": 0.8862001299858093,
518
+ "rewards/rejected": -1.8411887884140015,
519
  "step": 330
520
  },
521
  {
522
  "epoch": 0.71,
523
  "learning_rate": 8.133779948881513e-07,
524
+ "logits/chosen": 0.10294970124959946,
525
+ "logits/rejected": 0.09352216869592667,
526
+ "logps/chosen": -409.33770751953125,
527
+ "logps/rejected": -510.10089111328125,
528
+ "loss": 0.5217,
529
+ "rewards/accuracies": 0.7124999761581421,
530
+ "rewards/chosen": -1.1990526914596558,
531
+ "rewards/margins": 0.7596696615219116,
532
+ "rewards/rejected": -1.958722472190857,
533
  "step": 340
534
  },
535
  {
536
  "epoch": 0.73,
537
  "learning_rate": 7.989055300930704e-07,
538
+ "logits/chosen": 0.16737070679664612,
539
+ "logits/rejected": 0.21901166439056396,
540
+ "logps/chosen": -437.2059020996094,
541
+ "logps/rejected": -538.3087158203125,
542
+ "loss": 0.5091,
543
+ "rewards/accuracies": 0.7437499761581421,
544
+ "rewards/chosen": -1.2976617813110352,
545
+ "rewards/margins": 0.8197441101074219,
546
+ "rewards/rejected": -2.117405891418457,
547
  "step": 350
548
  },
549
  {
550
  "epoch": 0.75,
551
  "learning_rate": 7.840323733655778e-07,
552
+ "logits/chosen": 0.1267612874507904,
553
+ "logits/rejected": 0.20460394024848938,
554
+ "logps/chosen": -465.1019592285156,
555
+ "logps/rejected": -518.5805053710938,
556
+ "loss": 0.5185,
557
+ "rewards/accuracies": 0.7562500238418579,
558
+ "rewards/chosen": -1.174440860748291,
559
+ "rewards/margins": 0.9074214696884155,
560
+ "rewards/rejected": -2.081862211227417,
561
  "step": 360
562
  },
563
  {
564
  "epoch": 0.77,
565
  "learning_rate": 7.687784626235447e-07,
566
+ "logits/chosen": 0.18740633130073547,
567
+ "logits/rejected": 0.27840983867645264,
568
+ "logps/chosen": -468.21807861328125,
569
+ "logps/rejected": -493.06646728515625,
570
+ "loss": 0.5158,
571
+ "rewards/accuracies": 0.675000011920929,
572
+ "rewards/chosen": -1.1525232791900635,
573
+ "rewards/margins": 0.7130603790283203,
574
+ "rewards/rejected": -1.8655836582183838,
575
  "step": 370
576
  },
577
  {
578
  "epoch": 0.8,
579
  "learning_rate": 7.531642461971514e-07,
580
+ "logits/chosen": 0.15459200739860535,
581
+ "logits/rejected": 0.23283176124095917,
582
+ "logps/chosen": -453.07904052734375,
583
+ "logps/rejected": -535.761474609375,
584
+ "loss": 0.5392,
585
+ "rewards/accuracies": 0.7562500238418579,
586
+ "rewards/chosen": -1.2102587223052979,
587
+ "rewards/margins": 1.0239769220352173,
588
+ "rewards/rejected": -2.2342355251312256,
589
  "step": 380
590
  },
591
  {
592
  "epoch": 0.82,
593
  "learning_rate": 7.372106554172801e-07,
594
+ "logits/chosen": 0.19951777160167694,
595
+ "logits/rejected": 0.2895793318748474,
596
+ "logps/chosen": -411.4781799316406,
597
+ "logps/rejected": -466.41253662109375,
598
+ "loss": 0.5356,
599
+ "rewards/accuracies": 0.7124999761581421,
600
+ "rewards/chosen": -0.8807674646377563,
601
+ "rewards/margins": 0.7255697846412659,
602
+ "rewards/rejected": -1.606337308883667,
603
  "step": 390
604
  },
605
  {
606
  "epoch": 0.84,
607
  "learning_rate": 7.209390765564318e-07,
608
+ "logits/chosen": 0.1757899820804596,
609
+ "logits/rejected": 0.23791635036468506,
610
+ "logps/chosen": -390.8957214355469,
611
+ "logps/rejected": -478.49407958984375,
612
+ "loss": 0.5323,
613
+ "rewards/accuracies": 0.7437499761581421,
614
+ "rewards/chosen": -0.9380720257759094,
615
+ "rewards/margins": 0.7229949235916138,
616
+ "rewards/rejected": -1.661067008972168,
617
  "step": 400
618
  },
619
  {
620
  "epoch": 0.84,
621
+ "eval_logits/chosen": 0.25136542320251465,
622
+ "eval_logits/rejected": 0.28342366218566895,
623
+ "eval_logps/chosen": -394.2620849609375,
624
+ "eval_logps/rejected": -494.46002197265625,
625
+ "eval_loss": 0.521114706993103,
626
+ "eval_rewards/accuracies": 0.78125,
627
+ "eval_rewards/chosen": -0.90677410364151,
628
+ "eval_rewards/margins": 0.9215754270553589,
629
+ "eval_rewards/rejected": -1.8283497095108032,
630
+ "eval_runtime": 65.2477,
631
+ "eval_samples_per_second": 30.652,
632
+ "eval_steps_per_second": 0.49,
633
  "step": 400
634
  },
635
  {
636
  "epoch": 0.86,
637
  "learning_rate": 7.043713221597773e-07,
638
+ "logits/chosen": 0.1401471644639969,
639
+ "logits/rejected": 0.17819848656654358,
640
+ "logps/chosen": -401.8942565917969,
641
+ "logps/rejected": -474.2579650878906,
642
+ "loss": 0.5225,
643
+ "rewards/accuracies": 0.7749999761581421,
644
+ "rewards/chosen": -0.8892021179199219,
645
+ "rewards/margins": 0.906479001045227,
646
+ "rewards/rejected": -1.7956812381744385,
647
  "step": 410
648
  },
649
  {
650
  "epoch": 0.88,
651
  "learning_rate": 6.875296018047809e-07,
652
+ "logits/chosen": 0.1304786652326584,
653
+ "logits/rejected": 0.15569528937339783,
654
+ "logps/chosen": -400.5438537597656,
655
+ "logps/rejected": -468.2953186035156,
656
+ "loss": 0.5065,
657
+ "rewards/accuracies": 0.8187500238418579,
658
+ "rewards/chosen": -0.9208317995071411,
659
+ "rewards/margins": 0.9665653109550476,
660
+ "rewards/rejected": -1.887397050857544,
661
  "step": 420
662
  },
663
  {
664
  "epoch": 0.9,
665
  "learning_rate": 6.704364923285857e-07,
666
+ "logits/chosen": 0.09775003790855408,
667
+ "logits/rejected": 0.1501173973083496,
668
+ "logps/chosen": -476.73065185546875,
669
+ "logps/rejected": -533.7389526367188,
670
+ "loss": 0.5285,
671
+ "rewards/accuracies": 0.71875,
672
+ "rewards/chosen": -1.26286780834198,
673
+ "rewards/margins": 0.8789494633674622,
674
+ "rewards/rejected": -2.141817569732666,
675
  "step": 430
676
  },
677
  {
678
  "epoch": 0.92,
679
  "learning_rate": 6.531149075630796e-07,
680
+ "logits/chosen": 0.1202029138803482,
681
+ "logits/rejected": 0.1756385862827301,
682
+ "logps/chosen": -400.1629333496094,
683
+ "logps/rejected": -469.0196838378906,
684
+ "loss": 0.5074,
685
+ "rewards/accuracies": 0.6812499761581421,
686
+ "rewards/chosen": -1.0582826137542725,
687
+ "rewards/margins": 0.9171028137207031,
688
+ "rewards/rejected": -1.975385308265686,
689
  "step": 440
690
  },
691
  {
692
  "epoch": 0.94,
693
  "learning_rate": 6.355880676182085e-07,
694
+ "logits/chosen": 0.01758761703968048,
695
+ "logits/rejected": 0.15642888844013214,
696
+ "logps/chosen": -443.065185546875,
697
+ "logps/rejected": -484.41473388671875,
698
+ "loss": 0.5295,
699
+ "rewards/accuracies": 0.6875,
700
+ "rewards/chosen": -1.004433035850525,
701
+ "rewards/margins": 0.8090255856513977,
702
+ "rewards/rejected": -1.8134586811065674,
703
  "step": 450
704
  },
705
  {
706
  "epoch": 0.96,
707
  "learning_rate": 6.178794677547137e-07,
708
+ "logits/chosen": 0.012063628062605858,
709
+ "logits/rejected": 0.10572747141122818,
710
+ "logps/chosen": -408.4104919433594,
711
+ "logps/rejected": -449.46881103515625,
712
+ "loss": 0.5267,
713
+ "rewards/accuracies": 0.7437499761581421,
714
+ "rewards/chosen": -1.0622318983078003,
715
+ "rewards/margins": 0.6900007724761963,
716
+ "rewards/rejected": -1.752232551574707,
717
  "step": 460
718
  },
719
  {
720
  "epoch": 0.98,
721
  "learning_rate": 6.000128468880222e-07,
722
+ "logits/chosen": -0.05889149755239487,
723
+ "logits/rejected": -0.014351313933730125,
724
+ "logps/chosen": -449.07061767578125,
725
+ "logps/rejected": -538.423828125,
726
+ "loss": 0.5042,
727
+ "rewards/accuracies": 0.762499988079071,
728
+ "rewards/chosen": -1.1122030019760132,
729
+ "rewards/margins": 0.9293983578681946,
730
+ "rewards/rejected": -2.0416014194488525,
731
  "step": 470
732
  },
733
  {
734
  "epoch": 1.0,
735
  "learning_rate": 5.820121557655108e-07,
736
+ "logits/chosen": -0.045923542231321335,
737
+ "logits/rejected": 0.03086056187748909,
738
+ "logps/chosen": -422.5347595214844,
739
+ "logps/rejected": -529.2613525390625,
740
+ "loss": 0.4682,
741
+ "rewards/accuracies": 0.768750011920929,
742
+ "rewards/chosen": -1.275895118713379,
743
+ "rewards/margins": 1.037326455116272,
744
+ "rewards/rejected": -2.3132214546203613,
745
  "step": 480
746
  },
747
  {
748
  "epoch": 1.03,
749
  "learning_rate": 5.639015248598023e-07,
750
+ "logits/chosen": -0.05794327333569527,
751
+ "logits/rejected": -0.09440571069717407,
752
+ "logps/chosen": -441.08990478515625,
753
+ "logps/rejected": -601.30419921875,
754
+ "loss": 0.3493,
755
+ "rewards/accuracies": 0.862500011920929,
756
+ "rewards/chosen": -1.3565930128097534,
757
+ "rewards/margins": 1.8090267181396484,
758
+ "rewards/rejected": -3.1656198501586914,
759
  "step": 490
760
  },
761
  {
762
  "epoch": 1.05,
763
  "learning_rate": 5.457052320211339e-07,
764
+ "logits/chosen": -0.23803594708442688,
765
+ "logits/rejected": -0.29828980565071106,
766
+ "logps/chosen": -474.3367614746094,
767
+ "logps/rejected": -656.8623046875,
768
+ "loss": 0.352,
769
+ "rewards/accuracies": 0.8187500238418579,
770
+ "rewards/chosen": -1.730020523071289,
771
+ "rewards/margins": 1.8847625255584717,
772
+ "rewards/rejected": -3.6147830486297607,
773
  "step": 500
774
  },
775
  {
776
  "epoch": 1.05,
777
+ "eval_logits/chosen": -0.06535135954618454,
778
+ "eval_logits/rejected": -0.08458372950553894,
779
+ "eval_logps/chosen": -498.9117431640625,
780
+ "eval_logps/rejected": -653.2899169921875,
781
+ "eval_loss": 0.525809109210968,
782
+ "eval_rewards/accuracies": 0.796875,
783
+ "eval_rewards/chosen": -1.9532711505889893,
784
+ "eval_rewards/margins": 1.4633771181106567,
785
+ "eval_rewards/rejected": -3.4166483879089355,
786
+ "eval_runtime": 64.5447,
787
+ "eval_samples_per_second": 30.986,
788
+ "eval_steps_per_second": 0.496,
789
  "step": 500
790
  },
791
  {
792
  "epoch": 1.07,
793
  "learning_rate": 5.274476699321637e-07,
794
+ "logits/chosen": -0.17468394339084625,
795
+ "logits/rejected": -0.19271844625473022,
796
+ "logps/chosen": -464.8907165527344,
797
+ "logps/rejected": -632.8477172851562,
798
+ "loss": 0.3423,
799
+ "rewards/accuracies": 0.893750011920929,
800
+ "rewards/chosen": -1.695770263671875,
801
+ "rewards/margins": 1.8958046436309814,
802
+ "rewards/rejected": -3.5915749073028564,
803
  "step": 510
804
  },
805
  {
806
  "epoch": 1.09,
807
  "learning_rate": 5.091533134088387e-07,
808
+ "logits/chosen": -0.23788562417030334,
809
+ "logits/rejected": -0.20414999127388,
810
+ "logps/chosen": -485.94415283203125,
811
+ "logps/rejected": -640.9654541015625,
812
+ "loss": 0.3236,
813
+ "rewards/accuracies": 0.8999999761581421,
814
+ "rewards/chosen": -1.8264538049697876,
815
+ "rewards/margins": 1.7633371353149414,
816
+ "rewards/rejected": -3.5897908210754395,
817
  "step": 520
818
  },
819
  {
820
  "epoch": 1.11,
821
  "learning_rate": 4.908466865911614e-07,
822
+ "logits/chosen": -0.11999205499887466,
823
+ "logits/rejected": -0.09423510730266571,
824
+ "logps/chosen": -487.30810546875,
825
+ "logps/rejected": -646.9547119140625,
826
+ "loss": 0.3424,
827
+ "rewards/accuracies": 0.84375,
828
+ "rewards/chosen": -1.7824156284332275,
829
+ "rewards/margins": 1.7626768350601196,
830
+ "rewards/rejected": -3.5450921058654785,
831
  "step": 530
832
  },
833
  {
834
  "epoch": 1.13,
835
  "learning_rate": 4.7255233006783624e-07,
836
+ "logits/chosen": -0.16903451085090637,
837
+ "logits/rejected": -0.06715662032365799,
838
+ "logps/chosen": -530.5585327148438,
839
+ "logps/rejected": -645.1633911132812,
840
+ "loss": 0.3301,
841
+ "rewards/accuracies": 0.862500011920929,
842
+ "rewards/chosen": -1.7870140075683594,
843
+ "rewards/margins": 1.8271121978759766,
844
+ "rewards/rejected": -3.614126682281494,
845
  "step": 540
846
  },
847
  {
848
  "epoch": 1.15,
849
  "learning_rate": 4.5429476797886617e-07,
850
+ "logits/chosen": -0.03496643900871277,
851
+ "logits/rejected": -0.04560618847608566,
852
+ "logps/chosen": -483.15203857421875,
853
+ "logps/rejected": -634.9447631835938,
854
+ "loss": 0.3501,
855
+ "rewards/accuracies": 0.8187500238418579,
856
+ "rewards/chosen": -1.8454551696777344,
857
+ "rewards/margins": 1.5336250066757202,
858
+ "rewards/rejected": -3.379080295562744,
859
  "step": 550
860
  },
861
  {
862
  "epoch": 1.17,
863
  "learning_rate": 4.3609847514019763e-07,
864
+ "logits/chosen": 0.06872721016407013,
865
+ "logits/rejected": 0.0859331339597702,
866
+ "logps/chosen": -506.5009765625,
867
+ "logps/rejected": -646.7543334960938,
868
+ "loss": 0.3359,
869
+ "rewards/accuracies": 0.875,
870
+ "rewards/chosen": -1.7461020946502686,
871
+ "rewards/margins": 1.8654359579086304,
872
+ "rewards/rejected": -3.6115379333496094,
873
  "step": 560
874
  },
875
  {
876
  "epoch": 1.19,
877
  "learning_rate": 4.179878442344892e-07,
878
+ "logits/chosen": -0.04321649298071861,
879
+ "logits/rejected": 0.051123034209012985,
880
+ "logps/chosen": -496.735595703125,
881
+ "logps/rejected": -647.29150390625,
882
+ "loss": 0.3464,
883
+ "rewards/accuracies": 0.8062499761581421,
884
+ "rewards/chosen": -2.0714824199676514,
885
+ "rewards/margins": 1.762717843055725,
886
+ "rewards/rejected": -3.834200620651245,
887
  "step": 570
888
  },
889
  {
890
  "epoch": 1.21,
891
  "learning_rate": 3.9998715311197783e-07,
892
+ "logits/chosen": -0.004042728338390589,
893
+ "logits/rejected": 0.04070080816745758,
894
+ "logps/chosen": -512.9417724609375,
895
+ "logps/rejected": -696.821044921875,
896
+ "loss": 0.3365,
897
+ "rewards/accuracies": 0.862500011920929,
898
+ "rewards/chosen": -2.097195863723755,
899
+ "rewards/margins": 1.831916093826294,
900
+ "rewards/rejected": -3.9291114807128906,
901
  "step": 580
902
  },
903
  {
904
  "epoch": 1.23,
905
  "learning_rate": 3.821205322452863e-07,
906
+ "logits/chosen": 0.09215477854013443,
907
+ "logits/rejected": 0.05737446993589401,
908
+ "logps/chosen": -525.1434936523438,
909
+ "logps/rejected": -705.9063110351562,
910
+ "loss": 0.3158,
911
+ "rewards/accuracies": 0.856249988079071,
912
+ "rewards/chosen": -2.0721964836120605,
913
+ "rewards/margins": 1.9721500873565674,
914
+ "rewards/rejected": -4.044346809387207,
915
  "step": 590
916
  },
917
  {
918
  "epoch": 1.26,
919
  "learning_rate": 3.6441193238179146e-07,
920
+ "logits/chosen": 0.07227401435375214,
921
+ "logits/rejected": 0.03988388180732727,
922
+ "logps/chosen": -549.4615478515625,
923
+ "logps/rejected": -771.7275390625,
924
+ "loss": 0.3342,
925
+ "rewards/accuracies": 0.856249988079071,
926
+ "rewards/chosen": -2.2555530071258545,
927
+ "rewards/margins": 2.014453649520874,
928
+ "rewards/rejected": -4.2700066566467285,
929
  "step": 600
930
  },
931
  {
932
  "epoch": 1.26,
933
+ "eval_logits/chosen": 0.1344175487756729,
934
+ "eval_logits/rejected": 0.11280365288257599,
935
+ "eval_logps/chosen": -534.8101196289062,
936
+ "eval_logps/rejected": -684.085693359375,
937
+ "eval_loss": 0.5267595648765564,
938
+ "eval_rewards/accuracies": 0.79296875,
939
+ "eval_rewards/chosen": -2.3122546672821045,
940
+ "eval_rewards/margins": 1.412351369857788,
941
+ "eval_rewards/rejected": -3.7246060371398926,
942
+ "eval_runtime": 64.3013,
943
+ "eval_samples_per_second": 31.104,
944
+ "eval_steps_per_second": 0.498,
945
  "step": 600
946
  },
947
  {
948
  "epoch": 1.28,
949
  "learning_rate": 3.4688509243692034e-07,
950
+ "logits/chosen": 0.021421348676085472,
951
+ "logits/rejected": 0.08866464346647263,
952
+ "logps/chosen": -535.4236450195312,
953
+ "logps/rejected": -694.8360595703125,
954
+ "loss": 0.344,
955
+ "rewards/accuracies": 0.8687499761581421,
956
+ "rewards/chosen": -2.009089946746826,
957
+ "rewards/margins": 1.8043378591537476,
958
+ "rewards/rejected": -3.8134284019470215,
959
  "step": 610
960
  },
961
  {
962
  "epoch": 1.3,
963
  "learning_rate": 3.295635076714144e-07,
964
+ "logits/chosen": -0.06299210339784622,
965
+ "logits/rejected": -0.04097691923379898,
966
+ "logps/chosen": -499.0265197753906,
967
+ "logps/rejected": -653.1188354492188,
968
+ "loss": 0.3317,
969
+ "rewards/accuracies": 0.862500011920929,
970
+ "rewards/chosen": -1.9393279552459717,
971
+ "rewards/margins": 1.6952606439590454,
972
+ "rewards/rejected": -3.6345887184143066,
973
  "step": 620
974
  },
975
  {
976
  "epoch": 1.32,
977
  "learning_rate": 3.12470398195219e-07,
978
+ "logits/chosen": -0.013870243914425373,
979
+ "logits/rejected": 0.0671583041548729,
980
+ "logps/chosen": -512.1448974609375,
981
+ "logps/rejected": -680.2833251953125,
982
+ "loss": 0.3283,
983
+ "rewards/accuracies": 0.887499988079071,
984
+ "rewards/chosen": -1.9993736743927002,
985
+ "rewards/margins": 1.9316644668579102,
986
+ "rewards/rejected": -3.9310379028320312,
987
  "step": 630
988
  },
989
  {
990
  "epoch": 1.34,
991
  "learning_rate": 2.956286778402226e-07,
992
+ "logits/chosen": -0.11906696856021881,
993
+ "logits/rejected": -0.17396704852581024,
994
+ "logps/chosen": -495.10345458984375,
995
+ "logps/rejected": -677.45556640625,
996
+ "loss": 0.319,
997
+ "rewards/accuracies": 0.8812500238418579,
998
+ "rewards/chosen": -2.2106266021728516,
999
+ "rewards/margins": 1.9618394374847412,
1000
+ "rewards/rejected": -4.172466278076172,
1001
  "step": 640
1002
  },
1003
  {
1004
  "epoch": 1.36,
1005
  "learning_rate": 2.7906092344356826e-07,
1006
+ "logits/chosen": -0.07961982488632202,
1007
+ "logits/rejected": -0.12522803246974945,
1008
+ "logps/chosen": -516.12890625,
1009
+ "logps/rejected": -696.549560546875,
1010
+ "loss": 0.3233,
1011
+ "rewards/accuracies": 0.8374999761581421,
1012
+ "rewards/chosen": -2.1828126907348633,
1013
+ "rewards/margins": 1.8567641973495483,
1014
+ "rewards/rejected": -4.039577007293701,
1015
  "step": 650
1016
  },
1017
  {
1018
  "epoch": 1.38,
1019
  "learning_rate": 2.6278934458271996e-07,
1020
+ "logits/chosen": -0.15696656703948975,
1021
+ "logits/rejected": -0.12818947434425354,
1022
+ "logps/chosen": -568.789794921875,
1023
+ "logps/rejected": -722.9078369140625,
1024
+ "loss": 0.3273,
1025
+ "rewards/accuracies": 0.862500011920929,
1026
+ "rewards/chosen": -2.1225264072418213,
1027
+ "rewards/margins": 1.9797760248184204,
1028
+ "rewards/rejected": -4.102302551269531,
1029
  "step": 660
1030
  },
1031
  {
1032
  "epoch": 1.4,
1033
  "learning_rate": 2.468357538028487e-07,
1034
+ "logits/chosen": 0.01013887207955122,
1035
+ "logits/rejected": 0.029213298112154007,
1036
+ "logps/chosen": -541.7221069335938,
1037
+ "logps/rejected": -720.105712890625,
1038
+ "loss": 0.3153,
1039
+ "rewards/accuracies": 0.856249988079071,
1040
+ "rewards/chosen": -2.205817461013794,
1041
+ "rewards/margins": 2.0757954120635986,
1042
+ "rewards/rejected": -4.281612396240234,
1043
  "step": 670
1044
  },
1045
  {
1046
  "epoch": 1.42,
1047
  "learning_rate": 2.312215373764551e-07,
1048
+ "logits/chosen": -0.06487278640270233,
1049
+ "logits/rejected": -0.01965305209159851,
1050
+ "logps/chosen": -516.360107421875,
1051
+ "logps/rejected": -694.418212890625,
1052
+ "loss": 0.3262,
1053
+ "rewards/accuracies": 0.856249988079071,
1054
+ "rewards/chosen": -2.184447765350342,
1055
+ "rewards/margins": 1.861135721206665,
1056
+ "rewards/rejected": -4.045583248138428,
1057
  "step": 680
1058
  },
1059
  {
1060
  "epoch": 1.44,
1061
  "learning_rate": 2.1596762663442213e-07,
1062
+ "logits/chosen": -0.16129662096500397,
1063
+ "logits/rejected": -0.09581325948238373,
1064
+ "logps/chosen": -546.6666259765625,
1065
+ "logps/rejected": -722.95263671875,
1066
+ "loss": 0.3194,
1067
+ "rewards/accuracies": 0.8500000238418579,
1068
+ "rewards/chosen": -2.1821353435516357,
1069
+ "rewards/margins": 2.131826877593994,
1070
+ "rewards/rejected": -4.313961982727051,
1071
  "step": 690
1072
  },
1073
  {
1074
  "epoch": 1.47,
1075
  "learning_rate": 2.0109446990692963e-07,
1076
+ "logits/chosen": -0.11689990758895874,
1077
+ "logits/rejected": -0.2061731368303299,
1078
+ "logps/chosen": -537.8167724609375,
1079
+ "logps/rejected": -781.1466674804688,
1080
+ "loss": 0.337,
1081
+ "rewards/accuracies": 0.8812500238418579,
1082
+ "rewards/chosen": -2.2769298553466797,
1083
+ "rewards/margins": 2.1021370887756348,
1084
+ "rewards/rejected": -4.3790669441223145,
1085
  "step": 700
1086
  },
1087
  {
1088
  "epoch": 1.47,
1089
+ "eval_logits/chosen": 0.041396014392375946,
1090
+ "eval_logits/rejected": 0.009947247803211212,
1091
+ "eval_logps/chosen": -541.1116333007812,
1092
+ "eval_logps/rejected": -699.990966796875,
1093
+ "eval_loss": 0.5290427207946777,
1094
+ "eval_rewards/accuracies": 0.77734375,
1095
+ "eval_rewards/chosen": -2.375269651412964,
1096
+ "eval_rewards/margins": 1.5083887577056885,
1097
+ "eval_rewards/rejected": -3.883657932281494,
1098
+ "eval_runtime": 64.3756,
1099
+ "eval_samples_per_second": 31.068,
1100
+ "eval_steps_per_second": 0.497,
1101
  "step": 700
1102
  },
1103
  {
1104
  "epoch": 1.49,
1105
  "learning_rate": 1.8662200511184872e-07,
1106
+ "logits/chosen": -0.05749096721410751,
1107
+ "logits/rejected": -0.05046076700091362,
1108
+ "logps/chosen": -522.4590454101562,
1109
+ "logps/rejected": -701.3648071289062,
1110
+ "loss": 0.3387,
1111
+ "rewards/accuracies": 0.8812500238418579,
1112
+ "rewards/chosen": -2.1689815521240234,
1113
+ "rewards/margins": 2.023318290710449,
1114
+ "rewards/rejected": -4.192299842834473,
1115
  "step": 710
1116
  },
1117
  {
1118
  "epoch": 1.51,
1119
  "learning_rate": 1.725696330273575e-07,
1120
+ "logits/chosen": -0.12996384501457214,
1121
+ "logits/rejected": -0.20302283763885498,
1122
+ "logps/chosen": -513.8458251953125,
1123
+ "logps/rejected": -704.156982421875,
1124
+ "loss": 0.3363,
1125
+ "rewards/accuracies": 0.887499988079071,
1126
+ "rewards/chosen": -1.9350038766860962,
1127
+ "rewards/margins": 2.0845541954040527,
1128
+ "rewards/rejected": -4.019558429718018,
1129
  "step": 720
1130
  },
1131
  {
1132
  "epoch": 1.53,
1133
  "learning_rate": 1.589561912846089e-07,
1134
+ "logits/chosen": -0.02087187021970749,
1135
+ "logits/rejected": -0.01297883689403534,
1136
+ "logps/chosen": -513.4478149414062,
1137
+ "logps/rejected": -715.5861206054688,
1138
+ "loss": 0.3311,
1139
+ "rewards/accuracies": 0.8812500238418579,
1140
+ "rewards/chosen": -2.01027512550354,
1141
+ "rewards/margins": 1.9698785543441772,
1142
+ "rewards/rejected": -3.9801535606384277,
1143
  "step": 730
1144
  },
1145
  {
1146
  "epoch": 1.55,
1147
  "learning_rate": 1.4579992911531496e-07,
1148
+ "logits/chosen": -0.08378951251506805,
1149
+ "logits/rejected": -0.10333013534545898,
1150
+ "logps/chosen": -528.8599853515625,
1151
+ "logps/rejected": -697.0358276367188,
1152
+ "loss": 0.31,
1153
+ "rewards/accuracies": 0.887499988079071,
1154
+ "rewards/chosen": -2.1352083683013916,
1155
+ "rewards/margins": 2.004941940307617,
1156
+ "rewards/rejected": -4.140150547027588,
1157
  "step": 740
1158
  },
1159
  {
1160
  "epoch": 1.57,
1161
  "learning_rate": 1.3311848288809813e-07,
1162
+ "logits/chosen": -0.021279722452163696,
1163
+ "logits/rejected": -0.07772192358970642,
1164
+ "logps/chosen": -553.6995239257812,
1165
+ "logps/rejected": -696.1124267578125,
1166
+ "loss": 0.3325,
1167
+ "rewards/accuracies": 0.856249988079071,
1168
+ "rewards/chosen": -2.2102222442626953,
1169
+ "rewards/margins": 1.8731515407562256,
1170
+ "rewards/rejected": -4.083374500274658,
1171
  "step": 750
1172
  },
1173
  {
1174
  "epoch": 1.59,
1175
  "learning_rate": 1.209288524664029e-07,
1176
+ "logits/chosen": -0.1553444117307663,
1177
+ "logits/rejected": -0.07370997965335846,
1178
+ "logps/chosen": -559.6234130859375,
1179
+ "logps/rejected": -718.3536376953125,
1180
+ "loss": 0.3318,
1181
+ "rewards/accuracies": 0.84375,
1182
+ "rewards/chosen": -2.2175405025482178,
1183
+ "rewards/margins": 2.087991714477539,
1184
+ "rewards/rejected": -4.305531978607178,
1185
  "step": 760
1186
  },
1187
  {
1188
  "epoch": 1.61,
1189
  "learning_rate": 1.0924737841966497e-07,
1190
+ "logits/chosen": 0.06289811432361603,
1191
+ "logits/rejected": 0.08451451361179352,
1192
+ "logps/chosen": -502.3817443847656,
1193
+ "logps/rejected": -655.4021606445312,
1194
+ "loss": 0.3065,
1195
+ "rewards/accuracies": 0.856249988079071,
1196
+ "rewards/chosen": -2.215487241744995,
1197
+ "rewards/margins": 1.8723819255828857,
1198
+ "rewards/rejected": -4.087869644165039,
1199
  "step": 770
1200
  },
1201
  {
1202
  "epoch": 1.63,
1203
  "learning_rate": 9.808972011828054e-08,
1204
+ "logits/chosen": -0.028034457936882973,
1205
+ "logits/rejected": -0.07584713399410248,
1206
+ "logps/chosen": -507.1363220214844,
1207
+ "logps/rejected": -684.7667236328125,
1208
+ "loss": 0.321,
1209
+ "rewards/accuracies": 0.831250011920929,
1210
+ "rewards/chosen": -2.1353392601013184,
1211
+ "rewards/margins": 1.854984998703003,
1212
+ "rewards/rejected": -3.9903244972229004,
1213
  "step": 780
1214
  },
1215
  {
1216
  "epoch": 1.65,
1217
  "learning_rate": 8.747083474174527e-08,
1218
+ "logits/chosen": -0.027467548847198486,
1219
+ "logits/rejected": -0.11992067098617554,
1220
+ "logps/chosen": -513.2483520507812,
1221
+ "logps/rejected": -773.5486450195312,
1222
+ "loss": 0.3106,
1223
+ "rewards/accuracies": 0.9125000238418579,
1224
+ "rewards/chosen": -2.409447431564331,
1225
+ "rewards/margins": 2.4787163734436035,
1226
+ "rewards/rejected": -4.8881635665893555,
1227
  "step": 790
1228
  },
1229
  {
1230
  "epoch": 1.67,
1231
  "learning_rate": 7.740495722810269e-08,
1232
+ "logits/chosen": -0.22111931443214417,
1233
+ "logits/rejected": -0.16927292943000793,
1234
+ "logps/chosen": -594.322998046875,
1235
+ "logps/rejected": -767.4426879882812,
1236
+ "loss": 0.3398,
1237
+ "rewards/accuracies": 0.84375,
1238
+ "rewards/chosen": -2.4693970680236816,
1239
+ "rewards/margins": 2.0875487327575684,
1240
+ "rewards/rejected": -4.55694580078125,
1241
  "step": 800
1242
  },
1243
  {
1244
  "epoch": 1.67,
1245
+ "eval_logits/chosen": 0.0749908834695816,
1246
+ "eval_logits/rejected": 0.0380852147936821,
1247
+ "eval_logps/chosen": -554.5546264648438,
1248
+ "eval_logps/rejected": -712.9505615234375,
1249
+ "eval_loss": 0.5297122001647949,
1250
+ "eval_rewards/accuracies": 0.7734375,
1251
+ "eval_rewards/chosen": -2.509699821472168,
1252
+ "eval_rewards/margins": 1.5035548210144043,
1253
+ "eval_rewards/rejected": -4.013254642486572,
1254
+ "eval_runtime": 65.3757,
1255
+ "eval_samples_per_second": 30.592,
1256
+ "eval_steps_per_second": 0.489,
1257
  "step": 800
1258
  },
1259
  {
1260
  "epoch": 1.7,
1261
  "learning_rate": 6.790558119157597e-08,
1262
+ "logits/chosen": -0.03027234971523285,
1263
+ "logits/rejected": -0.026968132704496384,
1264
+ "logps/chosen": -558.7872924804688,
1265
+ "logps/rejected": -744.04736328125,
1266
+ "loss": 0.3134,
1267
+ "rewards/accuracies": 0.8812500238418579,
1268
+ "rewards/chosen": -2.323040246963501,
1269
+ "rewards/margins": 2.1539111137390137,
1270
+ "rewards/rejected": -4.476951599121094,
1271
  "step": 810
1272
  },
1273
  {
1274
  "epoch": 1.72,
1275
  "learning_rate": 5.898544083397e-08,
1276
+ "logits/chosen": -0.06089891865849495,
1277
+ "logits/rejected": -0.11520856618881226,
1278
+ "logps/chosen": -516.7492065429688,
1279
+ "logps/rejected": -698.2511596679688,
1280
+ "loss": 0.3131,
1281
+ "rewards/accuracies": 0.862500011920929,
1282
+ "rewards/chosen": -2.2953174114227295,
1283
+ "rewards/margins": 2.0010485649108887,
1284
+ "rewards/rejected": -4.296365737915039,
1285
  "step": 820
1286
  },
1287
  {
1288
  "epoch": 1.74,
1289
  "learning_rate": 5.065649387408705e-08,
1290
+ "logits/chosen": -0.026049736887216568,
1291
+ "logits/rejected": -0.11464808881282806,
1292
+ "logps/chosen": -556.2135620117188,
1293
+ "logps/rejected": -718.17578125,
1294
+ "loss": 0.3145,
1295
+ "rewards/accuracies": 0.8687499761581421,
1296
+ "rewards/chosen": -2.405728816986084,
1297
+ "rewards/margins": 1.908630132675171,
1298
+ "rewards/rejected": -4.314358711242676,
1299
  "step": 830
1300
  },
1301
  {
1302
  "epoch": 1.76,
1303
  "learning_rate": 4.292990551804171e-08,
1304
+ "logits/chosen": -0.12712730467319489,
1305
+ "logits/rejected": -0.10675887763500214,
1306
+ "logps/chosen": -529.3737182617188,
1307
+ "logps/rejected": -723.229248046875,
1308
+ "loss": 0.327,
1309
+ "rewards/accuracies": 0.8500000238418579,
1310
+ "rewards/chosen": -2.146822929382324,
1311
+ "rewards/margins": 2.177410125732422,
1312
+ "rewards/rejected": -4.324233055114746,
1313
  "step": 840
1314
  },
1315
  {
1316
  "epoch": 1.78,
1317
  "learning_rate": 3.581603349196371e-08,
1318
+ "logits/chosen": 0.01494809053838253,
1319
+ "logits/rejected": -0.08690011501312256,
1320
+ "logps/chosen": -540.9631958007812,
1321
+ "logps/rejected": -735.3316650390625,
1322
+ "loss": 0.3262,
1323
+ "rewards/accuracies": 0.918749988079071,
1324
+ "rewards/chosen": -2.145940065383911,
1325
+ "rewards/margins": 2.0986034870147705,
1326
+ "rewards/rejected": -4.24454402923584,
1327
  "step": 850
1328
  },
1329
  {
1330
  "epoch": 1.8,
1331
  "learning_rate": 2.9324414157151367e-08,
1332
+ "logits/chosen": -0.10003119707107544,
1333
+ "logits/rejected": -0.13639459013938904,
1334
+ "logps/chosen": -504.34552001953125,
1335
+ "logps/rejected": -681.4578247070312,
1336
+ "loss": 0.3261,
1337
+ "rewards/accuracies": 0.8812500238418579,
1338
+ "rewards/chosen": -2.0967519283294678,
1339
+ "rewards/margins": 1.8127696514129639,
1340
+ "rewards/rejected": -3.9095215797424316,
1341
  "step": 860
1342
  },
1343
  {
1344
  "epoch": 1.82,
1345
  "learning_rate": 2.3463749726290284e-08,
1346
+ "logits/chosen": -0.005720620043575764,
1347
+ "logits/rejected": -0.09044505655765533,
1348
+ "logps/chosen": -550.0,
1349
+ "logps/rejected": -727.3507690429688,
1350
+ "loss": 0.3221,
1351
+ "rewards/accuracies": 0.862500011920929,
1352
+ "rewards/chosen": -2.1374850273132324,
1353
+ "rewards/margins": 2.060269355773926,
1354
+ "rewards/rejected": -4.197754859924316,
1355
  "step": 870
1356
  },
1357
  {
1358
  "epoch": 1.84,
1359
  "learning_rate": 1.824189659787284e-08,
1360
+ "logits/chosen": -0.07426755130290985,
1361
+ "logits/rejected": -0.09251859039068222,
1362
+ "logps/chosen": -551.6723022460938,
1363
+ "logps/rejected": -753.9895629882812,
1364
+ "loss": 0.3162,
1365
+ "rewards/accuracies": 0.8812500238418579,
1366
+ "rewards/chosen": -2.1559104919433594,
1367
+ "rewards/margins": 2.1075100898742676,
1368
+ "rewards/rejected": -4.263420581817627,
1369
  "step": 880
1370
  },
1371
  {
1372
  "epoch": 1.86,
1373
  "learning_rate": 1.3665854824458035e-08,
1374
+ "logits/chosen": -0.05439913272857666,
1375
+ "logits/rejected": -0.06511974334716797,
1376
+ "logps/chosen": -540.4793090820312,
1377
+ "logps/rejected": -690.4094848632812,
1378
+ "loss": 0.3265,
1379
+ "rewards/accuracies": 0.8500000238418579,
1380
+ "rewards/chosen": -2.1754844188690186,
1381
+ "rewards/margins": 1.7633205652236938,
1382
+ "rewards/rejected": -3.938805103302002,
1383
  "step": 890
1384
  },
1385
  {
1386
  "epoch": 1.88,
1387
  "learning_rate": 9.741758728888217e-09,
1388
+ "logits/chosen": -0.06001782417297363,
1389
+ "logits/rejected": -0.016363339498639107,
1390
+ "logps/chosen": -546.7725830078125,
1391
+ "logps/rejected": -690.0701293945312,
1392
+ "loss": 0.307,
1393
+ "rewards/accuracies": 0.8812500238418579,
1394
+ "rewards/chosen": -2.229912281036377,
1395
+ "rewards/margins": 1.9347511529922485,
1396
+ "rewards/rejected": -4.164663314819336,
1397
  "step": 900
1398
  },
1399
  {
1400
  "epoch": 1.88,
1401
+ "eval_logits/chosen": 0.06628188490867615,
1402
+ "eval_logits/rejected": 0.02886618673801422,
1403
+ "eval_logps/chosen": -549.4910278320312,
1404
+ "eval_logps/rejected": -703.8400268554688,
1405
+ "eval_loss": 0.5260834097862244,
1406
+ "eval_rewards/accuracies": 0.77734375,
1407
+ "eval_rewards/chosen": -2.4590635299682617,
1408
+ "eval_rewards/margins": 1.4630858898162842,
1409
+ "eval_rewards/rejected": -3.922149658203125,
1410
+ "eval_runtime": 65.2404,
1411
+ "eval_samples_per_second": 30.656,
1412
+ "eval_steps_per_second": 0.49,
1413
  "step": 900
1414
  },
1415
  {
1416
  "epoch": 1.9,
1417
  "learning_rate": 6.474868681043577e-09,
1418
+ "logits/chosen": -0.0006331875920295715,
1419
+ "logits/rejected": 0.06078674644231796,
1420
+ "logps/chosen": -526.890869140625,
1421
+ "logps/rejected": -696.57080078125,
1422
+ "loss": 0.3133,
1423
+ "rewards/accuracies": 0.84375,
1424
+ "rewards/chosen": -2.2198734283447266,
1425
+ "rewards/margins": 1.9986340999603271,
1426
+ "rewards/rejected": -4.218507289886475,
1427
  "step": 910
1428
  },
1429
  {
1430
  "epoch": 1.93,
1431
  "learning_rate": 3.869564046156459e-09,
1432
+ "logits/chosen": -0.17484715580940247,
1433
+ "logits/rejected": -0.2491791695356369,
1434
+ "logps/chosen": -560.9608154296875,
1435
+ "logps/rejected": -718.45703125,
1436
+ "loss": 0.3088,
1437
+ "rewards/accuracies": 0.9125000238418579,
1438
+ "rewards/chosen": -2.2493414878845215,
1439
+ "rewards/margins": 2.0530190467834473,
1440
+ "rewards/rejected": -4.3023600578308105,
1441
  "step": 920
1442
  },
1443
  {
1444
  "epoch": 1.95,
1445
  "learning_rate": 1.929337314139412e-09,
1446
+ "logits/chosen": -0.0817941427230835,
1447
+ "logits/rejected": -0.2032664567232132,
1448
+ "logps/chosen": -562.2503662109375,
1449
+ "logps/rejected": -765.2064208984375,
1450
+ "loss": 0.3264,
1451
+ "rewards/accuracies": 0.8812500238418579,
1452
+ "rewards/chosen": -2.2634310722351074,
1453
+ "rewards/margins": 2.2729239463806152,
1454
+ "rewards/rejected": -4.536355018615723,
1455
  "step": 930
1456
  },
1457
  {
1458
  "epoch": 1.97,
1459
  "learning_rate": 6.567894177967325e-10,
1460
+ "logits/chosen": -0.061837755143642426,
1461
+ "logits/rejected": -0.09424273669719696,
1462
+ "logps/chosen": -570.0264892578125,
1463
+ "logps/rejected": -782.4632568359375,
1464
+ "loss": 0.3155,
1465
+ "rewards/accuracies": 0.8999999761581421,
1466
+ "rewards/chosen": -2.259819984436035,
1467
+ "rewards/margins": 2.1893467903137207,
1468
+ "rewards/rejected": -4.449166297912598,
1469
  "step": 940
1470
  },
1471
  {
1472
  "epoch": 1.99,
1473
  "learning_rate": 5.3626246194704575e-11,
1474
+ "logits/chosen": 0.019718965515494347,
1475
+ "logits/rejected": -0.05742845684289932,
1476
+ "logps/chosen": -491.19879150390625,
1477
+ "logps/rejected": -698.8577880859375,
1478
+ "loss": 0.3247,
1479
+ "rewards/accuracies": 0.8687499761581421,
1480
+ "rewards/chosen": -2.148132801055908,
1481
+ "rewards/margins": 2.056044816970825,
1482
+ "rewards/rejected": -4.204176902770996,
1483
  "step": 950
1484
  },
1485
  {
1486
  "epoch": 2.0,
1487
  "step": 954,
1488
  "total_flos": 0.0,
1489
+ "train_loss": 0.44779854000739333,
1490
+ "train_runtime": 8782.9823,
1491
+ "train_samples_per_second": 13.921,
1492
+ "train_steps_per_second": 0.109
1493
  }
1494
  ],
1495
  "logging_steps": 10,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7ebe8424565caecb3908e9e9e2a372a3ad6da08c02b93398d33a8308351a9f5c
3
  size 6648
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5afd5730214e6bd724e3cab0f3dcc26a9879ab9f6aff92cdb3a2b93fd0a49305
3
  size 6648