Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wzhouad commited on
Commit
91caf9d
•
1 Parent(s): 8a997da

Model save

Browse files
Files changed (8) hide show
  1. README.md +16 -0
  2. all_results.json +5 -5
  3. model-00001-of-00003.safetensors +1 -1
  4. model-00002-of-00003.safetensors +1 -1
  5. model-00003-of-00003.safetensors +1 -1
  6. train_results.json +5 -5
  7. trainer_state.json +616 -454
  8. training_args.bin +1 -1
README.md CHANGED
@@ -16,6 +16,16 @@ should probably proofread and complete it, then remove this comment. -->
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 
 
 
 
 
 
 
 
 
19
 
20
  ## Model description
21
 
@@ -50,6 +60,12 @@ The following hyperparameters were used during training:
50
 
51
  ### Training results
52
 
 
 
 
 
 
 
53
 
54
 
55
  ### Framework versions
 
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
+ It achieves the following results on the evaluation set:
20
+ - Loss: 0.1929
21
+ - Rewards/chosen: -1.0190
22
+ - Rewards/rejected: -1.6638
23
+ - Rewards/accuracies: 0.6953
24
+ - Rewards/margins: 0.6448
25
+ - Logps/rejected: -423.7316
26
+ - Logps/chosen: -358.9357
27
+ - Logits/rejected: -2.3314
28
+ - Logits/chosen: -2.3503
29
 
30
  ## Model description
31
 
 
60
 
61
  ### Training results
62
 
63
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 0.2523 | 0.21 | 100 | 0.2711 | -0.5124 | -0.8690 | 0.6641 | 0.3566 | -344.2501 | -308.2755 | -2.6690 | -2.6812 |
66
+ | 0.2026 | 0.42 | 200 | 0.2237 | -0.6974 | -1.2473 | 0.7227 | 0.5499 | -382.0790 | -326.7750 | -2.4918 | -2.5057 |
67
+ | 0.2078 | 0.63 | 300 | 0.2125 | -0.8323 | -1.4075 | 0.7266 | 0.5753 | -398.1045 | -340.2653 | -2.3816 | -2.3992 |
68
+ | 0.1969 | 0.84 | 400 | 0.1929 | -1.0190 | -1.6638 | 0.6953 | 0.6448 | -423.7316 | -358.9357 | -2.3314 | -2.3503 |
69
 
70
 
71
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.14623940638553948,
4
- "train_runtime": 3207.041,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 16.181,
7
- "train_steps_per_second": 0.126
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.2376738007347953,
4
+ "train_runtime": 3939.2333,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.519,
7
+ "train_steps_per_second": 0.121
8
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d2703b55dff69031b3f3ecaf50c0ed868a1d94df7abec9bb57ae4ecb8a13ad10
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:579ec4d763bf13dcf3857e3e54c53cd7750ab5279f10e689d5caea9dc4d4573e
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:89d3d7e08ba2c7ed0fe3b144a8cf69336851d2f0fefd997aba0e399ce8456238
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:540220d0a0000503f4a86326cc47ebc2af95c12378645d36e48507ff4c1ff184
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3e9453a6a9af4586367b2e727b6ccebbe9a638c7ea0dd62394a3b9b2a2f0db79
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c3c7884eea74e83964da773fba4766327d045830b4121df0423f6a9243e9e8b
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.14623940638553948,
4
- "train_runtime": 3207.041,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 16.181,
7
- "train_steps_per_second": 0.126
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.2376738007347953,
4
+ "train_runtime": 3939.2333,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.519,
7
+ "train_steps_per_second": 0.121
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.998766954377312,
5
- "eval_steps": 1000,
6
- "global_step": 405,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 1.2195121951219512e-08,
14
- "logits/chosen": -2.769179582595825,
15
- "logits/rejected": -2.6384379863739014,
16
- "logps/chosen": -511.7125244140625,
17
- "logps/rejected": -154.70135498046875,
18
- "loss": 0.5584,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -24,576 +24,738 @@
24
  },
25
  {
26
  "epoch": 0.02,
27
- "learning_rate": 1.219512195121951e-07,
28
- "logits/chosen": -2.7494113445281982,
29
- "logits/rejected": -2.719766616821289,
30
- "logps/chosen": -350.1738586425781,
31
- "logps/rejected": -113.84426879882812,
32
- "loss": 0.5524,
33
- "rewards/accuracies": 0.5486111044883728,
34
- "rewards/chosen": 0.0006897298735566437,
35
- "rewards/margins": 0.0009333027992397547,
36
- "rewards/rejected": -0.00024357291113119572,
37
  "step": 10
38
  },
39
  {
40
- "epoch": 0.05,
41
- "learning_rate": 2.439024390243902e-07,
42
- "logits/chosen": -2.7349042892456055,
43
- "logits/rejected": -2.6970717906951904,
44
- "logps/chosen": -417.70599365234375,
45
- "logps/rejected": -126.1716079711914,
46
- "loss": 0.5335,
47
- "rewards/accuracies": 0.762499988079071,
48
- "rewards/chosen": 0.016823332756757736,
49
- "rewards/margins": 0.03299414739012718,
50
- "rewards/rejected": -0.016170816496014595,
51
  "step": 20
52
  },
53
  {
54
- "epoch": 0.07,
55
- "learning_rate": 3.6585365853658536e-07,
56
- "logits/chosen": -2.699075222015381,
57
- "logits/rejected": -2.694394588470459,
58
- "logps/chosen": -394.01434326171875,
59
- "logps/rejected": -112.81089782714844,
60
- "loss": 0.4665,
61
- "rewards/accuracies": 0.762499988079071,
62
- "rewards/chosen": 0.06973811239004135,
63
- "rewards/margins": 0.1918165385723114,
64
- "rewards/rejected": -0.12207841873168945,
65
  "step": 30
66
  },
67
  {
68
- "epoch": 0.1,
69
- "learning_rate": 4.878048780487804e-07,
70
- "logits/chosen": -2.5450072288513184,
71
- "logits/rejected": -2.547879934310913,
72
- "logps/chosen": -388.6388854980469,
73
- "logps/rejected": -163.53250122070312,
74
- "loss": 0.3715,
75
- "rewards/accuracies": 0.762499988079071,
76
- "rewards/chosen": 0.025587651878595352,
77
- "rewards/margins": 0.4115292429924011,
78
- "rewards/rejected": -0.38594159483909607,
79
  "step": 40
80
  },
81
  {
82
- "epoch": 0.12,
83
- "learning_rate": 4.992461696250783e-07,
84
- "logits/chosen": -2.4270787239074707,
85
- "logits/rejected": -2.39143705368042,
86
- "logps/chosen": -440.15228271484375,
87
- "logps/rejected": -204.70932006835938,
88
- "loss": 0.2689,
89
- "rewards/accuracies": 0.8187500238418579,
90
- "rewards/chosen": -0.07494890689849854,
91
- "rewards/margins": 0.7927365303039551,
92
- "rewards/rejected": -0.8676854372024536,
93
  "step": 50
94
  },
95
  {
96
- "epoch": 0.15,
97
- "learning_rate": 4.966461721767899e-07,
98
- "logits/chosen": -2.369778633117676,
99
- "logits/rejected": -2.3275959491729736,
100
- "logps/chosen": -423.04144287109375,
101
- "logps/rejected": -231.8262939453125,
102
- "loss": 0.1992,
103
- "rewards/accuracies": 0.7875000238418579,
104
- "rewards/chosen": -0.29674965143203735,
105
- "rewards/margins": 0.9948006868362427,
106
- "rewards/rejected": -1.2915503978729248,
107
  "step": 60
108
  },
109
  {
110
- "epoch": 0.17,
111
- "learning_rate": 4.922100518015975e-07,
112
- "logits/chosen": -2.3612518310546875,
113
- "logits/rejected": -2.331784725189209,
114
- "logps/chosen": -452.7647399902344,
115
- "logps/rejected": -300.84552001953125,
116
- "loss": 0.1691,
117
- "rewards/accuracies": 0.7562500238418579,
118
- "rewards/chosen": -0.578091025352478,
119
- "rewards/margins": 1.2361681461334229,
120
- "rewards/rejected": -1.8142591714859009,
121
  "step": 70
122
  },
123
  {
124
- "epoch": 0.2,
125
- "learning_rate": 4.859708325770919e-07,
126
- "logits/chosen": -2.3421552181243896,
127
- "logits/rejected": -2.3237850666046143,
128
- "logps/chosen": -463.72821044921875,
129
- "logps/rejected": -334.5154724121094,
130
- "loss": 0.1669,
131
- "rewards/accuracies": 0.737500011920929,
132
- "rewards/chosen": -0.6629701852798462,
133
- "rewards/margins": 1.2861653566360474,
134
- "rewards/rejected": -1.949135422706604,
135
  "step": 80
136
  },
137
  {
138
- "epoch": 0.22,
139
- "learning_rate": 4.779749614980225e-07,
140
- "logits/chosen": -2.3320186138153076,
141
- "logits/rejected": -2.277460813522339,
142
- "logps/chosen": -532.4778442382812,
143
- "logps/rejected": -346.4347839355469,
144
- "loss": 0.1202,
145
- "rewards/accuracies": 0.8187500238418579,
146
- "rewards/chosen": -0.35028788447380066,
147
- "rewards/margins": 1.8641752004623413,
148
- "rewards/rejected": -2.214462995529175,
149
  "step": 90
150
  },
151
  {
152
- "epoch": 0.25,
153
- "learning_rate": 4.682819627081427e-07,
154
- "logits/chosen": -2.311751127243042,
155
- "logits/rejected": -2.2163002490997314,
156
- "logps/chosen": -536.274658203125,
157
- "logps/rejected": -369.44781494140625,
158
- "loss": 0.1156,
159
- "rewards/accuracies": 0.800000011920929,
160
- "rewards/chosen": -0.8115866780281067,
161
- "rewards/margins": 1.7770898342132568,
162
- "rewards/rejected": -2.588676691055298,
163
  "step": 100
164
  },
165
  {
166
- "epoch": 0.27,
167
- "learning_rate": 4.569639943810477e-07,
168
- "logits/chosen": -2.2905831336975098,
169
- "logits/rejected": -2.210843324661255,
170
- "logps/chosen": -507.51702880859375,
171
- "logps/rejected": -328.5555725097656,
172
- "loss": 0.1103,
173
- "rewards/accuracies": 0.7562500238418579,
174
- "rewards/chosen": -0.670477032661438,
175
- "rewards/margins": 1.5413503646850586,
176
- "rewards/rejected": -2.211827516555786,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
  "step": 110
178
  },
179
  {
180
- "epoch": 0.3,
181
- "learning_rate": 4.4410531154874543e-07,
182
- "logits/chosen": -2.3034403324127197,
183
- "logits/rejected": -2.234853982925415,
184
- "logps/chosen": -464.66168212890625,
185
- "logps/rejected": -350.87933349609375,
186
- "loss": 0.0998,
187
- "rewards/accuracies": 0.7749999761581421,
188
- "rewards/chosen": -0.7100422978401184,
189
- "rewards/margins": 1.7162730693817139,
190
- "rewards/rejected": -2.4263155460357666,
191
  "step": 120
192
  },
193
  {
194
- "epoch": 0.32,
195
- "learning_rate": 4.298016388768561e-07,
196
- "logits/chosen": -2.2450332641601562,
197
- "logits/rejected": -2.133479356765747,
198
- "logps/chosen": -486.155029296875,
199
- "logps/rejected": -390.749755859375,
200
- "loss": 0.0957,
201
- "rewards/accuracies": 0.793749988079071,
202
- "rewards/chosen": -1.152130365371704,
203
- "rewards/margins": 1.6959073543548584,
204
- "rewards/rejected": -2.8480374813079834,
205
  "step": 130
206
  },
207
  {
208
- "epoch": 0.35,
209
- "learning_rate": 4.1415945805573005e-07,
210
- "logits/chosen": -2.2405707836151123,
211
- "logits/rejected": -2.163384199142456,
212
- "logps/chosen": -506.66583251953125,
213
- "logps/rejected": -383.2144470214844,
214
- "loss": 0.112,
215
- "rewards/accuracies": 0.762499988079071,
216
- "rewards/chosen": -1.087276577949524,
217
- "rewards/margins": 1.5736323595046997,
218
- "rewards/rejected": -2.6609089374542236,
219
  "step": 140
220
  },
221
  {
222
- "epoch": 0.37,
223
- "learning_rate": 3.972952151123984e-07,
224
- "logits/chosen": -2.2938637733459473,
225
- "logits/rejected": -2.209303379058838,
226
- "logps/chosen": -459.609375,
227
- "logps/rejected": -328.35955810546875,
228
- "loss": 0.1276,
229
- "rewards/accuracies": 0.7749999761581421,
230
- "rewards/chosen": -0.7247152924537659,
231
- "rewards/margins": 1.5017669200897217,
232
- "rewards/rejected": -2.2264821529388428,
233
  "step": 150
234
  },
235
  {
236
- "epoch": 0.39,
237
- "learning_rate": 3.793344535444142e-07,
238
- "logits/chosen": -2.2822673320770264,
239
- "logits/rejected": -2.1807644367218018,
240
- "logps/chosen": -501.0845642089844,
241
- "logps/rejected": -365.5765075683594,
242
- "loss": 0.1175,
243
- "rewards/accuracies": 0.8187500238418579,
244
- "rewards/chosen": -0.5146036744117737,
245
- "rewards/margins": 1.9269710779190063,
246
- "rewards/rejected": -2.441574811935425,
247
  "step": 160
248
  },
249
  {
250
- "epoch": 0.42,
251
- "learning_rate": 3.604108797288461e-07,
252
- "logits/chosen": -2.2316362857818604,
253
- "logits/rejected": -2.135136127471924,
254
- "logps/chosen": -474.1004333496094,
255
- "logps/rejected": -387.35906982421875,
256
- "loss": 0.1052,
257
- "rewards/accuracies": 0.7875000238418579,
258
- "rewards/chosen": -1.0368373394012451,
259
- "rewards/margins": 1.7319509983062744,
260
- "rewards/rejected": -2.7687880992889404,
261
  "step": 170
262
  },
263
  {
264
- "epoch": 0.44,
265
- "learning_rate": 3.40665367563858e-07,
266
- "logits/chosen": -2.197690010070801,
267
- "logits/rejected": -2.077615976333618,
268
- "logps/chosen": -538.2199096679688,
269
- "logps/rejected": -420.12158203125,
270
- "loss": 0.0723,
271
- "rewards/accuracies": 0.8062499761581421,
272
- "rewards/chosen": -1.1170153617858887,
273
- "rewards/margins": 1.9479118585586548,
274
- "rewards/rejected": -3.064927339553833,
275
  "step": 180
276
  },
277
  {
278
- "epoch": 0.47,
279
- "learning_rate": 3.202449097526798e-07,
280
- "logits/chosen": -2.184736967086792,
281
- "logits/rejected": -2.0798563957214355,
282
- "logps/chosen": -509.679443359375,
283
- "logps/rejected": -437.552978515625,
284
- "loss": 0.0815,
285
- "rewards/accuracies": 0.824999988079071,
286
- "rewards/chosen": -1.3500667810440063,
287
- "rewards/margins": 1.90851628780365,
288
- "rewards/rejected": -3.2585830688476562,
289
  "step": 190
290
  },
291
  {
292
- "epoch": 0.49,
293
- "learning_rate": 2.993015235369905e-07,
294
- "logits/chosen": -2.149742603302002,
295
- "logits/rejected": -2.0623302459716797,
296
- "logps/chosen": -521.0450439453125,
297
- "logps/rejected": -463.20843505859375,
298
- "loss": 0.0798,
299
- "rewards/accuracies": 0.7749999761581421,
300
- "rewards/chosen": -1.382089376449585,
301
- "rewards/margins": 2.0558183193206787,
302
- "rewards/rejected": -3.4379076957702637,
303
  "step": 200
304
  },
305
  {
306
- "epoch": 0.52,
307
- "learning_rate": 2.7799111902582693e-07,
308
- "logits/chosen": -2.225651264190674,
309
- "logits/rejected": -2.0984432697296143,
310
- "logps/chosen": -563.5731811523438,
311
- "logps/rejected": -442.27191162109375,
312
- "loss": 0.0997,
313
- "rewards/accuracies": 0.8374999761581421,
314
- "rewards/chosen": -1.038576364517212,
315
- "rewards/margins": 2.1469340324401855,
316
- "rewards/rejected": -3.1855106353759766,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  "step": 210
318
  },
319
  {
320
- "epoch": 0.54,
321
- "learning_rate": 2.564723385445869e-07,
322
- "logits/chosen": -2.173677444458008,
323
- "logits/rejected": -2.1013877391815186,
324
- "logps/chosen": -484.97259521484375,
325
- "logps/rejected": -392.4546203613281,
326
- "loss": 0.1033,
327
- "rewards/accuracies": 0.7749999761581421,
328
- "rewards/chosen": -0.9937517046928406,
329
- "rewards/margins": 1.7565183639526367,
330
- "rewards/rejected": -2.750270128250122,
331
  "step": 220
332
  },
333
  {
334
- "epoch": 0.57,
335
- "learning_rate": 2.3490537564442845e-07,
336
- "logits/chosen": -2.194636821746826,
337
- "logits/rejected": -2.0953433513641357,
338
- "logps/chosen": -513.7042236328125,
339
- "logps/rejected": -395.5045166015625,
340
- "loss": 0.1263,
341
- "rewards/accuracies": 0.7875000238418579,
342
- "rewards/chosen": -0.8545423746109009,
343
- "rewards/margins": 1.8620414733886719,
344
- "rewards/rejected": -2.716583728790283,
345
  "step": 230
346
  },
347
  {
348
- "epoch": 0.59,
349
- "learning_rate": 2.1345078256378801e-07,
350
- "logits/chosen": -2.207188844680786,
351
- "logits/rejected": -2.108193874359131,
352
- "logps/chosen": -526.3621826171875,
353
- "logps/rejected": -398.5421142578125,
354
- "loss": 0.1013,
355
- "rewards/accuracies": 0.8125,
356
- "rewards/chosen": -0.8062858581542969,
357
- "rewards/margins": 2.0898499488830566,
358
- "rewards/rejected": -2.8961360454559326,
359
  "step": 240
360
  },
361
  {
362
- "epoch": 0.62,
363
- "learning_rate": 1.9226827501969865e-07,
364
- "logits/chosen": -2.140782594680786,
365
- "logits/rejected": -1.9999908208847046,
366
- "logps/chosen": -523.0418701171875,
367
- "logps/rejected": -424.337890625,
368
- "loss": 0.0789,
369
- "rewards/accuracies": 0.7875000238418579,
370
- "rewards/chosen": -1.1606338024139404,
371
- "rewards/margins": 2.0052971839904785,
372
- "rewards/rejected": -3.165931224822998,
373
  "step": 250
374
  },
375
  {
376
- "epoch": 0.64,
377
- "learning_rate": 1.715155432264775e-07,
378
- "logits/chosen": -2.1131765842437744,
379
- "logits/rejected": -1.9959485530853271,
380
- "logps/chosen": -505.00689697265625,
381
- "logps/rejected": -455.15850830078125,
382
- "loss": 0.0781,
383
- "rewards/accuracies": 0.8062499761581421,
384
- "rewards/chosen": -1.289726972579956,
385
- "rewards/margins": 2.1254703998565674,
386
- "rewards/rejected": -3.4151973724365234,
387
  "step": 260
388
  },
389
  {
390
- "epoch": 0.67,
391
- "learning_rate": 1.51347077992983e-07,
392
- "logits/chosen": -2.159238338470459,
393
- "logits/rejected": -2.043375253677368,
394
- "logps/chosen": -543.9025268554688,
395
- "logps/rejected": -457.2245178222656,
396
- "loss": 0.0875,
397
- "rewards/accuracies": 0.875,
398
- "rewards/chosen": -1.1506714820861816,
399
- "rewards/margins": 2.2577271461486816,
400
- "rewards/rejected": -3.408398389816284,
401
  "step": 270
402
  },
403
  {
404
- "epoch": 0.69,
405
- "learning_rate": 1.3191302063739906e-07,
406
- "logits/chosen": -2.2006659507751465,
407
- "logits/rejected": -2.11677885055542,
408
- "logps/chosen": -509.25091552734375,
409
- "logps/rejected": -413.2759704589844,
410
- "loss": 0.0897,
411
- "rewards/accuracies": 0.875,
412
- "rewards/chosen": -1.0559613704681396,
413
- "rewards/margins": 1.9989497661590576,
414
- "rewards/rejected": -3.0549111366271973,
415
  "step": 280
416
  },
417
  {
418
- "epoch": 0.72,
419
- "learning_rate": 1.1335804528119475e-07,
420
- "logits/chosen": -2.1818597316741943,
421
- "logits/rejected": -2.0668787956237793,
422
- "logps/chosen": -554.3623657226562,
423
- "logps/rejected": -471.38043212890625,
424
- "loss": 0.1041,
425
- "rewards/accuracies": 0.8374999761581421,
426
- "rewards/chosen": -1.1412699222564697,
427
- "rewards/margins": 2.2574570178985596,
428
- "rewards/rejected": -3.39872670173645,
429
  "step": 290
430
  },
431
  {
432
- "epoch": 0.74,
433
- "learning_rate": 9.582028184286423e-08,
434
- "logits/chosen": -2.153825283050537,
435
- "logits/rejected": -2.071503162384033,
436
- "logps/chosen": -486.72186279296875,
437
- "logps/rejected": -409.9599609375,
438
- "loss": 0.1008,
439
- "rewards/accuracies": 0.8125,
440
- "rewards/chosen": -1.0588269233703613,
441
- "rewards/margins": 1.9501113891601562,
442
- "rewards/rejected": -3.0089378356933594,
443
  "step": 300
444
  },
445
  {
446
- "epoch": 0.76,
447
- "learning_rate": 7.943028774907065e-08,
448
- "logits/chosen": -2.2051730155944824,
449
- "logits/rejected": -2.0893607139587402,
450
- "logps/chosen": -538.9556274414062,
451
- "logps/rejected": -393.6221618652344,
452
- "loss": 0.1001,
453
- "rewards/accuracies": 0.831250011920929,
454
- "rewards/chosen": -0.817173182964325,
455
- "rewards/margins": 2.02958345413208,
456
- "rewards/rejected": -2.84675669670105,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
457
  "step": 310
458
  },
459
  {
460
- "epoch": 0.79,
461
- "learning_rate": 6.431007601814637e-08,
462
- "logits/chosen": -2.155592441558838,
463
- "logits/rejected": -2.048361301422119,
464
- "logps/chosen": -504.9908142089844,
465
- "logps/rejected": -441.20257568359375,
466
- "loss": 0.099,
467
- "rewards/accuracies": 0.84375,
468
- "rewards/chosen": -1.055345892906189,
469
- "rewards/margins": 2.115941286087036,
470
- "rewards/rejected": -3.1712870597839355,
471
  "step": 320
472
  },
473
  {
474
- "epoch": 0.81,
475
- "learning_rate": 5.0572206951246e-08,
476
- "logits/chosen": -2.172407627105713,
477
- "logits/rejected": -2.0862064361572266,
478
- "logps/chosen": -459.5045471191406,
479
- "logps/rejected": -408.4109802246094,
480
- "loss": 0.0885,
481
- "rewards/accuracies": 0.768750011920929,
482
- "rewards/chosen": -1.082724690437317,
483
- "rewards/margins": 1.8569291830062866,
484
- "rewards/rejected": -2.9396538734436035,
485
  "step": 330
486
  },
487
  {
488
- "epoch": 0.84,
489
- "learning_rate": 3.831895019292897e-08,
490
- "logits/chosen": -2.197789430618286,
491
- "logits/rejected": -2.0894722938537598,
492
- "logps/chosen": -554.7324829101562,
493
- "logps/rejected": -459.5433654785156,
494
- "loss": 0.0897,
495
- "rewards/accuracies": 0.800000011920929,
496
- "rewards/chosen": -1.07448410987854,
497
- "rewards/margins": 2.295614719390869,
498
- "rewards/rejected": -3.3700993061065674,
499
  "step": 340
500
  },
501
  {
502
- "epoch": 0.86,
503
- "learning_rate": 2.764152339909756e-08,
504
- "logits/chosen": -2.1942572593688965,
505
- "logits/rejected": -2.0833990573883057,
506
- "logps/chosen": -550.0361938476562,
507
- "logps/rejected": -428.2833557128906,
508
- "loss": 0.1036,
509
- "rewards/accuracies": 0.793749988079071,
510
- "rewards/chosen": -0.7792487144470215,
511
- "rewards/margins": 2.310093879699707,
512
- "rewards/rejected": -3.0893425941467285,
513
  "step": 350
514
  },
515
  {
516
- "epoch": 0.89,
517
- "learning_rate": 1.861941317991664e-08,
518
- "logits/chosen": -2.0975775718688965,
519
- "logits/rejected": -2.0279173851013184,
520
- "logps/chosen": -480.5646057128906,
521
- "logps/rejected": -423.53277587890625,
522
- "loss": 0.0817,
523
- "rewards/accuracies": 0.7875000238418579,
524
- "rewards/chosen": -1.1758220195770264,
525
- "rewards/margins": 1.8986190557479858,
526
- "rewards/rejected": -3.0744411945343018,
527
  "step": 360
528
  },
529
  {
530
- "epoch": 0.91,
531
- "learning_rate": 1.13197833728636e-08,
532
- "logits/chosen": -2.1713950634002686,
533
- "logits/rejected": -2.0815377235412598,
534
- "logps/chosen": -517.8831787109375,
535
- "logps/rejected": -435.673828125,
536
- "loss": 0.0966,
537
- "rewards/accuracies": 0.75,
538
- "rewards/chosen": -1.24428391456604,
539
- "rewards/margins": 1.8842618465423584,
540
- "rewards/rejected": -3.1285459995269775,
541
  "step": 370
542
  },
543
  {
544
- "epoch": 0.94,
545
- "learning_rate": 5.79697505093521e-09,
546
- "logits/chosen": -2.2110981941223145,
547
- "logits/rejected": -2.0913751125335693,
548
- "logps/chosen": -547.6890258789062,
549
- "logps/rejected": -447.9027404785156,
550
- "loss": 0.0901,
551
- "rewards/accuracies": 0.8500000238418579,
552
- "rewards/chosen": -0.9516210556030273,
553
- "rewards/margins": 2.320674180984497,
554
- "rewards/rejected": -3.2722949981689453,
555
  "step": 380
556
  },
557
  {
558
- "epoch": 0.96,
559
- "learning_rate": 2.092101988131256e-09,
560
- "logits/chosen": -2.1908106803894043,
561
- "logits/rejected": -2.0610384941101074,
562
- "logps/chosen": -536.3062744140625,
563
- "logps/rejected": -412.360595703125,
564
- "loss": 0.09,
565
- "rewards/accuracies": 0.8062499761581421,
566
- "rewards/chosen": -1.071474552154541,
567
- "rewards/margins": 1.9356151819229126,
568
- "rewards/rejected": -3.007089614868164,
569
  "step": 390
570
  },
571
  {
572
- "epoch": 0.99,
573
- "learning_rate": 2.327445937151673e-10,
574
- "logits/chosen": -2.1998062133789062,
575
- "logits/rejected": -2.0742921829223633,
576
- "logps/chosen": -562.770751953125,
577
- "logps/rejected": -443.3938903808594,
578
- "loss": 0.0877,
579
- "rewards/accuracies": 0.793749988079071,
580
- "rewards/chosen": -1.2041308879852295,
581
- "rewards/margins": 1.9816064834594727,
582
- "rewards/rejected": -3.185737133026123,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
583
  "step": 400
584
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
585
  {
586
  "epoch": 1.0,
587
- "step": 405,
588
  "total_flos": 0.0,
589
- "train_loss": 0.14623940638553948,
590
- "train_runtime": 3207.041,
591
- "train_samples_per_second": 16.181,
592
- "train_steps_per_second": 0.126
593
  }
594
  ],
595
  "logging_steps": 10,
596
- "max_steps": 405,
597
  "num_train_epochs": 1,
598
  "save_steps": 1000,
599
  "total_flos": 0.0,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 100,
6
+ "global_step": 478,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.0416666666666666e-08,
14
+ "logits/chosen": -2.847970962524414,
15
+ "logits/rejected": -2.79160213470459,
16
+ "logps/chosen": -284.9612731933594,
17
+ "logps/rejected": -276.45928955078125,
18
+ "loss": 0.3915,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
24
  },
25
  {
26
  "epoch": 0.02,
27
+ "learning_rate": 1.0416666666666667e-07,
28
+ "logits/chosen": -2.7545580863952637,
29
+ "logits/rejected": -2.752746343612671,
30
+ "logps/chosen": -249.92703247070312,
31
+ "logps/rejected": -223.02972412109375,
32
+ "loss": 0.3789,
33
+ "rewards/accuracies": 0.3958333432674408,
34
+ "rewards/chosen": 0.00020758870232384652,
35
+ "rewards/margins": 2.5434936105739325e-05,
36
+ "rewards/rejected": 0.00018215375894214958,
37
  "step": 10
38
  },
39
  {
40
+ "epoch": 0.04,
41
+ "learning_rate": 2.0833333333333333e-07,
42
+ "logits/chosen": -2.7437922954559326,
43
+ "logits/rejected": -2.7442564964294434,
44
+ "logps/chosen": -257.4372863769531,
45
+ "logps/rejected": -247.53158569335938,
46
+ "loss": 0.3801,
47
+ "rewards/accuracies": 0.5687500238418579,
48
+ "rewards/chosen": 0.00018222886137664318,
49
+ "rewards/margins": 0.0011914785718545318,
50
+ "rewards/rejected": -0.0010092498268932104,
51
  "step": 20
52
  },
53
  {
54
+ "epoch": 0.06,
55
+ "learning_rate": 3.1249999999999997e-07,
56
+ "logits/chosen": -2.7938032150268555,
57
+ "logits/rejected": -2.7469935417175293,
58
+ "logps/chosen": -300.65521240234375,
59
+ "logps/rejected": -262.1036682128906,
60
+ "loss": 0.3821,
61
+ "rewards/accuracies": 0.737500011920929,
62
+ "rewards/chosen": -0.0007819951279088855,
63
+ "rewards/margins": 0.009336644783616066,
64
+ "rewards/rejected": -0.010118638165295124,
65
  "step": 30
66
  },
67
  {
68
+ "epoch": 0.08,
69
+ "learning_rate": 4.1666666666666667e-07,
70
+ "logits/chosen": -2.742980718612671,
71
+ "logits/rejected": -2.7314610481262207,
72
+ "logps/chosen": -257.135498046875,
73
+ "logps/rejected": -275.10113525390625,
74
+ "loss": 0.3828,
75
+ "rewards/accuracies": 0.6187499761581421,
76
+ "rewards/chosen": -0.007016231305897236,
77
+ "rewards/margins": 0.023161854594945908,
78
+ "rewards/rejected": -0.030178088694810867,
79
  "step": 40
80
  },
81
  {
82
+ "epoch": 0.1,
83
+ "learning_rate": 4.999733114418725e-07,
84
+ "logits/chosen": -2.7252087593078613,
85
+ "logits/rejected": -2.700192451477051,
86
+ "logps/chosen": -286.8847961425781,
87
+ "logps/rejected": -257.8277587890625,
88
+ "loss": 0.3717,
89
+ "rewards/accuracies": 0.6187499761581421,
90
+ "rewards/chosen": -0.04805406183004379,
91
+ "rewards/margins": 0.05319146439433098,
92
+ "rewards/rejected": -0.10124553740024567,
93
  "step": 50
94
  },
95
  {
96
+ "epoch": 0.13,
97
+ "learning_rate": 4.990398100856366e-07,
98
+ "logits/chosen": -2.67167329788208,
99
+ "logits/rejected": -2.662323474884033,
100
+ "logps/chosen": -285.72650146484375,
101
+ "logps/rejected": -259.37274169921875,
102
+ "loss": 0.3608,
103
+ "rewards/accuracies": 0.6875,
104
+ "rewards/chosen": -0.09842059016227722,
105
+ "rewards/margins": 0.09482800215482712,
106
+ "rewards/rejected": -0.19324858486652374,
107
  "step": 60
108
  },
109
  {
110
+ "epoch": 0.15,
111
+ "learning_rate": 4.967775735898179e-07,
112
+ "logits/chosen": -2.7104249000549316,
113
+ "logits/rejected": -2.672940731048584,
114
+ "logps/chosen": -281.9991760253906,
115
+ "logps/rejected": -259.15350341796875,
116
+ "loss": 0.3377,
117
+ "rewards/accuracies": 0.643750011920929,
118
+ "rewards/chosen": -0.14863751828670502,
119
+ "rewards/margins": 0.12349126487970352,
120
+ "rewards/rejected": -0.27212879061698914,
121
  "step": 70
122
  },
123
  {
124
+ "epoch": 0.17,
125
+ "learning_rate": 4.931986719649298e-07,
126
+ "logits/chosen": -2.715959310531616,
127
+ "logits/rejected": -2.69069242477417,
128
+ "logps/chosen": -279.71685791015625,
129
+ "logps/rejected": -311.9331359863281,
130
+ "loss": 0.3089,
131
+ "rewards/accuracies": 0.65625,
132
+ "rewards/chosen": -0.19363893568515778,
133
+ "rewards/margins": 0.20271730422973633,
134
+ "rewards/rejected": -0.3963562846183777,
135
  "step": 80
136
  },
137
  {
138
+ "epoch": 0.19,
139
+ "learning_rate": 4.883222001996351e-07,
140
+ "logits/chosen": -2.738593339920044,
141
+ "logits/rejected": -2.716672897338867,
142
+ "logps/chosen": -309.0019226074219,
143
+ "logps/rejected": -314.3983459472656,
144
+ "loss": 0.2719,
145
+ "rewards/accuracies": 0.7562500238418579,
146
+ "rewards/chosen": -0.3363509774208069,
147
+ "rewards/margins": 0.29564622044563293,
148
+ "rewards/rejected": -0.6319972276687622,
149
  "step": 90
150
  },
151
  {
152
+ "epoch": 0.21,
153
+ "learning_rate": 4.821741763807186e-07,
154
+ "logits/chosen": -2.7182154655456543,
155
+ "logits/rejected": -2.6891732215881348,
156
+ "logps/chosen": -342.1278381347656,
157
+ "logps/rejected": -345.8038330078125,
158
+ "loss": 0.2523,
159
+ "rewards/accuracies": 0.6875,
160
+ "rewards/chosen": -0.5316282510757446,
161
+ "rewards/margins": 0.30426135659217834,
162
+ "rewards/rejected": -0.8358896374702454,
163
  "step": 100
164
  },
165
  {
166
+ "epoch": 0.21,
167
+ "eval_logits/chosen": -2.6811912059783936,
168
+ "eval_logits/rejected": -2.6689701080322266,
169
+ "eval_logps/chosen": -308.2755126953125,
170
+ "eval_logps/rejected": -344.25006103515625,
171
+ "eval_loss": 0.2711004614830017,
172
+ "eval_rewards/accuracies": 0.6640625,
173
+ "eval_rewards/chosen": -0.5123588442802429,
174
+ "eval_rewards/margins": 0.3566090166568756,
175
+ "eval_rewards/rejected": -0.8689679503440857,
176
+ "eval_runtime": 53.3401,
177
+ "eval_samples_per_second": 37.495,
178
+ "eval_steps_per_second": 0.6,
179
+ "step": 100
180
+ },
181
+ {
182
+ "epoch": 0.23,
183
+ "learning_rate": 4.747874028753375e-07,
184
+ "logits/chosen": -2.6424639225006104,
185
+ "logits/rejected": -2.6373376846313477,
186
+ "logps/chosen": -316.6573791503906,
187
+ "logps/rejected": -369.0834045410156,
188
+ "loss": 0.2477,
189
+ "rewards/accuracies": 0.7124999761581421,
190
+ "rewards/chosen": -0.523940920829773,
191
+ "rewards/margins": 0.43978047370910645,
192
+ "rewards/rejected": -0.9637213945388794,
193
  "step": 110
194
  },
195
  {
196
+ "epoch": 0.25,
197
+ "learning_rate": 4.662012913161997e-07,
198
+ "logits/chosen": -2.6284713745117188,
199
+ "logits/rejected": -2.597938060760498,
200
+ "logps/chosen": -339.7657165527344,
201
+ "logps/rejected": -354.8578186035156,
202
+ "loss": 0.2235,
203
+ "rewards/accuracies": 0.6499999761581421,
204
+ "rewards/chosen": -0.7222594022750854,
205
+ "rewards/margins": 0.3525253236293793,
206
+ "rewards/rejected": -1.074784755706787,
207
  "step": 120
208
  },
209
  {
210
+ "epoch": 0.27,
211
+ "learning_rate": 4.5646165232345103e-07,
212
+ "logits/chosen": -2.5987703800201416,
213
+ "logits/rejected": -2.573577880859375,
214
+ "logps/chosen": -365.3623962402344,
215
+ "logps/rejected": -347.34552001953125,
216
+ "loss": 0.2335,
217
+ "rewards/accuracies": 0.65625,
218
+ "rewards/chosen": -0.7323095798492432,
219
+ "rewards/margins": 0.33871299028396606,
220
+ "rewards/rejected": -1.0710225105285645,
221
  "step": 130
222
  },
223
  {
224
+ "epoch": 0.29,
225
+ "learning_rate": 4.456204510851956e-07,
226
+ "logits/chosen": -2.5255465507507324,
227
+ "logits/rejected": -2.5133309364318848,
228
+ "logps/chosen": -305.7193603515625,
229
+ "logps/rejected": -312.73089599609375,
230
+ "loss": 0.2379,
231
+ "rewards/accuracies": 0.574999988079071,
232
+ "rewards/chosen": -0.5736026763916016,
233
+ "rewards/margins": 0.35608863830566406,
234
+ "rewards/rejected": -0.9296914339065552,
235
  "step": 140
236
  },
237
  {
238
+ "epoch": 0.31,
239
+ "learning_rate": 4.337355301007335e-07,
240
+ "logits/chosen": -2.5610547065734863,
241
+ "logits/rejected": -2.5525856018066406,
242
+ "logps/chosen": -308.0329895019531,
243
+ "logps/rejected": -313.34759521484375,
244
+ "loss": 0.2687,
245
+ "rewards/accuracies": 0.668749988079071,
246
+ "rewards/chosen": -0.42799726128578186,
247
+ "rewards/margins": 0.3058299422264099,
248
+ "rewards/rejected": -0.7338272333145142,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.33,
253
+ "learning_rate": 4.2087030056579986e-07,
254
+ "logits/chosen": -2.5564608573913574,
255
+ "logits/rejected": -2.5426363945007324,
256
+ "logps/chosen": -303.15679931640625,
257
+ "logps/rejected": -314.12493896484375,
258
+ "loss": 0.2498,
259
+ "rewards/accuracies": 0.7250000238418579,
260
+ "rewards/chosen": -0.421769917011261,
261
+ "rewards/margins": 0.4041440486907959,
262
+ "rewards/rejected": -0.8259140253067017,
263
  "step": 160
264
  },
265
  {
266
+ "epoch": 0.36,
267
+ "learning_rate": 4.070934040463998e-07,
268
+ "logits/chosen": -2.508781671524048,
269
+ "logits/rejected": -2.490255832672119,
270
+ "logps/chosen": -327.84521484375,
271
+ "logps/rejected": -347.8099365234375,
272
+ "loss": 0.243,
273
+ "rewards/accuracies": 0.706250011920929,
274
+ "rewards/chosen": -0.5918872952461243,
275
+ "rewards/margins": 0.37993279099464417,
276
+ "rewards/rejected": -0.971820056438446,
277
  "step": 170
278
  },
279
  {
280
+ "epoch": 0.38,
281
+ "learning_rate": 3.9247834624635404e-07,
282
+ "logits/chosen": -2.55600905418396,
283
+ "logits/rejected": -2.520674228668213,
284
+ "logps/chosen": -338.53125,
285
+ "logps/rejected": -360.1623840332031,
286
+ "loss": 0.2167,
287
+ "rewards/accuracies": 0.731249988079071,
288
+ "rewards/chosen": -0.6208189725875854,
289
+ "rewards/margins": 0.5013567805290222,
290
+ "rewards/rejected": -1.1221758127212524,
291
  "step": 180
292
  },
293
  {
294
+ "epoch": 0.4,
295
+ "learning_rate": 3.7710310482256523e-07,
296
+ "logits/chosen": -2.441803455352783,
297
+ "logits/rejected": -2.4242165088653564,
298
+ "logps/chosen": -364.0983581542969,
299
+ "logps/rejected": -375.9529113769531,
300
+ "loss": 0.1884,
301
+ "rewards/accuracies": 0.668749988079071,
302
+ "rewards/chosen": -1.0688598155975342,
303
+ "rewards/margins": 0.3224587142467499,
304
+ "rewards/rejected": -1.3913185596466064,
305
  "step": 190
306
  },
307
  {
308
+ "epoch": 0.42,
309
+ "learning_rate": 3.610497133404795e-07,
310
+ "logits/chosen": -2.5364327430725098,
311
+ "logits/rejected": -2.485218048095703,
312
+ "logps/chosen": -416.58319091796875,
313
+ "logps/rejected": -426.4273376464844,
314
+ "loss": 0.2026,
315
+ "rewards/accuracies": 0.6812499761581421,
316
+ "rewards/chosen": -0.8200664520263672,
317
+ "rewards/margins": 0.6451858282089233,
318
+ "rewards/rejected": -1.465252161026001,
319
  "step": 200
320
  },
321
  {
322
+ "epoch": 0.42,
323
+ "eval_logits/chosen": -2.5056710243225098,
324
+ "eval_logits/rejected": -2.4918205738067627,
325
+ "eval_logps/chosen": -326.7750244140625,
326
+ "eval_logps/rejected": -382.0789794921875,
327
+ "eval_loss": 0.223700150847435,
328
+ "eval_rewards/accuracies": 0.72265625,
329
+ "eval_rewards/chosen": -0.6973541378974915,
330
+ "eval_rewards/margins": 0.5499030351638794,
331
+ "eval_rewards/rejected": -1.2472572326660156,
332
+ "eval_runtime": 53.3154,
333
+ "eval_samples_per_second": 37.513,
334
+ "eval_steps_per_second": 0.6,
335
+ "step": 200
336
+ },
337
+ {
338
+ "epoch": 0.44,
339
+ "learning_rate": 3.4440382358952115e-07,
340
+ "logits/chosen": -2.493032932281494,
341
+ "logits/rejected": -2.4573144912719727,
342
+ "logps/chosen": -343.7293395996094,
343
+ "logps/rejected": -353.08441162109375,
344
+ "loss": 0.2256,
345
+ "rewards/accuracies": 0.6625000238418579,
346
+ "rewards/chosen": -0.7652268409729004,
347
+ "rewards/margins": 0.3613312542438507,
348
+ "rewards/rejected": -1.1265580654144287,
349
  "step": 210
350
  },
351
  {
352
+ "epoch": 0.46,
353
+ "learning_rate": 3.272542485937368e-07,
354
+ "logits/chosen": -2.466115951538086,
355
+ "logits/rejected": -2.4552156925201416,
356
+ "logps/chosen": -375.7996520996094,
357
+ "logps/rejected": -379.941650390625,
358
+ "loss": 0.2125,
359
+ "rewards/accuracies": 0.71875,
360
+ "rewards/chosen": -0.8123921155929565,
361
+ "rewards/margins": 0.5766803622245789,
362
+ "rewards/rejected": -1.3890724182128906,
363
  "step": 220
364
  },
365
  {
366
+ "epoch": 0.48,
367
+ "learning_rate": 3.096924887558854e-07,
368
+ "logits/chosen": -2.4573655128479004,
369
+ "logits/rejected": -2.4564032554626465,
370
+ "logps/chosen": -369.9993896484375,
371
+ "logps/rejected": -416.593017578125,
372
+ "loss": 0.2056,
373
+ "rewards/accuracies": 0.6312500238418579,
374
+ "rewards/chosen": -0.88189297914505,
375
+ "rewards/margins": 0.37537723779678345,
376
+ "rewards/rejected": -1.257270336151123,
377
  "step": 230
378
  },
379
  {
380
+ "epoch": 0.5,
381
+ "learning_rate": 2.9181224366319943e-07,
382
+ "logits/chosen": -2.4500741958618164,
383
+ "logits/rejected": -2.4248664379119873,
384
+ "logps/chosen": -349.2111511230469,
385
+ "logps/rejected": -400.8269958496094,
386
+ "loss": 0.2171,
387
+ "rewards/accuracies": 0.6812499761581421,
388
+ "rewards/chosen": -0.9550825357437134,
389
+ "rewards/margins": 0.4679138660430908,
390
+ "rewards/rejected": -1.4229964017868042,
391
  "step": 240
392
  },
393
  {
394
+ "epoch": 0.52,
395
+ "learning_rate": 2.7370891215954565e-07,
396
+ "logits/chosen": -2.441260576248169,
397
+ "logits/rejected": -2.400923252105713,
398
+ "logps/chosen": -347.01202392578125,
399
+ "logps/rejected": -366.32470703125,
400
+ "loss": 0.2117,
401
+ "rewards/accuracies": 0.6625000238418579,
402
+ "rewards/chosen": -0.8808671832084656,
403
+ "rewards/margins": 0.42813271284103394,
404
+ "rewards/rejected": -1.3089998960494995,
405
  "step": 250
406
  },
407
  {
408
+ "epoch": 0.54,
409
+ "learning_rate": 2.55479083351317e-07,
410
+ "logits/chosen": -2.4339940547943115,
411
+ "logits/rejected": -2.4294352531433105,
412
+ "logps/chosen": -347.2432861328125,
413
+ "logps/rejected": -361.55462646484375,
414
+ "loss": 0.2139,
415
+ "rewards/accuracies": 0.6000000238418579,
416
+ "rewards/chosen": -0.7863450050354004,
417
+ "rewards/margins": 0.34450870752334595,
418
+ "rewards/rejected": -1.1308536529541016,
419
  "step": 260
420
  },
421
  {
422
+ "epoch": 0.56,
423
+ "learning_rate": 2.3722002126275822e-07,
424
+ "logits/chosen": -2.3959548473358154,
425
+ "logits/rejected": -2.3859493732452393,
426
+ "logps/chosen": -365.3550720214844,
427
+ "logps/rejected": -398.8887023925781,
428
+ "loss": 0.1993,
429
+ "rewards/accuracies": 0.7562500238418579,
430
+ "rewards/chosen": -0.9050162434577942,
431
+ "rewards/margins": 0.5282605886459351,
432
+ "rewards/rejected": -1.4332767724990845,
433
  "step": 270
434
  },
435
  {
436
+ "epoch": 0.59,
437
+ "learning_rate": 2.19029145890313e-07,
438
+ "logits/chosen": -2.389608860015869,
439
+ "logits/rejected": -2.3633534908294678,
440
+ "logps/chosen": -326.2641906738281,
441
+ "logps/rejected": -359.22052001953125,
442
+ "loss": 0.2204,
443
+ "rewards/accuracies": 0.706250011920929,
444
+ "rewards/chosen": -0.8485859632492065,
445
+ "rewards/margins": 0.44154852628707886,
446
+ "rewards/rejected": -1.2901346683502197,
447
  "step": 280
448
  },
449
  {
450
+ "epoch": 0.61,
451
+ "learning_rate": 2.0100351342479216e-07,
452
+ "logits/chosen": -2.3498566150665283,
453
+ "logits/rejected": -2.3162522315979004,
454
+ "logps/chosen": -335.20721435546875,
455
+ "logps/rejected": -368.4345397949219,
456
+ "loss": 0.2053,
457
+ "rewards/accuracies": 0.6812499761581421,
458
+ "rewards/chosen": -0.8859604001045227,
459
+ "rewards/margins": 0.4559580385684967,
460
+ "rewards/rejected": -1.3419182300567627,
461
  "step": 290
462
  },
463
  {
464
+ "epoch": 0.63,
465
+ "learning_rate": 1.8323929841460178e-07,
466
+ "logits/chosen": -2.370981216430664,
467
+ "logits/rejected": -2.341125011444092,
468
+ "logps/chosen": -346.8121032714844,
469
+ "logps/rejected": -392.03216552734375,
470
+ "loss": 0.2078,
471
+ "rewards/accuracies": 0.6875,
472
+ "rewards/chosen": -0.9054353833198547,
473
+ "rewards/margins": 0.4413739740848541,
474
+ "rewards/rejected": -1.3468093872070312,
475
  "step": 300
476
  },
477
  {
478
+ "epoch": 0.63,
479
+ "eval_logits/chosen": -2.399155616760254,
480
+ "eval_logits/rejected": -2.381577253341675,
481
+ "eval_logps/chosen": -340.2652587890625,
482
+ "eval_logps/rejected": -398.1045227050781,
483
+ "eval_loss": 0.21246524155139923,
484
+ "eval_rewards/accuracies": 0.7265625,
485
+ "eval_rewards/chosen": -0.8322564363479614,
486
+ "eval_rewards/margins": 0.5752558708190918,
487
+ "eval_rewards/rejected": -1.4075124263763428,
488
+ "eval_runtime": 53.39,
489
+ "eval_samples_per_second": 37.46,
490
+ "eval_steps_per_second": 0.599,
491
+ "step": 300
492
+ },
493
+ {
494
+ "epoch": 0.65,
495
+ "learning_rate": 1.6583128063291573e-07,
496
+ "logits/chosen": -2.326611042022705,
497
+ "logits/rejected": -2.27211594581604,
498
+ "logps/chosen": -354.09429931640625,
499
+ "logps/rejected": -383.73358154296875,
500
+ "loss": 0.2121,
501
+ "rewards/accuracies": 0.762499988079071,
502
+ "rewards/chosen": -0.8091541528701782,
503
+ "rewards/margins": 0.5564786791801453,
504
+ "rewards/rejected": -1.3656327724456787,
505
  "step": 310
506
  },
507
  {
508
+ "epoch": 0.67,
509
+ "learning_rate": 1.488723393865766e-07,
510
+ "logits/chosen": -2.3466527462005615,
511
+ "logits/rejected": -2.331869602203369,
512
+ "logps/chosen": -383.13824462890625,
513
+ "logps/rejected": -369.4231262207031,
514
+ "loss": 0.2054,
515
+ "rewards/accuracies": 0.7124999761581421,
516
+ "rewards/chosen": -0.8675590753555298,
517
+ "rewards/margins": 0.472606897354126,
518
+ "rewards/rejected": -1.3401659727096558,
519
  "step": 320
520
  },
521
  {
522
+ "epoch": 0.69,
523
+ "learning_rate": 1.3245295796480788e-07,
524
+ "logits/chosen": -2.3689818382263184,
525
+ "logits/rejected": -2.338658094406128,
526
+ "logps/chosen": -335.17608642578125,
527
+ "logps/rejected": -359.6690368652344,
528
+ "loss": 0.2091,
529
+ "rewards/accuracies": 0.706250011920929,
530
+ "rewards/chosen": -0.829629123210907,
531
+ "rewards/margins": 0.4307525157928467,
532
+ "rewards/rejected": -1.2603815793991089,
533
  "step": 330
534
  },
535
  {
536
+ "epoch": 0.71,
537
+ "learning_rate": 1.1666074087171627e-07,
538
+ "logits/chosen": -2.314943790435791,
539
+ "logits/rejected": -2.298879384994507,
540
+ "logps/chosen": -336.0877685546875,
541
+ "logps/rejected": -375.93914794921875,
542
+ "loss": 0.2184,
543
+ "rewards/accuracies": 0.7124999761581421,
544
+ "rewards/chosen": -0.8484441041946411,
545
+ "rewards/margins": 0.4917185306549072,
546
+ "rewards/rejected": -1.3401626348495483,
547
  "step": 340
548
  },
549
  {
550
+ "epoch": 0.73,
551
+ "learning_rate": 1.0157994641835734e-07,
552
+ "logits/chosen": -2.320873975753784,
553
+ "logits/rejected": -2.291957378387451,
554
+ "logps/chosen": -313.73980712890625,
555
+ "logps/rejected": -385.08251953125,
556
+ "loss": 0.1947,
557
+ "rewards/accuracies": 0.7124999761581421,
558
+ "rewards/chosen": -0.8778961896896362,
559
+ "rewards/margins": 0.5659579038619995,
560
+ "rewards/rejected": -1.4438540935516357,
561
  "step": 350
562
  },
563
  {
564
+ "epoch": 0.75,
565
+ "learning_rate": 8.729103716819111e-08,
566
+ "logits/chosen": -2.303989887237549,
567
+ "logits/rejected": -2.2637851238250732,
568
+ "logps/chosen": -359.54522705078125,
569
+ "logps/rejected": -394.9120178222656,
570
+ "loss": 0.1848,
571
+ "rewards/accuracies": 0.6937500238418579,
572
+ "rewards/chosen": -0.9227705001831055,
573
+ "rewards/margins": 0.6329640746116638,
574
+ "rewards/rejected": -1.5557347536087036,
575
  "step": 360
576
  },
577
  {
578
+ "epoch": 0.77,
579
+ "learning_rate": 7.387025063449081e-08,
580
+ "logits/chosen": -2.275233745574951,
581
+ "logits/rejected": -2.272487163543701,
582
+ "logps/chosen": -346.81256103515625,
583
+ "logps/rejected": -408.8800048828125,
584
+ "loss": 0.1902,
585
+ "rewards/accuracies": 0.706250011920929,
586
+ "rewards/chosen": -0.9156505465507507,
587
+ "rewards/margins": 0.6304696202278137,
588
+ "rewards/rejected": -1.546120047569275,
589
  "step": 370
590
  },
591
  {
592
+ "epoch": 0.79,
593
+ "learning_rate": 6.138919252022435e-08,
594
+ "logits/chosen": -2.3174045085906982,
595
+ "logits/rejected": -2.3023200035095215,
596
+ "logps/chosen": -367.288818359375,
597
+ "logps/rejected": -387.2118835449219,
598
+ "loss": 0.1874,
599
+ "rewards/accuracies": 0.6812499761581421,
600
+ "rewards/chosen": -1.0145949125289917,
601
+ "rewards/margins": 0.4892576336860657,
602
+ "rewards/rejected": -1.5038524866104126,
603
  "step": 380
604
  },
605
  {
606
+ "epoch": 0.82,
607
+ "learning_rate": 4.991445467064689e-08,
608
+ "logits/chosen": -2.2982089519500732,
609
+ "logits/rejected": -2.285524606704712,
610
+ "logps/chosen": -354.4279479980469,
611
+ "logps/rejected": -404.1875,
612
+ "loss": 0.1749,
613
+ "rewards/accuracies": 0.731249988079071,
614
+ "rewards/chosen": -0.9722849130630493,
615
+ "rewards/margins": 0.6499310731887817,
616
+ "rewards/rejected": -1.622215986251831,
617
  "step": 390
618
  },
619
  {
620
+ "epoch": 0.84,
621
+ "learning_rate": 3.9507259776993954e-08,
622
+ "logits/chosen": -2.327887535095215,
623
+ "logits/rejected": -2.3192718029022217,
624
+ "logps/chosen": -408.1175842285156,
625
+ "logps/rejected": -433.46112060546875,
626
+ "loss": 0.1969,
627
+ "rewards/accuracies": 0.71875,
628
+ "rewards/chosen": -1.0795756578445435,
629
+ "rewards/margins": 0.5259208083152771,
630
+ "rewards/rejected": -1.6054964065551758,
631
+ "step": 400
632
+ },
633
+ {
634
+ "epoch": 0.84,
635
+ "eval_logits/chosen": -2.35032320022583,
636
+ "eval_logits/rejected": -2.3314263820648193,
637
+ "eval_logps/chosen": -358.9356689453125,
638
+ "eval_logps/rejected": -423.7315673828125,
639
+ "eval_loss": 0.1929483264684677,
640
+ "eval_rewards/accuracies": 0.6953125,
641
+ "eval_rewards/chosen": -1.0189602375030518,
642
+ "eval_rewards/margins": 0.6448227763175964,
643
+ "eval_rewards/rejected": -1.663783073425293,
644
+ "eval_runtime": 53.3094,
645
+ "eval_samples_per_second": 37.517,
646
+ "eval_steps_per_second": 0.6,
647
  "step": 400
648
  },
649
+ {
650
+ "epoch": 0.86,
651
+ "learning_rate": 3.022313472693447e-08,
652
+ "logits/chosen": -2.317988634109497,
653
+ "logits/rejected": -2.3179612159729004,
654
+ "logps/chosen": -354.805419921875,
655
+ "logps/rejected": -374.8717956542969,
656
+ "loss": 0.1942,
657
+ "rewards/accuracies": 0.606249988079071,
658
+ "rewards/chosen": -1.001160740852356,
659
+ "rewards/margins": 0.4229027330875397,
660
+ "rewards/rejected": -1.4240634441375732,
661
+ "step": 410
662
+ },
663
+ {
664
+ "epoch": 0.88,
665
+ "learning_rate": 2.2111614344599684e-08,
666
+ "logits/chosen": -2.3647549152374268,
667
+ "logits/rejected": -2.325208902359009,
668
+ "logps/chosen": -378.22607421875,
669
+ "logps/rejected": -411.9590759277344,
670
+ "loss": 0.1911,
671
+ "rewards/accuracies": 0.75,
672
+ "rewards/chosen": -1.0052746534347534,
673
+ "rewards/margins": 0.6176873445510864,
674
+ "rewards/rejected": -1.622962236404419,
675
+ "step": 420
676
+ },
677
+ {
678
+ "epoch": 0.9,
679
+ "learning_rate": 1.521597710086439e-08,
680
+ "logits/chosen": -2.368295192718506,
681
+ "logits/rejected": -2.3447699546813965,
682
+ "logps/chosen": -371.78778076171875,
683
+ "logps/rejected": -389.54345703125,
684
+ "loss": 0.1998,
685
+ "rewards/accuracies": 0.6875,
686
+ "rewards/chosen": -0.968346118927002,
687
+ "rewards/margins": 0.4705633521080017,
688
+ "rewards/rejected": -1.4389095306396484,
689
+ "step": 430
690
+ },
691
+ {
692
+ "epoch": 0.92,
693
+ "learning_rate": 9.57301420397924e-09,
694
+ "logits/chosen": -2.305577278137207,
695
+ "logits/rejected": -2.3033814430236816,
696
+ "logps/chosen": -378.256591796875,
697
+ "logps/rejected": -391.0841064453125,
698
+ "loss": 0.1993,
699
+ "rewards/accuracies": 0.637499988079071,
700
+ "rewards/chosen": -0.9999098777770996,
701
+ "rewards/margins": 0.5081675052642822,
702
+ "rewards/rejected": -1.5080773830413818,
703
+ "step": 440
704
+ },
705
+ {
706
+ "epoch": 0.94,
707
+ "learning_rate": 5.212833302556258e-09,
708
+ "logits/chosen": -2.322704792022705,
709
+ "logits/rejected": -2.2992866039276123,
710
+ "logps/chosen": -377.765380859375,
711
+ "logps/rejected": -427.04449462890625,
712
+ "loss": 0.1939,
713
+ "rewards/accuracies": 0.675000011920929,
714
+ "rewards/chosen": -1.0114778280258179,
715
+ "rewards/margins": 0.6102879643440247,
716
+ "rewards/rejected": -1.6217658519744873,
717
+ "step": 450
718
+ },
719
+ {
720
+ "epoch": 0.96,
721
+ "learning_rate": 2.158697848236607e-09,
722
+ "logits/chosen": -2.295013189315796,
723
+ "logits/rejected": -2.2902281284332275,
724
+ "logps/chosen": -356.6124572753906,
725
+ "logps/rejected": -393.17877197265625,
726
+ "loss": 0.1901,
727
+ "rewards/accuracies": 0.7250000238418579,
728
+ "rewards/chosen": -0.9131047129631042,
729
+ "rewards/margins": 0.5492212772369385,
730
+ "rewards/rejected": -1.462326169013977,
731
+ "step": 460
732
+ },
733
+ {
734
+ "epoch": 0.98,
735
+ "learning_rate": 4.269029751107489e-10,
736
+ "logits/chosen": -2.3391361236572266,
737
+ "logits/rejected": -2.3124499320983887,
738
+ "logps/chosen": -377.97857666015625,
739
+ "logps/rejected": -387.73883056640625,
740
+ "loss": 0.1979,
741
+ "rewards/accuracies": 0.699999988079071,
742
+ "rewards/chosen": -0.9211581945419312,
743
+ "rewards/margins": 0.5990012884140015,
744
+ "rewards/rejected": -1.5201594829559326,
745
+ "step": 470
746
+ },
747
  {
748
  "epoch": 1.0,
749
+ "step": 478,
750
  "total_flos": 0.0,
751
+ "train_loss": 0.2376738007347953,
752
+ "train_runtime": 3939.2333,
753
+ "train_samples_per_second": 15.519,
754
+ "train_steps_per_second": 0.121
755
  }
756
  ],
757
  "logging_steps": 10,
758
+ "max_steps": 478,
759
  "num_train_epochs": 1,
760
  "save_steps": 1000,
761
  "total_flos": 0.0,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0c8773390caa70c528e762ee16dcaba2dddc8cacf6287d6b9ba470b75637e007
3
  size 5944
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0e2da5c7ae2acc00ea795c5cebf06008933a2eef04170a73329077528d0c661
3
  size 5944