Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wzhouad commited on
Commit
046b871
•
1 Parent(s): ae79bfc

Model save

Browse files
Files changed (8) hide show
  1. README.md +16 -0
  2. all_results.json +5 -5
  3. model-00001-of-00003.safetensors +1 -1
  4. model-00002-of-00003.safetensors +1 -1
  5. model-00003-of-00003.safetensors +1 -1
  6. train_results.json +5 -5
  7. trainer_state.json +616 -454
  8. training_args.bin +1 -1
README.md CHANGED
@@ -16,6 +16,16 @@ should probably proofread and complete it, then remove this comment. -->
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 
 
 
 
 
 
 
 
 
19
 
20
  ## Model description
21
 
@@ -50,6 +60,12 @@ The following hyperparameters were used during training:
50
 
51
  ### Training results
52
 
 
 
 
 
 
 
53
 
54
 
55
  ### Framework versions
 
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
+ It achieves the following results on the evaluation set:
20
+ - Loss: 0.2143
21
+ - Rewards/chosen: -0.8956
22
+ - Rewards/rejected: -1.5167
23
+ - Rewards/accuracies: 0.7031
24
+ - Rewards/margins: 0.6212
25
+ - Logps/rejected: -409.0278
26
+ - Logps/chosen: -346.5983
27
+ - Logits/rejected: -2.4275
28
+ - Logits/chosen: -2.4425
29
 
30
  ## Model description
31
 
 
60
 
61
  ### Training results
62
 
63
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 0.2667 | 0.21 | 100 | 0.2670 | -0.4530 | -0.7921 | 0.6797 | 0.3391 | -336.5636 | -302.3352 | -2.7593 | -2.7741 |
66
+ | 0.2068 | 0.42 | 200 | 0.2087 | -0.8343 | -1.3671 | 0.6836 | 0.5328 | -394.0588 | -340.4660 | -2.5512 | -2.5673 |
67
+ | 0.2095 | 0.63 | 300 | 0.2233 | -0.8384 | -1.4377 | 0.7109 | 0.5993 | -401.1194 | -340.8771 | -2.4645 | -2.4791 |
68
+ | 0.204 | 0.84 | 400 | 0.2143 | -0.8956 | -1.5167 | 0.7031 | 0.6212 | -409.0278 | -346.5983 | -2.4275 | -2.4425 |
69
 
70
 
71
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.1562451661368947,
4
- "train_runtime": 3209.2418,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 16.17,
7
- "train_steps_per_second": 0.126
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.24105885531363627,
4
+ "train_runtime": 3941.0619,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.512,
7
+ "train_steps_per_second": 0.121
8
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e746bdb8da0ed3cb92cfb0eac3514bd900c00b492876034642796dd591398a5b
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:242aeaf1f5600cc8209d4cbd27461aa2ac305da616b77df745a1a8aab656591f
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:54e0fc894931d1546a89c1600598d813b49ed88255568fc1be6110de495a5182
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85852849c7130db331236e6c094e423a37fca1aed57d8382fa6b57ddd47de542
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:eaef24f314b05e2dbdc3675cb3e5a19536af34dec86696772f465d0801e7198e
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd59e16a23b1e7bff035765ae311afad5b0da3456eaaac784867e10f9710abc6
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.1562451661368947,
4
- "train_runtime": 3209.2418,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 16.17,
7
- "train_steps_per_second": 0.126
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.24105885531363627,
4
+ "train_runtime": 3941.0619,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.512,
7
+ "train_steps_per_second": 0.121
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.998766954377312,
5
- "eval_steps": 1000,
6
- "global_step": 405,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 1.2195121951219512e-08,
14
- "logits/chosen": -2.8088459968566895,
15
- "logits/rejected": -2.7595884799957275,
16
- "logps/chosen": -368.90777587890625,
17
- "logps/rejected": -133.10202026367188,
18
- "loss": 0.5469,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -24,576 +24,738 @@
24
  },
25
  {
26
  "epoch": 0.02,
27
- "learning_rate": 1.219512195121951e-07,
28
- "logits/chosen": -2.838653802871704,
29
- "logits/rejected": -2.824901819229126,
30
- "logps/chosen": -433.81378173828125,
31
- "logps/rejected": -114.705810546875,
32
- "loss": 0.55,
33
- "rewards/accuracies": 0.5,
34
- "rewards/chosen": 0.0011021953541785479,
35
- "rewards/margins": 0.0018466737819835544,
36
- "rewards/rejected": -0.0007444784860126674,
37
  "step": 10
38
  },
39
  {
40
- "epoch": 0.05,
41
- "learning_rate": 2.439024390243902e-07,
42
- "logits/chosen": -2.797428607940674,
43
- "logits/rejected": -2.7644600868225098,
44
- "logps/chosen": -436.551025390625,
45
- "logps/rejected": -109.42466735839844,
46
- "loss": 0.5381,
47
- "rewards/accuracies": 0.7437499761581421,
48
- "rewards/chosen": 0.02190575934946537,
49
- "rewards/margins": 0.03880878537893295,
50
- "rewards/rejected": -0.01690302975475788,
51
  "step": 20
52
  },
53
  {
54
- "epoch": 0.07,
55
- "learning_rate": 3.6585365853658536e-07,
56
- "logits/chosen": -2.7145814895629883,
57
- "logits/rejected": -2.6879427433013916,
58
- "logps/chosen": -422.17218017578125,
59
- "logps/rejected": -128.7976531982422,
60
- "loss": 0.4635,
61
- "rewards/accuracies": 0.7749999761581421,
62
- "rewards/chosen": 0.07188864052295685,
63
- "rewards/margins": 0.20736002922058105,
64
- "rewards/rejected": -0.1354713886976242,
65
  "step": 30
66
  },
67
  {
68
- "epoch": 0.1,
69
- "learning_rate": 4.878048780487804e-07,
70
- "logits/chosen": -2.588033676147461,
71
- "logits/rejected": -2.5705184936523438,
72
- "logps/chosen": -397.0795593261719,
73
- "logps/rejected": -140.2168426513672,
74
- "loss": 0.3735,
75
- "rewards/accuracies": 0.762499988079071,
76
- "rewards/chosen": 0.016153398901224136,
77
- "rewards/margins": 0.42458558082580566,
78
- "rewards/rejected": -0.40843215584754944,
79
  "step": 40
80
  },
81
  {
82
- "epoch": 0.12,
83
- "learning_rate": 4.992461696250783e-07,
84
- "logits/chosen": -2.4239001274108887,
85
- "logits/rejected": -2.3979756832122803,
86
- "logps/chosen": -445.3406677246094,
87
- "logps/rejected": -201.51806640625,
88
- "loss": 0.2783,
89
- "rewards/accuracies": 0.8062499761581421,
90
- "rewards/chosen": -0.030274126678705215,
91
- "rewards/margins": 0.8344534635543823,
92
- "rewards/rejected": -0.8647276163101196,
93
  "step": 50
94
  },
95
  {
96
- "epoch": 0.15,
97
- "learning_rate": 4.966461721767899e-07,
98
- "logits/chosen": -2.383493661880493,
99
- "logits/rejected": -2.3332314491271973,
100
- "logps/chosen": -423.5155334472656,
101
- "logps/rejected": -253.60073852539062,
102
- "loss": 0.2228,
103
- "rewards/accuracies": 0.7437499761581421,
104
- "rewards/chosen": -0.36428865790367126,
105
- "rewards/margins": 0.920581042766571,
106
- "rewards/rejected": -1.2848697900772095,
107
  "step": 60
108
  },
109
  {
110
- "epoch": 0.17,
111
- "learning_rate": 4.922100518015975e-07,
112
- "logits/chosen": -2.4096710681915283,
113
- "logits/rejected": -2.364241123199463,
114
- "logps/chosen": -422.70513916015625,
115
- "logps/rejected": -279.1031188964844,
116
- "loss": 0.1891,
117
- "rewards/accuracies": 0.7749999761581421,
118
- "rewards/chosen": -0.3884132504463196,
119
- "rewards/margins": 1.2157753705978394,
120
- "rewards/rejected": -1.6041886806488037,
121
  "step": 70
122
  },
123
  {
124
- "epoch": 0.2,
125
- "learning_rate": 4.859708325770919e-07,
126
- "logits/chosen": -2.38008451461792,
127
- "logits/rejected": -2.344496011734009,
128
- "logps/chosen": -455.79339599609375,
129
- "logps/rejected": -303.01690673828125,
130
- "loss": 0.1269,
131
- "rewards/accuracies": 0.706250011920929,
132
- "rewards/chosen": -0.647363543510437,
133
- "rewards/margins": 1.3260728120803833,
134
- "rewards/rejected": -1.9734363555908203,
135
  "step": 80
136
  },
137
  {
138
- "epoch": 0.22,
139
- "learning_rate": 4.779749614980225e-07,
140
- "logits/chosen": -2.374379873275757,
141
- "logits/rejected": -2.3444790840148926,
142
- "logps/chosen": -532.1400146484375,
143
- "logps/rejected": -380.60955810546875,
144
- "loss": 0.1095,
145
- "rewards/accuracies": 0.8374999761581421,
146
- "rewards/chosen": -0.560777485370636,
147
- "rewards/margins": 1.9465181827545166,
148
- "rewards/rejected": -2.507295846939087,
149
  "step": 90
150
  },
151
  {
152
- "epoch": 0.25,
153
- "learning_rate": 4.682819627081427e-07,
154
- "logits/chosen": -2.3339896202087402,
155
- "logits/rejected": -2.2830748558044434,
156
- "logps/chosen": -467.65374755859375,
157
- "logps/rejected": -348.66156005859375,
158
- "loss": 0.1477,
159
- "rewards/accuracies": 0.8187500238418579,
160
- "rewards/chosen": -0.54865962266922,
161
- "rewards/margins": 1.753379464149475,
162
- "rewards/rejected": -2.30203914642334,
163
  "step": 100
164
  },
165
  {
166
- "epoch": 0.27,
167
- "learning_rate": 4.569639943810477e-07,
168
- "logits/chosen": -2.3383262157440186,
169
- "logits/rejected": -2.2902047634124756,
170
- "logps/chosen": -484.0919494628906,
171
- "logps/rejected": -367.74505615234375,
172
- "loss": 0.1193,
173
- "rewards/accuracies": 0.8062499761581421,
174
- "rewards/chosen": -0.769287645816803,
175
- "rewards/margins": 1.7386033535003662,
176
- "rewards/rejected": -2.5078909397125244,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
  "step": 110
178
  },
179
  {
180
- "epoch": 0.3,
181
- "learning_rate": 4.4410531154874543e-07,
182
- "logits/chosen": -2.3726840019226074,
183
- "logits/rejected": -2.317364454269409,
184
- "logps/chosen": -516.4107666015625,
185
- "logps/rejected": -385.29571533203125,
186
- "loss": 0.1169,
187
- "rewards/accuracies": 0.7875000238418579,
188
- "rewards/chosen": -0.6457246541976929,
189
- "rewards/margins": 1.9019176959991455,
190
- "rewards/rejected": -2.547642469406128,
191
  "step": 120
192
  },
193
  {
194
- "epoch": 0.32,
195
- "learning_rate": 4.298016388768561e-07,
196
- "logits/chosen": -2.413625717163086,
197
- "logits/rejected": -2.3727028369903564,
198
- "logps/chosen": -501.69561767578125,
199
- "logps/rejected": -364.65997314453125,
200
- "loss": 0.1158,
201
- "rewards/accuracies": 0.875,
202
- "rewards/chosen": -0.36493119597435,
203
- "rewards/margins": 2.0498671531677246,
204
- "rewards/rejected": -2.4147982597351074,
205
  "step": 130
206
  },
207
  {
208
- "epoch": 0.35,
209
- "learning_rate": 4.1415945805573005e-07,
210
- "logits/chosen": -2.3137855529785156,
211
- "logits/rejected": -2.2661328315734863,
212
- "logps/chosen": -494.91546630859375,
213
- "logps/rejected": -381.90924072265625,
214
- "loss": 0.1181,
215
- "rewards/accuracies": 0.7875000238418579,
216
- "rewards/chosen": -0.7394388914108276,
217
- "rewards/margins": 1.7537353038787842,
218
- "rewards/rejected": -2.4931740760803223,
219
  "step": 140
220
  },
221
  {
222
- "epoch": 0.37,
223
- "learning_rate": 3.972952151123984e-07,
224
- "logits/chosen": -2.3150975704193115,
225
- "logits/rejected": -2.2541096210479736,
226
- "logps/chosen": -429.13299560546875,
227
- "logps/rejected": -321.36676025390625,
228
- "loss": 0.1415,
229
- "rewards/accuracies": 0.78125,
230
- "rewards/chosen": -0.5441495180130005,
231
- "rewards/margins": 1.660264253616333,
232
- "rewards/rejected": -2.204413890838623,
233
  "step": 150
234
  },
235
  {
236
- "epoch": 0.39,
237
- "learning_rate": 3.793344535444142e-07,
238
- "logits/chosen": -2.2927441596984863,
239
- "logits/rejected": -2.235689401626587,
240
- "logps/chosen": -517.9212646484375,
241
- "logps/rejected": -355.93096923828125,
242
- "loss": 0.1116,
243
- "rewards/accuracies": 0.793749988079071,
244
- "rewards/chosen": -0.5073868036270142,
245
- "rewards/margins": 1.9403215646743774,
246
- "rewards/rejected": -2.4477083683013916,
247
  "step": 160
248
  },
249
  {
250
- "epoch": 0.42,
251
- "learning_rate": 3.604108797288461e-07,
252
- "logits/chosen": -2.2924787998199463,
253
- "logits/rejected": -2.2269301414489746,
254
- "logps/chosen": -528.6913452148438,
255
- "logps/rejected": -407.18780517578125,
256
- "loss": 0.0933,
257
- "rewards/accuracies": 0.831250011920929,
258
- "rewards/chosen": -0.8971579670906067,
259
- "rewards/margins": 2.0700173377990723,
260
- "rewards/rejected": -2.9671754837036133,
261
  "step": 170
262
  },
263
  {
264
- "epoch": 0.44,
265
- "learning_rate": 3.40665367563858e-07,
266
- "logits/chosen": -2.288649320602417,
267
- "logits/rejected": -2.230454683303833,
268
- "logps/chosen": -500.1395568847656,
269
- "logps/rejected": -393.0097961425781,
270
- "loss": 0.0995,
271
- "rewards/accuracies": 0.78125,
272
- "rewards/chosen": -0.9004061818122864,
273
- "rewards/margins": 1.8742806911468506,
274
- "rewards/rejected": -2.774686813354492,
275
  "step": 180
276
  },
277
  {
278
- "epoch": 0.47,
279
- "learning_rate": 3.202449097526798e-07,
280
- "logits/chosen": -2.3139257431030273,
281
- "logits/rejected": -2.2718071937561035,
282
- "logps/chosen": -465.964111328125,
283
- "logps/rejected": -354.6561584472656,
284
- "loss": 0.1167,
285
- "rewards/accuracies": 0.8187500238418579,
286
- "rewards/chosen": -0.6382254362106323,
287
- "rewards/margins": 1.8348891735076904,
288
- "rewards/rejected": -2.4731147289276123,
289
  "step": 190
290
  },
291
  {
292
- "epoch": 0.49,
293
- "learning_rate": 2.993015235369905e-07,
294
- "logits/chosen": -2.2759037017822266,
295
- "logits/rejected": -2.2043704986572266,
296
- "logps/chosen": -518.3172607421875,
297
- "logps/rejected": -417.5638122558594,
298
- "loss": 0.0996,
299
- "rewards/accuracies": 0.831250011920929,
300
- "rewards/chosen": -0.7904411554336548,
301
- "rewards/margins": 2.2067065238952637,
302
- "rewards/rejected": -2.997147798538208,
303
  "step": 200
304
  },
305
  {
306
- "epoch": 0.52,
307
- "learning_rate": 2.7799111902582693e-07,
308
- "logits/chosen": -2.2700018882751465,
309
- "logits/rejected": -2.207021951675415,
310
- "logps/chosen": -489.39801025390625,
311
- "logps/rejected": -369.2752685546875,
312
- "loss": 0.1109,
313
- "rewards/accuracies": 0.78125,
314
- "rewards/chosen": -0.889120876789093,
315
- "rewards/margins": 1.7826087474822998,
316
- "rewards/rejected": -2.671729803085327,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  "step": 210
318
  },
319
  {
320
- "epoch": 0.54,
321
- "learning_rate": 2.564723385445869e-07,
322
- "logits/chosen": -2.324565887451172,
323
- "logits/rejected": -2.267853260040283,
324
- "logps/chosen": -490.59130859375,
325
- "logps/rejected": -388.76690673828125,
326
- "loss": 0.1348,
327
- "rewards/accuracies": 0.8062499761581421,
328
- "rewards/chosen": -0.7297108769416809,
329
- "rewards/margins": 1.9148566722869873,
330
- "rewards/rejected": -2.6445674896240234,
331
  "step": 220
332
  },
333
  {
334
- "epoch": 0.57,
335
- "learning_rate": 2.3490537564442845e-07,
336
- "logits/chosen": -2.267577648162842,
337
- "logits/rejected": -2.186471462249756,
338
- "logps/chosen": -497.12335205078125,
339
- "logps/rejected": -387.3985900878906,
340
- "loss": 0.1239,
341
- "rewards/accuracies": 0.731249988079071,
342
- "rewards/chosen": -1.051777958869934,
343
- "rewards/margins": 1.7568168640136719,
344
- "rewards/rejected": -2.8085949420928955,
345
  "step": 230
346
  },
347
  {
348
- "epoch": 0.59,
349
- "learning_rate": 2.1345078256378801e-07,
350
- "logits/chosen": -2.311931610107422,
351
- "logits/rejected": -2.2356672286987305,
352
- "logps/chosen": -485.4923400878906,
353
- "logps/rejected": -404.1107177734375,
354
- "loss": 0.1251,
355
- "rewards/accuracies": 0.768750011920929,
356
- "rewards/chosen": -0.7667996287345886,
357
- "rewards/margins": 2.0932888984680176,
358
- "rewards/rejected": -2.860088586807251,
359
  "step": 240
360
  },
361
  {
362
- "epoch": 0.62,
363
- "learning_rate": 1.9226827501969865e-07,
364
- "logits/chosen": -2.326636552810669,
365
- "logits/rejected": -2.2703440189361572,
366
- "logps/chosen": -509.9071350097656,
367
- "logps/rejected": -414.39874267578125,
368
- "loss": 0.1196,
369
- "rewards/accuracies": 0.8500000238418579,
370
- "rewards/chosen": -0.6723321676254272,
371
- "rewards/margins": 2.289646625518799,
372
- "rewards/rejected": -2.9619784355163574,
373
  "step": 250
374
  },
375
  {
376
- "epoch": 0.64,
377
- "learning_rate": 1.715155432264775e-07,
378
- "logits/chosen": -2.3231379985809326,
379
- "logits/rejected": -2.2714104652404785,
380
- "logps/chosen": -511.625732421875,
381
- "logps/rejected": -397.1949157714844,
382
- "loss": 0.104,
383
- "rewards/accuracies": 0.8187500238418579,
384
- "rewards/chosen": -0.7959054708480835,
385
- "rewards/margins": 2.012528896331787,
386
- "rewards/rejected": -2.80843448638916,
387
  "step": 260
388
  },
389
  {
390
- "epoch": 0.67,
391
- "learning_rate": 1.51347077992983e-07,
392
- "logits/chosen": -2.3125240802764893,
393
- "logits/rejected": -2.269193410873413,
394
- "logps/chosen": -496.95001220703125,
395
- "logps/rejected": -402.70782470703125,
396
- "loss": 0.0976,
397
- "rewards/accuracies": 0.78125,
398
- "rewards/chosen": -0.9325113296508789,
399
- "rewards/margins": 1.8711084127426147,
400
- "rewards/rejected": -2.803619861602783,
401
  "step": 270
402
  },
403
  {
404
- "epoch": 0.69,
405
- "learning_rate": 1.3191302063739906e-07,
406
- "logits/chosen": -2.282691717147827,
407
- "logits/rejected": -2.233623743057251,
408
- "logps/chosen": -469.02630615234375,
409
- "logps/rejected": -393.0565185546875,
410
- "loss": 0.0978,
411
- "rewards/accuracies": 0.8062499761581421,
412
- "rewards/chosen": -0.8983050584793091,
413
- "rewards/margins": 1.9013340473175049,
414
- "rewards/rejected": -2.7996389865875244,
415
  "step": 280
416
  },
417
  {
418
- "epoch": 0.72,
419
- "learning_rate": 1.1335804528119475e-07,
420
- "logits/chosen": -2.3497660160064697,
421
- "logits/rejected": -2.2719693183898926,
422
- "logps/chosen": -515.9114990234375,
423
- "logps/rejected": -393.408935546875,
424
- "loss": 0.1062,
425
- "rewards/accuracies": 0.7875000238418579,
426
- "rewards/chosen": -0.8015244603157043,
427
- "rewards/margins": 2.1254332065582275,
428
- "rewards/rejected": -2.926957607269287,
429
  "step": 290
430
  },
431
  {
432
- "epoch": 0.74,
433
- "learning_rate": 9.582028184286423e-08,
434
- "logits/chosen": -2.2509076595306396,
435
- "logits/rejected": -2.2074227333068848,
436
- "logps/chosen": -466.32305908203125,
437
- "logps/rejected": -419.3622131347656,
438
- "loss": 0.1048,
439
- "rewards/accuracies": 0.75,
440
- "rewards/chosen": -1.0646774768829346,
441
- "rewards/margins": 1.9130761623382568,
442
- "rewards/rejected": -2.9777536392211914,
443
  "step": 300
444
  },
445
  {
446
- "epoch": 0.76,
447
- "learning_rate": 7.943028774907065e-08,
448
- "logits/chosen": -2.250230312347412,
449
- "logits/rejected": -2.195244789123535,
450
- "logps/chosen": -476.79998779296875,
451
- "logps/rejected": -408.64068603515625,
452
- "loss": 0.1044,
453
- "rewards/accuracies": 0.768750011920929,
454
- "rewards/chosen": -0.923287034034729,
455
- "rewards/margins": 1.9779703617095947,
456
- "rewards/rejected": -2.901257276535034,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
457
  "step": 310
458
  },
459
  {
460
- "epoch": 0.79,
461
- "learning_rate": 6.431007601814637e-08,
462
- "logits/chosen": -2.2714295387268066,
463
- "logits/rejected": -2.2263479232788086,
464
- "logps/chosen": -431.47601318359375,
465
- "logps/rejected": -397.78521728515625,
466
- "loss": 0.0946,
467
- "rewards/accuracies": 0.8062499761581421,
468
- "rewards/chosen": -1.0377047061920166,
469
- "rewards/margins": 1.8672853708267212,
470
- "rewards/rejected": -2.9049899578094482,
471
  "step": 320
472
  },
473
  {
474
- "epoch": 0.81,
475
- "learning_rate": 5.0572206951246e-08,
476
- "logits/chosen": -2.243610382080078,
477
- "logits/rejected": -2.1739163398742676,
478
- "logps/chosen": -469.3233337402344,
479
- "logps/rejected": -400.6234130859375,
480
- "loss": 0.1024,
481
- "rewards/accuracies": 0.78125,
482
- "rewards/chosen": -1.0177555084228516,
483
- "rewards/margins": 1.9254471063613892,
484
- "rewards/rejected": -2.943202495574951,
485
  "step": 330
486
  },
487
  {
488
- "epoch": 0.84,
489
- "learning_rate": 3.831895019292897e-08,
490
- "logits/chosen": -2.302597761154175,
491
- "logits/rejected": -2.245525360107422,
492
- "logps/chosen": -527.4336547851562,
493
- "logps/rejected": -453.01116943359375,
494
- "loss": 0.1089,
495
- "rewards/accuracies": 0.8062499761581421,
496
- "rewards/chosen": -0.9302037954330444,
497
- "rewards/margins": 2.4225857257843018,
498
- "rewards/rejected": -3.3527894020080566,
499
  "step": 340
500
  },
501
  {
502
- "epoch": 0.86,
503
- "learning_rate": 2.764152339909756e-08,
504
- "logits/chosen": -2.2584733963012695,
505
- "logits/rejected": -2.1962692737579346,
506
- "logps/chosen": -499.47576904296875,
507
- "logps/rejected": -363.5857238769531,
508
- "loss": 0.1036,
509
- "rewards/accuracies": 0.8125,
510
- "rewards/chosen": -0.7420647144317627,
511
- "rewards/margins": 1.9064794778823853,
512
- "rewards/rejected": -2.6485438346862793,
513
  "step": 350
514
  },
515
  {
516
- "epoch": 0.89,
517
- "learning_rate": 1.861941317991664e-08,
518
- "logits/chosen": -2.300192356109619,
519
- "logits/rejected": -2.2127695083618164,
520
- "logps/chosen": -531.5907592773438,
521
- "logps/rejected": -412.7438049316406,
522
- "loss": 0.1013,
523
- "rewards/accuracies": 0.8374999761581421,
524
- "rewards/chosen": -0.7529584765434265,
525
- "rewards/margins": 2.228832960128784,
526
- "rewards/rejected": -2.9817919731140137,
527
  "step": 360
528
  },
529
  {
530
- "epoch": 0.91,
531
- "learning_rate": 1.13197833728636e-08,
532
- "logits/chosen": -2.25828218460083,
533
- "logits/rejected": -2.198098659515381,
534
- "logps/chosen": -485.60052490234375,
535
- "logps/rejected": -423.1136779785156,
536
- "loss": 0.095,
537
- "rewards/accuracies": 0.8187500238418579,
538
- "rewards/chosen": -0.8292403221130371,
539
- "rewards/margins": 2.2823524475097656,
540
- "rewards/rejected": -3.1115927696228027,
541
  "step": 370
542
  },
543
  {
544
- "epoch": 0.94,
545
- "learning_rate": 5.79697505093521e-09,
546
- "logits/chosen": -2.2520506381988525,
547
- "logits/rejected": -2.1905932426452637,
548
- "logps/chosen": -501.0511169433594,
549
- "logps/rejected": -402.864501953125,
550
- "loss": 0.1084,
551
- "rewards/accuracies": 0.7749999761581421,
552
- "rewards/chosen": -0.9875413775444031,
553
- "rewards/margins": 1.987672209739685,
554
- "rewards/rejected": -2.9752135276794434,
555
  "step": 380
556
  },
557
  {
558
- "epoch": 0.96,
559
- "learning_rate": 2.092101988131256e-09,
560
- "logits/chosen": -2.2959237098693848,
561
- "logits/rejected": -2.1943726539611816,
562
- "logps/chosen": -542.3916015625,
563
- "logps/rejected": -426.2535095214844,
564
- "loss": 0.092,
565
- "rewards/accuracies": 0.8374999761581421,
566
- "rewards/chosen": -0.7925306558609009,
567
- "rewards/margins": 2.3787803649902344,
568
- "rewards/rejected": -3.1713109016418457,
569
  "step": 390
570
  },
571
  {
572
- "epoch": 0.99,
573
- "learning_rate": 2.327445937151673e-10,
574
- "logits/chosen": -2.2838008403778076,
575
- "logits/rejected": -2.2221412658691406,
576
- "logps/chosen": -538.8756103515625,
577
- "logps/rejected": -445.8970642089844,
578
- "loss": 0.0998,
579
- "rewards/accuracies": 0.824999988079071,
580
- "rewards/chosen": -0.9025734066963196,
581
- "rewards/margins": 2.2714860439300537,
582
- "rewards/rejected": -3.1740598678588867,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
583
  "step": 400
584
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
585
  {
586
  "epoch": 1.0,
587
- "step": 405,
588
  "total_flos": 0.0,
589
- "train_loss": 0.1562451661368947,
590
- "train_runtime": 3209.2418,
591
- "train_samples_per_second": 16.17,
592
- "train_steps_per_second": 0.126
593
  }
594
  ],
595
  "logging_steps": 10,
596
- "max_steps": 405,
597
  "num_train_epochs": 1,
598
  "save_steps": 1000,
599
  "total_flos": 0.0,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 100,
6
+ "global_step": 478,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.0416666666666666e-08,
14
+ "logits/chosen": -2.7515339851379395,
15
+ "logits/rejected": -2.7739691734313965,
16
+ "logps/chosen": -226.12843322753906,
17
+ "logps/rejected": -195.60536193847656,
18
+ "loss": 0.3749,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
24
  },
25
  {
26
  "epoch": 0.02,
27
+ "learning_rate": 1.0416666666666667e-07,
28
+ "logits/chosen": -2.7849607467651367,
29
+ "logits/rejected": -2.7732653617858887,
30
+ "logps/chosen": -296.5021667480469,
31
+ "logps/rejected": -257.125732421875,
32
+ "loss": 0.3841,
33
+ "rewards/accuracies": 0.4166666567325592,
34
+ "rewards/chosen": 9.646587386669125e-06,
35
+ "rewards/margins": -4.828277451451868e-05,
36
+ "rewards/rejected": 5.7929337344830856e-05,
37
  "step": 10
38
  },
39
  {
40
+ "epoch": 0.04,
41
+ "learning_rate": 2.0833333333333333e-07,
42
+ "logits/chosen": -2.7799129486083984,
43
+ "logits/rejected": -2.7518584728240967,
44
+ "logps/chosen": -267.0304870605469,
45
+ "logps/rejected": -265.03131103515625,
46
+ "loss": 0.3837,
47
+ "rewards/accuracies": 0.612500011920929,
48
+ "rewards/chosen": -1.761986277415417e-05,
49
+ "rewards/margins": 0.0019770762883126736,
50
+ "rewards/rejected": -0.001994695980101824,
51
  "step": 20
52
  },
53
  {
54
+ "epoch": 0.06,
55
+ "learning_rate": 3.1249999999999997e-07,
56
+ "logits/chosen": -2.7776148319244385,
57
+ "logits/rejected": -2.7521443367004395,
58
+ "logps/chosen": -246.96499633789062,
59
+ "logps/rejected": -252.1887664794922,
60
+ "loss": 0.3827,
61
+ "rewards/accuracies": 0.6875,
62
+ "rewards/chosen": -0.000771665945649147,
63
+ "rewards/margins": 0.007972861640155315,
64
+ "rewards/rejected": -0.008744527585804462,
65
  "step": 30
66
  },
67
  {
68
+ "epoch": 0.08,
69
+ "learning_rate": 4.1666666666666667e-07,
70
+ "logits/chosen": -2.759718418121338,
71
+ "logits/rejected": -2.7176804542541504,
72
+ "logps/chosen": -276.6578674316406,
73
+ "logps/rejected": -259.2043151855469,
74
+ "loss": 0.3725,
75
+ "rewards/accuracies": 0.5874999761581421,
76
+ "rewards/chosen": -0.01379473228007555,
77
+ "rewards/margins": 0.010001211427152157,
78
+ "rewards/rejected": -0.023795943707227707,
79
  "step": 40
80
  },
81
  {
82
+ "epoch": 0.1,
83
+ "learning_rate": 4.999733114418725e-07,
84
+ "logits/chosen": -2.740699291229248,
85
+ "logits/rejected": -2.732423782348633,
86
+ "logps/chosen": -310.40411376953125,
87
+ "logps/rejected": -309.11676025390625,
88
+ "loss": 0.3717,
89
+ "rewards/accuracies": 0.643750011920929,
90
+ "rewards/chosen": -0.020774774253368378,
91
+ "rewards/margins": 0.06383714824914932,
92
+ "rewards/rejected": -0.0846119225025177,
93
  "step": 50
94
  },
95
  {
96
+ "epoch": 0.13,
97
+ "learning_rate": 4.990398100856366e-07,
98
+ "logits/chosen": -2.707352638244629,
99
+ "logits/rejected": -2.6780617237091064,
100
+ "logps/chosen": -292.3264465332031,
101
+ "logps/rejected": -283.150634765625,
102
+ "loss": 0.3518,
103
+ "rewards/accuracies": 0.668749988079071,
104
+ "rewards/chosen": -0.09367800503969193,
105
+ "rewards/margins": 0.06513573229312897,
106
+ "rewards/rejected": -0.1588137447834015,
107
  "step": 60
108
  },
109
  {
110
+ "epoch": 0.15,
111
+ "learning_rate": 4.967775735898179e-07,
112
+ "logits/chosen": -2.7030491828918457,
113
+ "logits/rejected": -2.6767783164978027,
114
+ "logps/chosen": -272.22491455078125,
115
+ "logps/rejected": -275.2875671386719,
116
+ "loss": 0.331,
117
+ "rewards/accuracies": 0.625,
118
+ "rewards/chosen": -0.18802210688591003,
119
+ "rewards/margins": 0.11484960466623306,
120
+ "rewards/rejected": -0.3028717637062073,
121
  "step": 70
122
  },
123
  {
124
+ "epoch": 0.17,
125
+ "learning_rate": 4.931986719649298e-07,
126
+ "logits/chosen": -2.7163443565368652,
127
+ "logits/rejected": -2.7092785835266113,
128
+ "logps/chosen": -290.34381103515625,
129
+ "logps/rejected": -307.81549072265625,
130
+ "loss": 0.2975,
131
+ "rewards/accuracies": 0.675000011920929,
132
+ "rewards/chosen": -0.30055341124534607,
133
+ "rewards/margins": 0.24591656029224396,
134
+ "rewards/rejected": -0.5464699864387512,
135
  "step": 80
136
  },
137
  {
138
+ "epoch": 0.19,
139
+ "learning_rate": 4.883222001996351e-07,
140
+ "logits/chosen": -2.7387337684631348,
141
+ "logits/rejected": -2.747392416000366,
142
+ "logps/chosen": -279.77520751953125,
143
+ "logps/rejected": -291.8474426269531,
144
+ "loss": 0.2752,
145
+ "rewards/accuracies": 0.6812499761581421,
146
+ "rewards/chosen": -0.40700283646583557,
147
+ "rewards/margins": 0.30104246735572815,
148
+ "rewards/rejected": -0.7080453634262085,
149
  "step": 90
150
  },
151
  {
152
+ "epoch": 0.21,
153
+ "learning_rate": 4.821741763807186e-07,
154
+ "logits/chosen": -2.7377610206604004,
155
+ "logits/rejected": -2.7128090858459473,
156
+ "logps/chosen": -286.3442687988281,
157
+ "logps/rejected": -326.53643798828125,
158
+ "loss": 0.2667,
159
+ "rewards/accuracies": 0.6812499761581421,
160
+ "rewards/chosen": -0.4444156587123871,
161
+ "rewards/margins": 0.26415666937828064,
162
+ "rewards/rejected": -0.708572268486023,
163
  "step": 100
164
  },
165
  {
166
+ "epoch": 0.21,
167
+ "eval_logits/chosen": -2.7741379737854004,
168
+ "eval_logits/rejected": -2.7592568397521973,
169
+ "eval_logps/chosen": -302.3351745605469,
170
+ "eval_logps/rejected": -336.5635986328125,
171
+ "eval_loss": 0.26699498295783997,
172
+ "eval_rewards/accuracies": 0.6796875,
173
+ "eval_rewards/chosen": -0.4529554545879364,
174
+ "eval_rewards/margins": 0.3391478657722473,
175
+ "eval_rewards/rejected": -0.7921034097671509,
176
+ "eval_runtime": 53.381,
177
+ "eval_samples_per_second": 37.466,
178
+ "eval_steps_per_second": 0.599,
179
+ "step": 100
180
+ },
181
+ {
182
+ "epoch": 0.23,
183
+ "learning_rate": 4.747874028753375e-07,
184
+ "logits/chosen": -2.7634060382843018,
185
+ "logits/rejected": -2.7215521335601807,
186
+ "logps/chosen": -326.7304382324219,
187
+ "logps/rejected": -301.48797607421875,
188
+ "loss": 0.2584,
189
+ "rewards/accuracies": 0.625,
190
+ "rewards/chosen": -0.5723937749862671,
191
+ "rewards/margins": 0.20699183642864227,
192
+ "rewards/rejected": -0.7793856263160706,
193
  "step": 110
194
  },
195
  {
196
+ "epoch": 0.25,
197
+ "learning_rate": 4.662012913161997e-07,
198
+ "logits/chosen": -2.7727789878845215,
199
+ "logits/rejected": -2.747377872467041,
200
+ "logps/chosen": -332.1087951660156,
201
+ "logps/rejected": -324.8017883300781,
202
+ "loss": 0.2269,
203
+ "rewards/accuracies": 0.706250011920929,
204
+ "rewards/chosen": -0.5284228920936584,
205
+ "rewards/margins": 0.3542647063732147,
206
+ "rewards/rejected": -0.8826876878738403,
207
  "step": 120
208
  },
209
  {
210
+ "epoch": 0.27,
211
+ "learning_rate": 4.5646165232345103e-07,
212
+ "logits/chosen": -2.7477874755859375,
213
+ "logits/rejected": -2.763334274291992,
214
+ "logps/chosen": -318.76171875,
215
+ "logps/rejected": -344.4560852050781,
216
+ "loss": 0.2668,
217
+ "rewards/accuracies": 0.6875,
218
+ "rewards/chosen": -0.4845491051673889,
219
+ "rewards/margins": 0.34464871883392334,
220
+ "rewards/rejected": -0.829197883605957,
221
  "step": 130
222
  },
223
  {
224
+ "epoch": 0.29,
225
+ "learning_rate": 4.456204510851956e-07,
226
+ "logits/chosen": -2.707592248916626,
227
+ "logits/rejected": -2.695403575897217,
228
+ "logps/chosen": -314.1436462402344,
229
+ "logps/rejected": -335.11773681640625,
230
+ "loss": 0.2446,
231
+ "rewards/accuracies": 0.675000011920929,
232
+ "rewards/chosen": -0.5327329039573669,
233
+ "rewards/margins": 0.3184421956539154,
234
+ "rewards/rejected": -0.8511751294136047,
235
  "step": 140
236
  },
237
  {
238
+ "epoch": 0.31,
239
+ "learning_rate": 4.337355301007335e-07,
240
+ "logits/chosen": -2.7098329067230225,
241
+ "logits/rejected": -2.701035976409912,
242
+ "logps/chosen": -322.45220947265625,
243
+ "logps/rejected": -355.2679748535156,
244
+ "loss": 0.227,
245
+ "rewards/accuracies": 0.6812499761581421,
246
+ "rewards/chosen": -0.6261399984359741,
247
+ "rewards/margins": 0.47124552726745605,
248
+ "rewards/rejected": -1.0973855257034302,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.33,
253
+ "learning_rate": 4.2087030056579986e-07,
254
+ "logits/chosen": -2.603577136993408,
255
+ "logits/rejected": -2.5530428886413574,
256
+ "logps/chosen": -349.7845764160156,
257
+ "logps/rejected": -361.4437561035156,
258
+ "loss": 0.2254,
259
+ "rewards/accuracies": 0.637499988079071,
260
+ "rewards/chosen": -0.7117608189582825,
261
+ "rewards/margins": 0.37124234437942505,
262
+ "rewards/rejected": -1.083003282546997,
263
  "step": 160
264
  },
265
  {
266
+ "epoch": 0.36,
267
+ "learning_rate": 4.070934040463998e-07,
268
+ "logits/chosen": -2.6051487922668457,
269
+ "logits/rejected": -2.566930055618286,
270
+ "logps/chosen": -350.85601806640625,
271
+ "logps/rejected": -397.63140869140625,
272
+ "loss": 0.2194,
273
+ "rewards/accuracies": 0.675000011920929,
274
+ "rewards/chosen": -0.7997169494628906,
275
+ "rewards/margins": 0.4845522940158844,
276
+ "rewards/rejected": -1.2842693328857422,
277
  "step": 170
278
  },
279
  {
280
+ "epoch": 0.38,
281
+ "learning_rate": 3.9247834624635404e-07,
282
+ "logits/chosen": -2.6019983291625977,
283
+ "logits/rejected": -2.587660312652588,
284
+ "logps/chosen": -366.35992431640625,
285
+ "logps/rejected": -348.6190185546875,
286
+ "loss": 0.2364,
287
+ "rewards/accuracies": 0.699999988079071,
288
+ "rewards/chosen": -0.6343470215797424,
289
+ "rewards/margins": 0.3612142503261566,
290
+ "rewards/rejected": -0.9955613017082214,
291
  "step": 180
292
  },
293
  {
294
+ "epoch": 0.4,
295
+ "learning_rate": 3.7710310482256523e-07,
296
+ "logits/chosen": -2.6051242351531982,
297
+ "logits/rejected": -2.577193021774292,
298
+ "logps/chosen": -329.8570556640625,
299
+ "logps/rejected": -313.30438232421875,
300
+ "loss": 0.2243,
301
+ "rewards/accuracies": 0.6499999761581421,
302
+ "rewards/chosen": -0.6197746396064758,
303
+ "rewards/margins": 0.35030925273895264,
304
+ "rewards/rejected": -0.9700838327407837,
305
  "step": 190
306
  },
307
  {
308
+ "epoch": 0.42,
309
+ "learning_rate": 3.610497133404795e-07,
310
+ "logits/chosen": -2.5546486377716064,
311
+ "logits/rejected": -2.511190176010132,
312
+ "logps/chosen": -349.89056396484375,
313
+ "logps/rejected": -380.19647216796875,
314
+ "loss": 0.2068,
315
+ "rewards/accuracies": 0.6312500238418579,
316
+ "rewards/chosen": -0.8633190393447876,
317
+ "rewards/margins": 0.445193350315094,
318
+ "rewards/rejected": -1.3085120916366577,
319
  "step": 200
320
  },
321
  {
322
+ "epoch": 0.42,
323
+ "eval_logits/chosen": -2.5673255920410156,
324
+ "eval_logits/rejected": -2.5512256622314453,
325
+ "eval_logps/chosen": -340.4659729003906,
326
+ "eval_logps/rejected": -394.05877685546875,
327
+ "eval_loss": 0.2086561620235443,
328
+ "eval_rewards/accuracies": 0.68359375,
329
+ "eval_rewards/chosen": -0.8342636823654175,
330
+ "eval_rewards/margins": 0.532791256904602,
331
+ "eval_rewards/rejected": -1.3670549392700195,
332
+ "eval_runtime": 53.3257,
333
+ "eval_samples_per_second": 37.505,
334
+ "eval_steps_per_second": 0.6,
335
+ "step": 200
336
+ },
337
+ {
338
+ "epoch": 0.44,
339
+ "learning_rate": 3.4440382358952115e-07,
340
+ "logits/chosen": -2.49550199508667,
341
+ "logits/rejected": -2.4403936862945557,
342
+ "logps/chosen": -356.9299011230469,
343
+ "logps/rejected": -390.6685791015625,
344
+ "loss": 0.1834,
345
+ "rewards/accuracies": 0.762499988079071,
346
+ "rewards/chosen": -0.8836824297904968,
347
+ "rewards/margins": 0.6037726998329163,
348
+ "rewards/rejected": -1.487455129623413,
349
  "step": 210
350
  },
351
  {
352
+ "epoch": 0.46,
353
+ "learning_rate": 3.272542485937368e-07,
354
+ "logits/chosen": -2.481822967529297,
355
+ "logits/rejected": -2.4627151489257812,
356
+ "logps/chosen": -325.0804138183594,
357
+ "logps/rejected": -375.494873046875,
358
+ "loss": 0.1975,
359
+ "rewards/accuracies": 0.71875,
360
+ "rewards/chosen": -0.8965877294540405,
361
+ "rewards/margins": 0.5446338653564453,
362
+ "rewards/rejected": -1.4412214756011963,
363
  "step": 220
364
  },
365
  {
366
+ "epoch": 0.48,
367
+ "learning_rate": 3.096924887558854e-07,
368
+ "logits/chosen": -2.5111708641052246,
369
+ "logits/rejected": -2.5036470890045166,
370
+ "logps/chosen": -354.4523010253906,
371
+ "logps/rejected": -382.4290466308594,
372
+ "loss": 0.216,
373
+ "rewards/accuracies": 0.6875,
374
+ "rewards/chosen": -0.7817985415458679,
375
+ "rewards/margins": 0.5005447268486023,
376
+ "rewards/rejected": -1.2823432683944702,
377
  "step": 230
378
  },
379
  {
380
+ "epoch": 0.5,
381
+ "learning_rate": 2.9181224366319943e-07,
382
+ "logits/chosen": -2.5602169036865234,
383
+ "logits/rejected": -2.5355300903320312,
384
+ "logps/chosen": -356.17913818359375,
385
+ "logps/rejected": -396.00250244140625,
386
+ "loss": 0.1939,
387
+ "rewards/accuracies": 0.71875,
388
+ "rewards/chosen": -0.9149402379989624,
389
+ "rewards/margins": 0.42837992310523987,
390
+ "rewards/rejected": -1.3433201313018799,
391
  "step": 240
392
  },
393
  {
394
+ "epoch": 0.52,
395
+ "learning_rate": 2.7370891215954565e-07,
396
+ "logits/chosen": -2.5077805519104004,
397
+ "logits/rejected": -2.4881882667541504,
398
+ "logps/chosen": -377.3251953125,
399
+ "logps/rejected": -386.83270263671875,
400
+ "loss": 0.209,
401
+ "rewards/accuracies": 0.6187499761581421,
402
+ "rewards/chosen": -0.9626044034957886,
403
+ "rewards/margins": 0.3591587543487549,
404
+ "rewards/rejected": -1.321763277053833,
405
  "step": 250
406
  },
407
  {
408
+ "epoch": 0.54,
409
+ "learning_rate": 2.55479083351317e-07,
410
+ "logits/chosen": -2.534468173980713,
411
+ "logits/rejected": -2.5206878185272217,
412
+ "logps/chosen": -350.94195556640625,
413
+ "logps/rejected": -361.4455261230469,
414
+ "loss": 0.2209,
415
+ "rewards/accuracies": 0.6937500238418579,
416
+ "rewards/chosen": -0.7781239748001099,
417
+ "rewards/margins": 0.5087354779243469,
418
+ "rewards/rejected": -1.2868595123291016,
419
  "step": 260
420
  },
421
  {
422
+ "epoch": 0.56,
423
+ "learning_rate": 2.3722002126275822e-07,
424
+ "logits/chosen": -2.4776508808135986,
425
+ "logits/rejected": -2.455139398574829,
426
+ "logps/chosen": -372.4471435546875,
427
+ "logps/rejected": -390.30316162109375,
428
+ "loss": 0.2018,
429
+ "rewards/accuracies": 0.6812499761581421,
430
+ "rewards/chosen": -0.8379491567611694,
431
+ "rewards/margins": 0.4740463197231293,
432
+ "rewards/rejected": -1.311995506286621,
433
  "step": 270
434
  },
435
  {
436
+ "epoch": 0.59,
437
+ "learning_rate": 2.19029145890313e-07,
438
+ "logits/chosen": -2.4426798820495605,
439
+ "logits/rejected": -2.421225070953369,
440
+ "logps/chosen": -383.3365478515625,
441
+ "logps/rejected": -408.34246826171875,
442
+ "loss": 0.1931,
443
+ "rewards/accuracies": 0.7250000238418579,
444
+ "rewards/chosen": -1.0771123170852661,
445
+ "rewards/margins": 0.5110739469528198,
446
+ "rewards/rejected": -1.5881863832473755,
447
  "step": 280
448
  },
449
  {
450
+ "epoch": 0.61,
451
+ "learning_rate": 2.0100351342479216e-07,
452
+ "logits/chosen": -2.4331817626953125,
453
+ "logits/rejected": -2.4107987880706787,
454
+ "logps/chosen": -416.7959899902344,
455
+ "logps/rejected": -442.81951904296875,
456
+ "loss": 0.1776,
457
+ "rewards/accuracies": 0.6812499761581421,
458
+ "rewards/chosen": -1.1279302835464478,
459
+ "rewards/margins": 0.5917536020278931,
460
+ "rewards/rejected": -1.7196838855743408,
461
  "step": 290
462
  },
463
  {
464
+ "epoch": 0.63,
465
+ "learning_rate": 1.8323929841460178e-07,
466
+ "logits/chosen": -2.44761323928833,
467
+ "logits/rejected": -2.425476312637329,
468
+ "logps/chosen": -379.3946838378906,
469
+ "logps/rejected": -423.1104431152344,
470
+ "loss": 0.2095,
471
+ "rewards/accuracies": 0.699999988079071,
472
+ "rewards/chosen": -1.083051085472107,
473
+ "rewards/margins": 0.5393560528755188,
474
+ "rewards/rejected": -1.6224071979522705,
475
  "step": 300
476
  },
477
  {
478
+ "epoch": 0.63,
479
+ "eval_logits/chosen": -2.479083776473999,
480
+ "eval_logits/rejected": -2.464531421661377,
481
+ "eval_logps/chosen": -340.87713623046875,
482
+ "eval_logps/rejected": -401.1194152832031,
483
+ "eval_loss": 0.22332347929477692,
484
+ "eval_rewards/accuracies": 0.7109375,
485
+ "eval_rewards/chosen": -0.8383752107620239,
486
+ "eval_rewards/margins": 0.5992860794067383,
487
+ "eval_rewards/rejected": -1.4376611709594727,
488
+ "eval_runtime": 53.3117,
489
+ "eval_samples_per_second": 37.515,
490
+ "eval_steps_per_second": 0.6,
491
+ "step": 300
492
+ },
493
+ {
494
+ "epoch": 0.65,
495
+ "learning_rate": 1.6583128063291573e-07,
496
+ "logits/chosen": -2.459578037261963,
497
+ "logits/rejected": -2.4456562995910645,
498
+ "logps/chosen": -363.7841796875,
499
+ "logps/rejected": -435.59478759765625,
500
+ "loss": 0.2183,
501
+ "rewards/accuracies": 0.737500011920929,
502
+ "rewards/chosen": -0.8949993848800659,
503
+ "rewards/margins": 0.6970807313919067,
504
+ "rewards/rejected": -1.592079997062683,
505
  "step": 310
506
  },
507
  {
508
+ "epoch": 0.67,
509
+ "learning_rate": 1.488723393865766e-07,
510
+ "logits/chosen": -2.4861679077148438,
511
+ "logits/rejected": -2.437126874923706,
512
+ "logps/chosen": -382.5135498046875,
513
+ "logps/rejected": -415.387939453125,
514
+ "loss": 0.2205,
515
+ "rewards/accuracies": 0.7124999761581421,
516
+ "rewards/chosen": -0.8941608667373657,
517
+ "rewards/margins": 0.5314512252807617,
518
+ "rewards/rejected": -1.4256120920181274,
519
  "step": 320
520
  },
521
  {
522
+ "epoch": 0.69,
523
+ "learning_rate": 1.3245295796480788e-07,
524
+ "logits/chosen": -2.408893346786499,
525
+ "logits/rejected": -2.405259609222412,
526
+ "logps/chosen": -369.69049072265625,
527
+ "logps/rejected": -420.62384033203125,
528
+ "loss": 0.211,
529
+ "rewards/accuracies": 0.675000011920929,
530
+ "rewards/chosen": -0.9662488102912903,
531
+ "rewards/margins": 0.5407388210296631,
532
+ "rewards/rejected": -1.5069875717163086,
533
  "step": 330
534
  },
535
  {
536
+ "epoch": 0.71,
537
+ "learning_rate": 1.1666074087171627e-07,
538
+ "logits/chosen": -2.400749444961548,
539
+ "logits/rejected": -2.397270917892456,
540
+ "logps/chosen": -420.1078186035156,
541
+ "logps/rejected": -454.15069580078125,
542
+ "loss": 0.2107,
543
+ "rewards/accuracies": 0.675000011920929,
544
+ "rewards/chosen": -1.1604535579681396,
545
+ "rewards/margins": 0.46516379714012146,
546
+ "rewards/rejected": -1.625617265701294,
547
  "step": 340
548
  },
549
  {
550
+ "epoch": 0.73,
551
+ "learning_rate": 1.0157994641835734e-07,
552
+ "logits/chosen": -2.448854684829712,
553
+ "logits/rejected": -2.404040813446045,
554
+ "logps/chosen": -332.3216247558594,
555
+ "logps/rejected": -371.5892028808594,
556
+ "loss": 0.208,
557
+ "rewards/accuracies": 0.71875,
558
+ "rewards/chosen": -0.8533551096916199,
559
+ "rewards/margins": 0.6172915697097778,
560
+ "rewards/rejected": -1.470646619796753,
561
  "step": 350
562
  },
563
  {
564
+ "epoch": 0.75,
565
+ "learning_rate": 8.729103716819111e-08,
566
+ "logits/chosen": -2.3937747478485107,
567
+ "logits/rejected": -2.3905582427978516,
568
+ "logps/chosen": -368.26849365234375,
569
+ "logps/rejected": -392.41937255859375,
570
+ "loss": 0.2182,
571
+ "rewards/accuracies": 0.71875,
572
+ "rewards/chosen": -0.8935198783874512,
573
+ "rewards/margins": 0.5953238010406494,
574
+ "rewards/rejected": -1.4888436794281006,
575
  "step": 360
576
  },
577
  {
578
+ "epoch": 0.77,
579
+ "learning_rate": 7.387025063449081e-08,
580
+ "logits/chosen": -2.410921096801758,
581
+ "logits/rejected": -2.3979973793029785,
582
+ "logps/chosen": -351.77972412109375,
583
+ "logps/rejected": -412.9110412597656,
584
+ "loss": 0.2042,
585
+ "rewards/accuracies": 0.75,
586
+ "rewards/chosen": -0.9797185063362122,
587
+ "rewards/margins": 0.6103909611701965,
588
+ "rewards/rejected": -1.5901095867156982,
589
  "step": 370
590
  },
591
  {
592
+ "epoch": 0.79,
593
+ "learning_rate": 6.138919252022435e-08,
594
+ "logits/chosen": -2.4055371284484863,
595
+ "logits/rejected": -2.365029811859131,
596
+ "logps/chosen": -429.22540283203125,
597
+ "logps/rejected": -424.9950256347656,
598
+ "loss": 0.211,
599
+ "rewards/accuracies": 0.675000011920929,
600
+ "rewards/chosen": -1.043166160583496,
601
+ "rewards/margins": 0.5213008522987366,
602
+ "rewards/rejected": -1.5644668340682983,
603
  "step": 380
604
  },
605
  {
606
+ "epoch": 0.82,
607
+ "learning_rate": 4.991445467064689e-08,
608
+ "logits/chosen": -2.3971471786499023,
609
+ "logits/rejected": -2.3788516521453857,
610
+ "logps/chosen": -353.85052490234375,
611
+ "logps/rejected": -394.6015319824219,
612
+ "loss": 0.2059,
613
+ "rewards/accuracies": 0.6937500238418579,
614
+ "rewards/chosen": -0.9903608560562134,
615
+ "rewards/margins": 0.528742253780365,
616
+ "rewards/rejected": -1.519102931022644,
617
  "step": 390
618
  },
619
  {
620
+ "epoch": 0.84,
621
+ "learning_rate": 3.9507259776993954e-08,
622
+ "logits/chosen": -2.4183950424194336,
623
+ "logits/rejected": -2.3998031616210938,
624
+ "logps/chosen": -349.13397216796875,
625
+ "logps/rejected": -394.29034423828125,
626
+ "loss": 0.204,
627
+ "rewards/accuracies": 0.731249988079071,
628
+ "rewards/chosen": -0.842970073223114,
629
+ "rewards/margins": 0.6520211696624756,
630
+ "rewards/rejected": -1.4949913024902344,
631
+ "step": 400
632
+ },
633
+ {
634
+ "epoch": 0.84,
635
+ "eval_logits/chosen": -2.4425089359283447,
636
+ "eval_logits/rejected": -2.4274942874908447,
637
+ "eval_logps/chosen": -346.5982666015625,
638
+ "eval_logps/rejected": -409.02783203125,
639
+ "eval_loss": 0.21434982120990753,
640
+ "eval_rewards/accuracies": 0.703125,
641
+ "eval_rewards/chosen": -0.8955863118171692,
642
+ "eval_rewards/margins": 0.6211590766906738,
643
+ "eval_rewards/rejected": -1.5167453289031982,
644
+ "eval_runtime": 53.3158,
645
+ "eval_samples_per_second": 37.512,
646
+ "eval_steps_per_second": 0.6,
647
  "step": 400
648
  },
649
+ {
650
+ "epoch": 0.86,
651
+ "learning_rate": 3.022313472693447e-08,
652
+ "logits/chosen": -2.346083164215088,
653
+ "logits/rejected": -2.3530008792877197,
654
+ "logps/chosen": -359.29718017578125,
655
+ "logps/rejected": -380.91094970703125,
656
+ "loss": 0.2113,
657
+ "rewards/accuracies": 0.7250000238418579,
658
+ "rewards/chosen": -0.9364844560623169,
659
+ "rewards/margins": 0.4689885973930359,
660
+ "rewards/rejected": -1.4054731130599976,
661
+ "step": 410
662
+ },
663
+ {
664
+ "epoch": 0.88,
665
+ "learning_rate": 2.2111614344599684e-08,
666
+ "logits/chosen": -2.3877267837524414,
667
+ "logits/rejected": -2.3703396320343018,
668
+ "logps/chosen": -347.065673828125,
669
+ "logps/rejected": -389.3710021972656,
670
+ "loss": 0.2072,
671
+ "rewards/accuracies": 0.7124999761581421,
672
+ "rewards/chosen": -0.8826552629470825,
673
+ "rewards/margins": 0.5836518406867981,
674
+ "rewards/rejected": -1.4663069248199463,
675
+ "step": 420
676
+ },
677
+ {
678
+ "epoch": 0.9,
679
+ "learning_rate": 1.521597710086439e-08,
680
+ "logits/chosen": -2.427062511444092,
681
+ "logits/rejected": -2.3968257904052734,
682
+ "logps/chosen": -384.0503845214844,
683
+ "logps/rejected": -394.9156799316406,
684
+ "loss": 0.2142,
685
+ "rewards/accuracies": 0.699999988079071,
686
+ "rewards/chosen": -1.0325697660446167,
687
+ "rewards/margins": 0.47820836305618286,
688
+ "rewards/rejected": -1.5107781887054443,
689
+ "step": 430
690
+ },
691
+ {
692
+ "epoch": 0.92,
693
+ "learning_rate": 9.57301420397924e-09,
694
+ "logits/chosen": -2.4799857139587402,
695
+ "logits/rejected": -2.432332992553711,
696
+ "logps/chosen": -368.3861083984375,
697
+ "logps/rejected": -357.58380126953125,
698
+ "loss": 0.2142,
699
+ "rewards/accuracies": 0.7124999761581421,
700
+ "rewards/chosen": -0.9250670671463013,
701
+ "rewards/margins": 0.450716108083725,
702
+ "rewards/rejected": -1.375783085823059,
703
+ "step": 440
704
+ },
705
+ {
706
+ "epoch": 0.94,
707
+ "learning_rate": 5.212833302556258e-09,
708
+ "logits/chosen": -2.4059414863586426,
709
+ "logits/rejected": -2.3920960426330566,
710
+ "logps/chosen": -371.8204650878906,
711
+ "logps/rejected": -386.82269287109375,
712
+ "loss": 0.215,
713
+ "rewards/accuracies": 0.71875,
714
+ "rewards/chosen": -0.8487032651901245,
715
+ "rewards/margins": 0.4881046414375305,
716
+ "rewards/rejected": -1.3368079662322998,
717
+ "step": 450
718
+ },
719
+ {
720
+ "epoch": 0.96,
721
+ "learning_rate": 2.158697848236607e-09,
722
+ "logits/chosen": -2.456326961517334,
723
+ "logits/rejected": -2.4320716857910156,
724
+ "logps/chosen": -348.0980529785156,
725
+ "logps/rejected": -381.2003173828125,
726
+ "loss": 0.2155,
727
+ "rewards/accuracies": 0.706250011920929,
728
+ "rewards/chosen": -0.8624986410140991,
729
+ "rewards/margins": 0.5545080900192261,
730
+ "rewards/rejected": -1.4170067310333252,
731
+ "step": 460
732
+ },
733
+ {
734
+ "epoch": 0.98,
735
+ "learning_rate": 4.269029751107489e-10,
736
+ "logits/chosen": -2.4110119342803955,
737
+ "logits/rejected": -2.383718967437744,
738
+ "logps/chosen": -323.4446716308594,
739
+ "logps/rejected": -392.60064697265625,
740
+ "loss": 0.2178,
741
+ "rewards/accuracies": 0.768750011920929,
742
+ "rewards/chosen": -0.8438596725463867,
743
+ "rewards/margins": 0.6573068499565125,
744
+ "rewards/rejected": -1.501166582107544,
745
+ "step": 470
746
+ },
747
  {
748
  "epoch": 1.0,
749
+ "step": 478,
750
  "total_flos": 0.0,
751
+ "train_loss": 0.24105885531363627,
752
+ "train_runtime": 3941.0619,
753
+ "train_samples_per_second": 15.512,
754
+ "train_steps_per_second": 0.121
755
  }
756
  ],
757
  "logging_steps": 10,
758
+ "max_steps": 478,
759
  "num_train_epochs": 1,
760
  "save_steps": 1000,
761
  "total_flos": 0.0,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9ee5499d0ce2b903ad76a9c35cc9bf4c38d09435955bae297aa02a619c188930
3
  size 5944
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfad7eb6d35cc2b71f5772e76bb519a9fc483f6905c2bffa88433f2ae47cf5d1
3
  size 5944