Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wzhouad commited on
Commit
b35357b
•
1 Parent(s): d110e33

Model save

Browse files
Files changed (8) hide show
  1. README.md +17 -1
  2. all_results.json +5 -5
  3. model-00001-of-00003.safetensors +1 -1
  4. model-00002-of-00003.safetensors +1 -1
  5. model-00003-of-00003.safetensors +1 -1
  6. train_results.json +5 -5
  7. trainer_state.json +616 -454
  8. training_args.bin +1 -1
README.md CHANGED
@@ -16,6 +16,16 @@ should probably proofread and complete it, then remove this comment. -->
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 
 
 
 
 
 
 
 
 
19
 
20
  ## Model description
21
 
@@ -37,7 +47,7 @@ The following hyperparameters were used during training:
37
  - learning_rate: 5e-07
38
  - train_batch_size: 8
39
  - eval_batch_size: 8
40
- - seed: 2
41
  - distributed_type: multi-GPU
42
  - num_devices: 8
43
  - gradient_accumulation_steps: 2
@@ -50,6 +60,12 @@ The following hyperparameters were used during training:
50
 
51
  ### Training results
52
 
 
 
 
 
 
 
53
 
54
 
55
  ### Framework versions
 
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
+ It achieves the following results on the evaluation set:
20
+ - Loss: 0.0647
21
+ - Rewards/chosen: -1.7376
22
+ - Rewards/rejected: -2.4950
23
+ - Rewards/accuracies: 0.7227
24
+ - Rewards/margins: 0.7575
25
+ - Logps/rejected: -506.8580
26
+ - Logps/chosen: -430.7989
27
+ - Logits/rejected: -2.2904
28
+ - Logits/chosen: -2.3071
29
 
30
  ## Model description
31
 
 
47
  - learning_rate: 5e-07
48
  - train_batch_size: 8
49
  - eval_batch_size: 8
50
+ - seed: 1
51
  - distributed_type: multi-GPU
52
  - num_devices: 8
53
  - gradient_accumulation_steps: 2
 
60
 
61
  ### Training results
62
 
63
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 0.1116 | 0.21 | 100 | 0.1134 | -0.6750 | -1.0356 | 0.6680 | 0.3607 | -360.9161 | -324.5356 | -2.7165 | -2.7313 |
66
+ | 0.0842 | 0.42 | 200 | 0.0857 | -0.9947 | -1.6349 | 0.7109 | 0.6402 | -420.8391 | -356.5085 | -2.5327 | -2.5472 |
67
+ | 0.0618 | 0.63 | 300 | 0.0657 | -1.6331 | -2.3472 | 0.7227 | 0.7141 | -492.0749 | -420.3542 | -2.3598 | -2.3751 |
68
+ | 0.0678 | 0.84 | 400 | 0.0647 | -1.7376 | -2.4950 | 0.7227 | 0.7575 | -506.8580 | -430.7989 | -2.2904 | -2.3071 |
69
 
70
 
71
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.1212116445656176,
4
- "train_runtime": 3243.8169,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 15.998,
7
- "train_steps_per_second": 0.125
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.10414429289526521,
4
+ "train_runtime": 3960.4871,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.436,
7
+ "train_steps_per_second": 0.121
8
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a96a8f59a3234f45c373e15314f297b1df0813fe7ae34e8fb0bf712151eb7557
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa2eebd9f84367836d1bfbeefa89db5cf9eebf0591011ea641f93647899bb724
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:871de568f23629d6f51ce1f757f84fb8c3db7ec5e1cd2f45bb7ed93a75c85413
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1892eec6aa1f3e10aa42b60e4938d1158a2bb7ca1cd8b5f3548e0ca830255e53
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9599c26b77fb588d3bafc982a1354089f7e5e1107510c4f6d943f5f1e8c752dc
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ee31fcfc2de32c32ae8591c1cba8df2a10bf87a57f763d1399cb34933766e15
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.1212116445656176,
4
- "train_runtime": 3243.8169,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 15.998,
7
- "train_steps_per_second": 0.125
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.10414429289526521,
4
+ "train_runtime": 3960.4871,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.436,
7
+ "train_steps_per_second": 0.121
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.998766954377312,
5
- "eval_steps": 1000,
6
- "global_step": 405,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 1.2195121951219512e-08,
14
- "logits/chosen": -2.8412017822265625,
15
- "logits/rejected": -2.798035144805908,
16
- "logps/chosen": -481.1488037109375,
17
- "logps/rejected": -89.30835723876953,
18
- "loss": 0.4086,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -24,576 +24,738 @@
24
  },
25
  {
26
  "epoch": 0.02,
27
- "learning_rate": 1.219512195121951e-07,
28
- "logits/chosen": -2.760491132736206,
29
- "logits/rejected": -2.721938133239746,
30
- "logps/chosen": -418.5057678222656,
31
- "logps/rejected": -116.76482391357422,
32
- "loss": 0.4084,
33
- "rewards/accuracies": 0.5555555820465088,
34
- "rewards/chosen": 0.0007802958134561777,
35
- "rewards/margins": 0.0012957266299054027,
36
- "rewards/rejected": -0.000515430816449225,
37
  "step": 10
38
  },
39
  {
40
- "epoch": 0.05,
41
- "learning_rate": 2.439024390243902e-07,
42
- "logits/chosen": -2.7858235836029053,
43
- "logits/rejected": -2.739994764328003,
44
- "logps/chosen": -395.92962646484375,
45
- "logps/rejected": -109.29634094238281,
46
- "loss": 0.4172,
47
- "rewards/accuracies": 0.7562500238418579,
48
- "rewards/chosen": 0.014929535798728466,
49
- "rewards/margins": 0.029574846848845482,
50
- "rewards/rejected": -0.014645315706729889,
51
  "step": 20
52
  },
53
  {
54
- "epoch": 0.07,
55
- "learning_rate": 3.6585365853658536e-07,
56
- "logits/chosen": -2.707629919052124,
57
- "logits/rejected": -2.673509359359741,
58
- "logps/chosen": -375.37286376953125,
59
- "logps/rejected": -121.48030853271484,
60
- "loss": 0.3945,
61
- "rewards/accuracies": 0.78125,
62
- "rewards/chosen": 0.06351201981306076,
63
- "rewards/margins": 0.17929671704769135,
64
- "rewards/rejected": -0.11578470468521118,
65
  "step": 30
66
  },
67
  {
68
- "epoch": 0.1,
69
- "learning_rate": 4.878048780487804e-07,
70
- "logits/chosen": -2.5399930477142334,
71
- "logits/rejected": -2.5156164169311523,
72
- "logps/chosen": -426.90631103515625,
73
- "logps/rejected": -164.5863800048828,
74
- "loss": 0.3588,
75
- "rewards/accuracies": 0.84375,
76
- "rewards/chosen": 0.06860475242137909,
77
- "rewards/margins": 0.513036847114563,
78
- "rewards/rejected": -0.4444320797920227,
79
  "step": 40
80
  },
81
  {
82
- "epoch": 0.12,
83
- "learning_rate": 4.992461696250783e-07,
84
- "logits/chosen": -2.426210403442383,
85
- "logits/rejected": -2.422363042831421,
86
- "logps/chosen": -366.1980285644531,
87
- "logps/rejected": -200.22198486328125,
88
- "loss": 0.293,
89
- "rewards/accuracies": 0.768750011920929,
90
- "rewards/chosen": -0.1592792570590973,
91
- "rewards/margins": 0.7571996450424194,
92
- "rewards/rejected": -0.9164789319038391,
93
  "step": 50
94
  },
95
  {
96
- "epoch": 0.15,
97
- "learning_rate": 4.966461721767899e-07,
98
- "logits/chosen": -2.38622784614563,
99
- "logits/rejected": -2.3021979331970215,
100
- "logps/chosen": -491.4566955566406,
101
- "logps/rejected": -255.14169311523438,
102
- "loss": 0.2145,
103
- "rewards/accuracies": 0.8187500238418579,
104
- "rewards/chosen": -0.23122337460517883,
105
- "rewards/margins": 1.2772090435028076,
106
- "rewards/rejected": -1.5084326267242432,
107
  "step": 60
108
  },
109
  {
110
- "epoch": 0.17,
111
- "learning_rate": 4.922100518015975e-07,
112
- "logits/chosen": -2.350123643875122,
113
- "logits/rejected": -2.2967426776885986,
114
- "logps/chosen": -493.40936279296875,
115
- "logps/rejected": -324.7308044433594,
116
- "loss": 0.1433,
117
- "rewards/accuracies": 0.793749988079071,
118
- "rewards/chosen": -0.6797299385070801,
119
- "rewards/margins": 1.5318721532821655,
120
- "rewards/rejected": -2.211601972579956,
121
  "step": 70
122
  },
123
  {
124
- "epoch": 0.2,
125
- "learning_rate": 4.859708325770919e-07,
126
- "logits/chosen": -2.3943259716033936,
127
- "logits/rejected": -2.342501163482666,
128
- "logps/chosen": -480.4920959472656,
129
- "logps/rejected": -323.8282775878906,
130
- "loss": 0.1593,
131
- "rewards/accuracies": 0.762499988079071,
132
- "rewards/chosen": -0.47934165596961975,
133
- "rewards/margins": 1.7221958637237549,
134
- "rewards/rejected": -2.201537609100342,
135
  "step": 80
136
  },
137
  {
138
- "epoch": 0.22,
139
- "learning_rate": 4.779749614980225e-07,
140
- "logits/chosen": -2.350449800491333,
141
- "logits/rejected": -2.2739322185516357,
142
- "logps/chosen": -466.1890563964844,
143
- "logps/rejected": -354.5547180175781,
144
- "loss": 0.1338,
145
- "rewards/accuracies": 0.824999988079071,
146
- "rewards/chosen": -0.593730628490448,
147
- "rewards/margins": 1.857090711593628,
148
- "rewards/rejected": -2.4508211612701416,
149
  "step": 90
150
  },
151
  {
152
- "epoch": 0.25,
153
- "learning_rate": 4.682819627081427e-07,
154
- "logits/chosen": -2.3722572326660156,
155
- "logits/rejected": -2.292999029159546,
156
- "logps/chosen": -479.42425537109375,
157
- "logps/rejected": -326.6515808105469,
158
- "loss": 0.1447,
159
- "rewards/accuracies": 0.84375,
160
- "rewards/chosen": -0.455019474029541,
161
- "rewards/margins": 1.7410026788711548,
162
- "rewards/rejected": -2.1960222721099854,
163
  "step": 100
164
  },
165
  {
166
- "epoch": 0.27,
167
- "learning_rate": 4.569639943810477e-07,
168
- "logits/chosen": -2.338348388671875,
169
- "logits/rejected": -2.2420578002929688,
170
- "logps/chosen": -526.6338500976562,
171
- "logps/rejected": -412.49298095703125,
172
- "loss": 0.0927,
173
- "rewards/accuracies": 0.7749999761581421,
174
- "rewards/chosen": -0.917411208152771,
175
- "rewards/margins": 2.043461322784424,
176
- "rewards/rejected": -2.9608726501464844,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
  "step": 110
178
  },
179
  {
180
- "epoch": 0.3,
181
- "learning_rate": 4.4410531154874543e-07,
182
- "logits/chosen": -2.3342597484588623,
183
- "logits/rejected": -2.230992078781128,
184
- "logps/chosen": -502.6327209472656,
185
- "logps/rejected": -375.29559326171875,
186
- "loss": 0.103,
187
- "rewards/accuracies": 0.856249988079071,
188
- "rewards/chosen": -0.7662944793701172,
189
- "rewards/margins": 1.8728885650634766,
190
- "rewards/rejected": -2.6391828060150146,
191
  "step": 120
192
  },
193
  {
194
- "epoch": 0.32,
195
- "learning_rate": 4.298016388768561e-07,
196
- "logits/chosen": -2.321582078933716,
197
- "logits/rejected": -2.228271961212158,
198
- "logps/chosen": -482.85345458984375,
199
- "logps/rejected": -400.3795166015625,
200
- "loss": 0.0895,
201
- "rewards/accuracies": 0.887499988079071,
202
- "rewards/chosen": -0.8461068868637085,
203
- "rewards/margins": 2.0464694499969482,
204
- "rewards/rejected": -2.8925764560699463,
205
  "step": 130
206
  },
207
  {
208
- "epoch": 0.35,
209
- "learning_rate": 4.1415945805573005e-07,
210
- "logits/chosen": -2.272134304046631,
211
- "logits/rejected": -2.164999485015869,
212
- "logps/chosen": -509.80596923828125,
213
- "logps/rejected": -427.83056640625,
214
- "loss": 0.0791,
215
- "rewards/accuracies": 0.8187500238418579,
216
- "rewards/chosen": -0.8160644769668579,
217
- "rewards/margins": 2.3560843467712402,
218
- "rewards/rejected": -3.1721489429473877,
219
  "step": 140
220
  },
221
  {
222
- "epoch": 0.37,
223
- "learning_rate": 3.972952151123984e-07,
224
- "logits/chosen": -2.247396230697632,
225
- "logits/rejected": -2.182502031326294,
226
- "logps/chosen": -508.01214599609375,
227
- "logps/rejected": -420.20452880859375,
228
- "loss": 0.0797,
229
- "rewards/accuracies": 0.8062499761581421,
230
- "rewards/chosen": -1.0292510986328125,
231
- "rewards/margins": 1.8891801834106445,
232
- "rewards/rejected": -2.918431282043457,
233
  "step": 150
234
  },
235
  {
236
- "epoch": 0.39,
237
- "learning_rate": 3.793344535444142e-07,
238
- "logits/chosen": -2.2382795810699463,
239
- "logits/rejected": -2.133326292037964,
240
- "logps/chosen": -566.8648681640625,
241
- "logps/rejected": -486.3053283691406,
242
- "loss": 0.0786,
243
- "rewards/accuracies": 0.793749988079071,
244
- "rewards/chosen": -1.4423755407333374,
245
- "rewards/margins": 2.125662088394165,
246
- "rewards/rejected": -3.568037509918213,
247
  "step": 160
248
  },
249
  {
250
- "epoch": 0.42,
251
- "learning_rate": 3.604108797288461e-07,
252
- "logits/chosen": -2.2419562339782715,
253
- "logits/rejected": -2.1632020473480225,
254
- "logps/chosen": -515.9493408203125,
255
- "logps/rejected": -404.2837829589844,
256
- "loss": 0.0838,
257
- "rewards/accuracies": 0.8500000238418579,
258
- "rewards/chosen": -0.9661751985549927,
259
- "rewards/margins": 2.032658100128174,
260
- "rewards/rejected": -2.998833179473877,
261
  "step": 170
262
  },
263
  {
264
- "epoch": 0.44,
265
- "learning_rate": 3.40665367563858e-07,
266
- "logits/chosen": -2.2232542037963867,
267
- "logits/rejected": -2.1410865783691406,
268
- "logps/chosen": -539.5465087890625,
269
- "logps/rejected": -444.18353271484375,
270
- "loss": 0.0752,
271
- "rewards/accuracies": 0.800000011920929,
272
- "rewards/chosen": -1.1667969226837158,
273
- "rewards/margins": 2.0110697746276855,
274
- "rewards/rejected": -3.1778664588928223,
275
  "step": 180
276
  },
277
  {
278
- "epoch": 0.47,
279
- "learning_rate": 3.202449097526798e-07,
280
- "logits/chosen": -2.2497739791870117,
281
- "logits/rejected": -2.1552443504333496,
282
- "logps/chosen": -548.9888916015625,
283
- "logps/rejected": -500.2518005371094,
284
- "loss": 0.0713,
285
- "rewards/accuracies": 0.824999988079071,
286
- "rewards/chosen": -1.1663403511047363,
287
- "rewards/margins": 2.576674699783325,
288
- "rewards/rejected": -3.7430148124694824,
289
  "step": 190
290
  },
291
  {
292
- "epoch": 0.49,
293
- "learning_rate": 2.993015235369905e-07,
294
- "logits/chosen": -2.2876505851745605,
295
- "logits/rejected": -2.19895339012146,
296
- "logps/chosen": -559.0614013671875,
297
- "logps/rejected": -472.5868225097656,
298
- "loss": 0.0772,
299
- "rewards/accuracies": 0.824999988079071,
300
- "rewards/chosen": -1.1247196197509766,
301
- "rewards/margins": 2.44679856300354,
302
- "rewards/rejected": -3.5715184211730957,
303
  "step": 200
304
  },
305
  {
306
- "epoch": 0.52,
307
- "learning_rate": 2.7799111902582693e-07,
308
- "logits/chosen": -2.246555805206299,
309
- "logits/rejected": -2.1683576107025146,
310
- "logps/chosen": -541.7609252929688,
311
- "logps/rejected": -469.8389587402344,
312
- "loss": 0.0822,
313
- "rewards/accuracies": 0.824999988079071,
314
- "rewards/chosen": -1.1446309089660645,
315
- "rewards/margins": 2.2059168815612793,
316
- "rewards/rejected": -3.3505473136901855,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  "step": 210
318
  },
319
  {
320
- "epoch": 0.54,
321
- "learning_rate": 2.564723385445869e-07,
322
- "logits/chosen": -2.216153621673584,
323
- "logits/rejected": -2.1477255821228027,
324
- "logps/chosen": -498.4830017089844,
325
- "logps/rejected": -422.98095703125,
326
- "loss": 0.0916,
327
- "rewards/accuracies": 0.8374999761581421,
328
- "rewards/chosen": -1.1203062534332275,
329
- "rewards/margins": 1.7864818572998047,
330
- "rewards/rejected": -2.906787872314453,
331
  "step": 220
332
  },
333
  {
334
- "epoch": 0.57,
335
- "learning_rate": 2.3490537564442845e-07,
336
- "logits/chosen": -2.2267045974731445,
337
- "logits/rejected": -2.1489691734313965,
338
- "logps/chosen": -498.1656188964844,
339
- "logps/rejected": -436.73345947265625,
340
- "loss": 0.0785,
341
- "rewards/accuracies": 0.84375,
342
- "rewards/chosen": -1.1378052234649658,
343
- "rewards/margins": 2.084202289581299,
344
- "rewards/rejected": -3.222008228302002,
345
  "step": 230
346
  },
347
  {
348
- "epoch": 0.59,
349
- "learning_rate": 2.1345078256378801e-07,
350
- "logits/chosen": -2.1999192237854004,
351
- "logits/rejected": -2.0882999897003174,
352
- "logps/chosen": -517.1456298828125,
353
- "logps/rejected": -464.6072692871094,
354
- "loss": 0.0721,
355
- "rewards/accuracies": 0.831250011920929,
356
- "rewards/chosen": -1.4198516607284546,
357
- "rewards/margins": 2.0667641162872314,
358
- "rewards/rejected": -3.4866161346435547,
359
  "step": 240
360
  },
361
  {
362
- "epoch": 0.62,
363
- "learning_rate": 1.9226827501969865e-07,
364
- "logits/chosen": -2.2528138160705566,
365
- "logits/rejected": -2.141622304916382,
366
- "logps/chosen": -545.8248901367188,
367
- "logps/rejected": -453.86181640625,
368
- "loss": 0.0707,
369
- "rewards/accuracies": 0.887499988079071,
370
- "rewards/chosen": -1.0264685153961182,
371
- "rewards/margins": 2.3678536415100098,
372
- "rewards/rejected": -3.394321918487549,
373
  "step": 250
374
  },
375
  {
376
- "epoch": 0.64,
377
- "learning_rate": 1.715155432264775e-07,
378
- "logits/chosen": -2.2155823707580566,
379
- "logits/rejected": -2.1469390392303467,
380
- "logps/chosen": -520.0255737304688,
381
- "logps/rejected": -439.7793884277344,
382
- "loss": 0.0718,
383
- "rewards/accuracies": 0.793749988079071,
384
- "rewards/chosen": -1.084135890007019,
385
- "rewards/margins": 2.117866039276123,
386
- "rewards/rejected": -3.2020020484924316,
387
  "step": 260
388
  },
389
  {
390
- "epoch": 0.67,
391
- "learning_rate": 1.51347077992983e-07,
392
- "logits/chosen": -2.243293285369873,
393
- "logits/rejected": -2.1296281814575195,
394
- "logps/chosen": -515.3222045898438,
395
- "logps/rejected": -466.12689208984375,
396
- "loss": 0.0708,
397
- "rewards/accuracies": 0.8374999761581421,
398
- "rewards/chosen": -1.0565736293792725,
399
- "rewards/margins": 2.363142728805542,
400
- "rewards/rejected": -3.4197163581848145,
401
  "step": 270
402
  },
403
  {
404
- "epoch": 0.69,
405
- "learning_rate": 1.3191302063739906e-07,
406
- "logits/chosen": -2.2050838470458984,
407
- "logits/rejected": -2.0947039127349854,
408
- "logps/chosen": -512.220947265625,
409
- "logps/rejected": -453.70501708984375,
410
- "loss": 0.0686,
411
- "rewards/accuracies": 0.875,
412
- "rewards/chosen": -1.0570018291473389,
413
- "rewards/margins": 2.3901729583740234,
414
- "rewards/rejected": -3.4471752643585205,
415
  "step": 280
416
  },
417
  {
418
- "epoch": 0.72,
419
- "learning_rate": 1.1335804528119475e-07,
420
- "logits/chosen": -2.2694661617279053,
421
- "logits/rejected": -2.159458875656128,
422
- "logps/chosen": -525.7506713867188,
423
- "logps/rejected": -468.66949462890625,
424
- "loss": 0.0685,
425
- "rewards/accuracies": 0.8374999761581421,
426
- "rewards/chosen": -1.2382738590240479,
427
- "rewards/margins": 2.2864315509796143,
428
- "rewards/rejected": -3.524705410003662,
429
  "step": 290
430
  },
431
  {
432
- "epoch": 0.74,
433
- "learning_rate": 9.582028184286423e-08,
434
- "logits/chosen": -2.252445936203003,
435
- "logits/rejected": -2.1464462280273438,
436
- "logps/chosen": -572.5509643554688,
437
- "logps/rejected": -493.2998046875,
438
- "loss": 0.0736,
439
- "rewards/accuracies": 0.856249988079071,
440
- "rewards/chosen": -1.1083300113677979,
441
- "rewards/margins": 2.5646698474884033,
442
- "rewards/rejected": -3.672999620437622,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
443
  "step": 300
444
  },
445
  {
446
- "epoch": 0.76,
447
- "learning_rate": 7.943028774907065e-08,
448
- "logits/chosen": -2.2152347564697266,
449
- "logits/rejected": -2.114929437637329,
450
- "logps/chosen": -549.380126953125,
451
- "logps/rejected": -491.04229736328125,
452
- "loss": 0.0655,
453
- "rewards/accuracies": 0.8374999761581421,
454
- "rewards/chosen": -1.296120047569275,
455
- "rewards/margins": 2.3816938400268555,
456
- "rewards/rejected": -3.677813768386841,
457
  "step": 310
458
  },
459
  {
460
- "epoch": 0.79,
461
- "learning_rate": 6.431007601814637e-08,
462
- "logits/chosen": -2.2423970699310303,
463
- "logits/rejected": -2.14815616607666,
464
- "logps/chosen": -539.1422119140625,
465
- "logps/rejected": -484.86474609375,
466
- "loss": 0.0688,
467
- "rewards/accuracies": 0.8125,
468
- "rewards/chosen": -1.375579595565796,
469
- "rewards/margins": 2.2692549228668213,
470
- "rewards/rejected": -3.644834041595459,
471
  "step": 320
472
  },
473
  {
474
- "epoch": 0.81,
475
- "learning_rate": 5.0572206951246e-08,
476
- "logits/chosen": -2.257805824279785,
477
- "logits/rejected": -2.1624794006347656,
478
- "logps/chosen": -547.17578125,
479
- "logps/rejected": -499.7318420410156,
480
- "loss": 0.0798,
481
- "rewards/accuracies": 0.8374999761581421,
482
- "rewards/chosen": -1.2940938472747803,
483
- "rewards/margins": 2.3461661338806152,
484
- "rewards/rejected": -3.6402602195739746,
485
  "step": 330
486
  },
487
  {
488
- "epoch": 0.84,
489
- "learning_rate": 3.831895019292897e-08,
490
- "logits/chosen": -2.2013723850250244,
491
- "logits/rejected": -2.080230474472046,
492
- "logps/chosen": -506.1536560058594,
493
- "logps/rejected": -451.22857666015625,
494
- "loss": 0.0659,
495
- "rewards/accuracies": 0.8500000238418579,
496
- "rewards/chosen": -1.195672631263733,
497
- "rewards/margins": 2.2886624336242676,
498
- "rewards/rejected": -3.484334945678711,
499
  "step": 340
500
  },
501
  {
502
- "epoch": 0.86,
503
- "learning_rate": 2.764152339909756e-08,
504
- "logits/chosen": -2.26171612739563,
505
- "logits/rejected": -2.1387059688568115,
506
- "logps/chosen": -543.2042846679688,
507
- "logps/rejected": -484.20233154296875,
508
- "loss": 0.0565,
509
- "rewards/accuracies": 0.8687499761581421,
510
- "rewards/chosen": -1.251793622970581,
511
- "rewards/margins": 2.41098690032959,
512
- "rewards/rejected": -3.66278076171875,
513
  "step": 350
514
  },
515
  {
516
- "epoch": 0.89,
517
- "learning_rate": 1.861941317991664e-08,
518
- "logits/chosen": -2.231370449066162,
519
- "logits/rejected": -2.127504825592041,
520
- "logps/chosen": -501.075927734375,
521
- "logps/rejected": -474.69140625,
522
- "loss": 0.0662,
523
- "rewards/accuracies": 0.84375,
524
- "rewards/chosen": -1.3116228580474854,
525
- "rewards/margins": 2.331501007080078,
526
- "rewards/rejected": -3.6431241035461426,
527
  "step": 360
528
  },
529
  {
530
- "epoch": 0.91,
531
- "learning_rate": 1.13197833728636e-08,
532
- "logits/chosen": -2.2208430767059326,
533
- "logits/rejected": -2.110964298248291,
534
- "logps/chosen": -540.8277587890625,
535
- "logps/rejected": -475.9794006347656,
536
- "loss": 0.0617,
537
- "rewards/accuracies": 0.862500011920929,
538
- "rewards/chosen": -1.1789363622665405,
539
- "rewards/margins": 2.4740519523620605,
540
- "rewards/rejected": -3.6529879570007324,
541
  "step": 370
542
  },
543
  {
544
- "epoch": 0.94,
545
- "learning_rate": 5.79697505093521e-09,
546
- "logits/chosen": -2.1521689891815186,
547
- "logits/rejected": -2.0585670471191406,
548
- "logps/chosen": -530.9478759765625,
549
- "logps/rejected": -478.56365966796875,
550
- "loss": 0.0646,
551
- "rewards/accuracies": 0.8687499761581421,
552
- "rewards/chosen": -1.3565860986709595,
553
- "rewards/margins": 2.3372037410736084,
554
- "rewards/rejected": -3.6937899589538574,
555
  "step": 380
556
  },
557
  {
558
- "epoch": 0.96,
559
- "learning_rate": 2.092101988131256e-09,
560
- "logits/chosen": -2.2695422172546387,
561
- "logits/rejected": -2.1558218002319336,
562
- "logps/chosen": -576.0869750976562,
563
- "logps/rejected": -490.7723693847656,
564
- "loss": 0.0611,
565
- "rewards/accuracies": 0.862500011920929,
566
- "rewards/chosen": -1.238217830657959,
567
- "rewards/margins": 2.507056713104248,
568
- "rewards/rejected": -3.745274782180786,
569
  "step": 390
570
  },
571
  {
572
- "epoch": 0.99,
573
- "learning_rate": 2.327445937151673e-10,
574
- "logits/chosen": -2.2124767303466797,
575
- "logits/rejected": -2.11603045463562,
576
- "logps/chosen": -578.0953979492188,
577
- "logps/rejected": -490.62139892578125,
578
- "loss": 0.0567,
579
- "rewards/accuracies": 0.8125,
580
- "rewards/chosen": -1.406029462814331,
581
- "rewards/margins": 2.2935640811920166,
582
- "rewards/rejected": -3.6995933055877686,
583
  "step": 400
584
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
585
  {
586
  "epoch": 1.0,
587
- "step": 405,
588
  "total_flos": 0.0,
589
- "train_loss": 0.1212116445656176,
590
- "train_runtime": 3243.8169,
591
- "train_samples_per_second": 15.998,
592
- "train_steps_per_second": 0.125
593
  }
594
  ],
595
  "logging_steps": 10,
596
- "max_steps": 405,
597
  "num_train_epochs": 1,
598
  "save_steps": 1000,
599
  "total_flos": 0.0,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 100,
6
+ "global_step": 478,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.0416666666666666e-08,
14
+ "logits/chosen": -2.847970962524414,
15
+ "logits/rejected": -2.79160213470459,
16
+ "logps/chosen": -284.9612731933594,
17
+ "logps/rejected": -276.45928955078125,
18
+ "loss": 0.2876,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
24
  },
25
  {
26
  "epoch": 0.02,
27
+ "learning_rate": 1.0416666666666667e-07,
28
+ "logits/chosen": -2.7547035217285156,
29
+ "logits/rejected": -2.752760887145996,
30
+ "logps/chosen": -249.89892578125,
31
+ "logps/rejected": -223.0311279296875,
32
+ "loss": 0.2721,
33
+ "rewards/accuracies": 0.4444444477558136,
34
+ "rewards/chosen": 0.0004881812783423811,
35
+ "rewards/margins": 0.00032018052297644317,
36
+ "rewards/rejected": 0.00016800071171019226,
37
  "step": 10
38
  },
39
  {
40
+ "epoch": 0.04,
41
+ "learning_rate": 2.0833333333333333e-07,
42
+ "logits/chosen": -2.744856834411621,
43
+ "logits/rejected": -2.745373010635376,
44
+ "logps/chosen": -257.4269104003906,
45
+ "logps/rejected": -247.50137329101562,
46
+ "loss": 0.2734,
47
+ "rewards/accuracies": 0.550000011920929,
48
+ "rewards/chosen": 0.0002862844557967037,
49
+ "rewards/margins": 0.0009933979017660022,
50
+ "rewards/rejected": -0.0007071135332807899,
51
  "step": 20
52
  },
53
  {
54
+ "epoch": 0.06,
55
+ "learning_rate": 3.1249999999999997e-07,
56
+ "logits/chosen": -2.8001036643981934,
57
+ "logits/rejected": -2.7527127265930176,
58
+ "logps/chosen": -300.578857421875,
59
+ "logps/rejected": -261.8973083496094,
60
+ "loss": 0.2748,
61
+ "rewards/accuracies": 0.7124999761581421,
62
+ "rewards/chosen": -1.820516808948014e-05,
63
+ "rewards/margins": 0.008036722429096699,
64
+ "rewards/rejected": -0.008054928854107857,
65
  "step": 30
66
  },
67
  {
68
+ "epoch": 0.08,
69
+ "learning_rate": 4.1666666666666667e-07,
70
+ "logits/chosen": -2.760335683822632,
71
+ "logits/rejected": -2.7481069564819336,
72
+ "logps/chosen": -256.87750244140625,
73
+ "logps/rejected": -274.76934814453125,
74
+ "loss": 0.2752,
75
+ "rewards/accuracies": 0.6875,
76
+ "rewards/chosen": -0.004436298739165068,
77
+ "rewards/margins": 0.022424206137657166,
78
+ "rewards/rejected": -0.02686050534248352,
79
  "step": 40
80
  },
81
  {
82
+ "epoch": 0.1,
83
+ "learning_rate": 4.999733114418725e-07,
84
+ "logits/chosen": -2.7565062046051025,
85
+ "logits/rejected": -2.728579044342041,
86
+ "logps/chosen": -286.0480041503906,
87
+ "logps/rejected": -257.09088134765625,
88
+ "loss": 0.2618,
89
+ "rewards/accuracies": 0.643750011920929,
90
+ "rewards/chosen": -0.03968585282564163,
91
+ "rewards/margins": 0.054190658032894135,
92
+ "rewards/rejected": -0.09387651085853577,
93
  "step": 50
94
  },
95
  {
96
+ "epoch": 0.13,
97
+ "learning_rate": 4.990398100856366e-07,
98
+ "logits/chosen": -2.7141618728637695,
99
+ "logits/rejected": -2.7023863792419434,
100
+ "logps/chosen": -286.9661560058594,
101
+ "logps/rejected": -260.3489685058594,
102
+ "loss": 0.24,
103
+ "rewards/accuracies": 0.6625000238418579,
104
+ "rewards/chosen": -0.11081676185131073,
105
+ "rewards/margins": 0.09219430387020111,
106
+ "rewards/rejected": -0.20301106572151184,
107
  "step": 60
108
  },
109
  {
110
+ "epoch": 0.15,
111
+ "learning_rate": 4.967775735898179e-07,
112
+ "logits/chosen": -2.7555909156799316,
113
+ "logits/rejected": -2.7137351036071777,
114
+ "logps/chosen": -284.78546142578125,
115
+ "logps/rejected": -262.90203857421875,
116
+ "loss": 0.2086,
117
+ "rewards/accuracies": 0.668749988079071,
118
+ "rewards/chosen": -0.17650075256824493,
119
+ "rewards/margins": 0.13311314582824707,
120
+ "rewards/rejected": -0.3096138834953308,
121
  "step": 70
122
  },
123
  {
124
+ "epoch": 0.17,
125
+ "learning_rate": 4.931986719649298e-07,
126
+ "logits/chosen": -2.7589242458343506,
127
+ "logits/rejected": -2.7324187755584717,
128
+ "logps/chosen": -284.3084411621094,
129
+ "logps/rejected": -316.8328857421875,
130
+ "loss": 0.1772,
131
+ "rewards/accuracies": 0.6312500238418579,
132
+ "rewards/chosen": -0.23955480754375458,
133
+ "rewards/margins": 0.20579905807971954,
134
+ "rewards/rejected": -0.4453538954257965,
135
  "step": 80
136
  },
137
  {
138
+ "epoch": 0.19,
139
+ "learning_rate": 4.883222001996351e-07,
140
+ "logits/chosen": -2.7832489013671875,
141
+ "logits/rejected": -2.758307695388794,
142
+ "logps/chosen": -321.67462158203125,
143
+ "logps/rejected": -327.8370666503906,
144
+ "loss": 0.1309,
145
+ "rewards/accuracies": 0.706250011920929,
146
+ "rewards/chosen": -0.4630777835845947,
147
+ "rewards/margins": 0.3033062815666199,
148
+ "rewards/rejected": -0.7663841247558594,
149
  "step": 90
150
  },
151
  {
152
+ "epoch": 0.21,
153
+ "learning_rate": 4.821741763807186e-07,
154
+ "logits/chosen": -2.7674899101257324,
155
+ "logits/rejected": -2.7356114387512207,
156
+ "logps/chosen": -354.02459716796875,
157
+ "logps/rejected": -358.9905700683594,
158
+ "loss": 0.1116,
159
+ "rewards/accuracies": 0.6875,
160
+ "rewards/chosen": -0.6505959630012512,
161
+ "rewards/margins": 0.3171614408493042,
162
+ "rewards/rejected": -0.9677574038505554,
163
  "step": 100
164
  },
165
  {
166
+ "epoch": 0.21,
167
+ "eval_logits/chosen": -2.731274127960205,
168
+ "eval_logits/rejected": -2.7165141105651855,
169
+ "eval_logps/chosen": -324.53558349609375,
170
+ "eval_logps/rejected": -360.9161071777344,
171
+ "eval_loss": 0.11337568610906601,
172
+ "eval_rewards/accuracies": 0.66796875,
173
+ "eval_rewards/chosen": -0.674959659576416,
174
+ "eval_rewards/margins": 0.36066874861717224,
175
+ "eval_rewards/rejected": -1.0356284379959106,
176
+ "eval_runtime": 53.5443,
177
+ "eval_samples_per_second": 37.352,
178
+ "eval_steps_per_second": 0.598,
179
+ "step": 100
180
+ },
181
+ {
182
+ "epoch": 0.23,
183
+ "learning_rate": 4.747874028753375e-07,
184
+ "logits/chosen": -2.6898159980773926,
185
+ "logits/rejected": -2.6853067874908447,
186
+ "logps/chosen": -333.8702697753906,
187
+ "logps/rejected": -386.415771484375,
188
+ "loss": 0.1045,
189
+ "rewards/accuracies": 0.71875,
190
+ "rewards/chosen": -0.696069598197937,
191
+ "rewards/margins": 0.44097524881362915,
192
+ "rewards/rejected": -1.1370446681976318,
193
  "step": 110
194
  },
195
  {
196
+ "epoch": 0.25,
197
+ "learning_rate": 4.662012913161997e-07,
198
+ "logits/chosen": -2.6818103790283203,
199
+ "logits/rejected": -2.6488022804260254,
200
+ "logps/chosen": -349.8975524902344,
201
+ "logps/rejected": -365.45355224609375,
202
+ "loss": 0.0967,
203
+ "rewards/accuracies": 0.637499988079071,
204
+ "rewards/chosen": -0.823577880859375,
205
+ "rewards/margins": 0.35716408491134644,
206
+ "rewards/rejected": -1.1807419061660767,
207
  "step": 120
208
  },
209
  {
210
+ "epoch": 0.27,
211
+ "learning_rate": 4.5646165232345103e-07,
212
+ "logits/chosen": -2.644099473953247,
213
+ "logits/rejected": -2.612097978591919,
214
+ "logps/chosen": -383.2303161621094,
215
+ "logps/rejected": -368.6571044921875,
216
+ "loss": 0.0963,
217
+ "rewards/accuracies": 0.668749988079071,
218
+ "rewards/chosen": -0.9109883308410645,
219
+ "rewards/margins": 0.37315014004707336,
220
+ "rewards/rejected": -1.2841384410858154,
221
  "step": 130
222
  },
223
  {
224
+ "epoch": 0.29,
225
+ "learning_rate": 4.456204510851956e-07,
226
+ "logits/chosen": -2.5526227951049805,
227
+ "logits/rejected": -2.5359854698181152,
228
+ "logps/chosen": -338.05474853515625,
229
+ "logps/rejected": -350.3619689941406,
230
+ "loss": 0.0897,
231
+ "rewards/accuracies": 0.606249988079071,
232
+ "rewards/chosen": -0.8969566226005554,
233
+ "rewards/margins": 0.40904563665390015,
234
+ "rewards/rejected": -1.3060022592544556,
235
  "step": 140
236
  },
237
  {
238
+ "epoch": 0.31,
239
+ "learning_rate": 4.337355301007335e-07,
240
+ "logits/chosen": -2.6042182445526123,
241
+ "logits/rejected": -2.5939276218414307,
242
+ "logps/chosen": -332.9805603027344,
243
+ "logps/rejected": -342.162109375,
244
+ "loss": 0.1113,
245
+ "rewards/accuracies": 0.6875,
246
+ "rewards/chosen": -0.6774733662605286,
247
+ "rewards/margins": 0.3444991409778595,
248
+ "rewards/rejected": -1.0219725370407104,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.33,
253
+ "learning_rate": 4.2087030056579986e-07,
254
+ "logits/chosen": -2.591458320617676,
255
+ "logits/rejected": -2.577643871307373,
256
+ "logps/chosen": -327.47149658203125,
257
+ "logps/rejected": -341.7593688964844,
258
+ "loss": 0.1044,
259
+ "rewards/accuracies": 0.6875,
260
+ "rewards/chosen": -0.6649166941642761,
261
+ "rewards/margins": 0.43734127283096313,
262
+ "rewards/rejected": -1.1022579669952393,
263
  "step": 160
264
  },
265
  {
266
+ "epoch": 0.36,
267
+ "learning_rate": 4.070934040463998e-07,
268
+ "logits/chosen": -2.5269131660461426,
269
+ "logits/rejected": -2.5048739910125732,
270
+ "logps/chosen": -358.63671875,
271
+ "logps/rejected": -383.0482177734375,
272
+ "loss": 0.0919,
273
+ "rewards/accuracies": 0.7124999761581421,
274
+ "rewards/chosen": -0.899802565574646,
275
+ "rewards/margins": 0.42440056800842285,
276
+ "rewards/rejected": -1.3242031335830688,
277
  "step": 170
278
  },
279
  {
280
+ "epoch": 0.38,
281
+ "learning_rate": 3.9247834624635404e-07,
282
+ "logits/chosen": -2.5789132118225098,
283
+ "logits/rejected": -2.538067579269409,
284
+ "logps/chosen": -374.2618103027344,
285
+ "logps/rejected": -407.06170654296875,
286
+ "loss": 0.0762,
287
+ "rewards/accuracies": 0.731249988079071,
288
+ "rewards/chosen": -0.9781246185302734,
289
+ "rewards/margins": 0.6130446791648865,
290
+ "rewards/rejected": -1.5911692380905151,
291
  "step": 180
292
  },
293
  {
294
+ "epoch": 0.4,
295
+ "learning_rate": 3.7710310482256523e-07,
296
+ "logits/chosen": -2.4771628379821777,
297
+ "logits/rejected": -2.4575798511505127,
298
+ "logps/chosen": -378.3219299316406,
299
+ "logps/rejected": -404.56805419921875,
300
+ "loss": 0.0802,
301
+ "rewards/accuracies": 0.699999988079071,
302
+ "rewards/chosen": -1.2110950946807861,
303
+ "rewards/margins": 0.4663744866847992,
304
+ "rewards/rejected": -1.6774696111679077,
305
  "step": 190
306
  },
307
  {
308
+ "epoch": 0.42,
309
+ "learning_rate": 3.610497133404795e-07,
310
+ "logits/chosen": -2.5874526500701904,
311
+ "logits/rejected": -2.5326857566833496,
312
+ "logps/chosen": -428.63458251953125,
313
+ "logps/rejected": -449.66937255859375,
314
+ "loss": 0.0842,
315
+ "rewards/accuracies": 0.6937500238418579,
316
+ "rewards/chosen": -0.9405802488327026,
317
+ "rewards/margins": 0.7570930123329163,
318
+ "rewards/rejected": -1.6976732015609741,
319
  "step": 200
320
  },
321
  {
322
+ "epoch": 0.42,
323
+ "eval_logits/chosen": -2.5472469329833984,
324
+ "eval_logits/rejected": -2.5327436923980713,
325
+ "eval_logps/chosen": -356.50848388671875,
326
+ "eval_logps/rejected": -420.8390808105469,
327
+ "eval_loss": 0.08571309596300125,
328
+ "eval_rewards/accuracies": 0.7109375,
329
+ "eval_rewards/chosen": -0.9946882724761963,
330
+ "eval_rewards/margins": 0.6401697397232056,
331
+ "eval_rewards/rejected": -1.6348580121994019,
332
+ "eval_runtime": 53.5097,
333
+ "eval_samples_per_second": 37.376,
334
+ "eval_steps_per_second": 0.598,
335
+ "step": 200
336
+ },
337
+ {
338
+ "epoch": 0.44,
339
+ "learning_rate": 3.4440382358952115e-07,
340
+ "logits/chosen": -2.513122081756592,
341
+ "logits/rejected": -2.4689273834228516,
342
+ "logps/chosen": -391.703857421875,
343
+ "logps/rejected": -414.6632385253906,
344
+ "loss": 0.0766,
345
+ "rewards/accuracies": 0.668749988079071,
346
+ "rewards/chosen": -1.2449716329574585,
347
+ "rewards/margins": 0.4973742365837097,
348
+ "rewards/rejected": -1.7423460483551025,
349
  "step": 210
350
  },
351
  {
352
+ "epoch": 0.46,
353
+ "learning_rate": 3.272542485937368e-07,
354
+ "logits/chosen": -2.482513904571533,
355
+ "logits/rejected": -2.471379518508911,
356
+ "logps/chosen": -419.3297424316406,
357
+ "logps/rejected": -430.79425048828125,
358
+ "loss": 0.0766,
359
+ "rewards/accuracies": 0.6937500238418579,
360
+ "rewards/chosen": -1.2476927042007446,
361
+ "rewards/margins": 0.6499053239822388,
362
+ "rewards/rejected": -1.8975979089736938,
363
  "step": 220
364
  },
365
  {
366
+ "epoch": 0.48,
367
+ "learning_rate": 3.096924887558854e-07,
368
+ "logits/chosen": -2.483941078186035,
369
+ "logits/rejected": -2.483389139175415,
370
+ "logps/chosen": -408.7511901855469,
371
+ "logps/rejected": -466.270751953125,
372
+ "loss": 0.0767,
373
+ "rewards/accuracies": 0.6812499761581421,
374
+ "rewards/chosen": -1.269411325454712,
375
+ "rewards/margins": 0.4846363663673401,
376
+ "rewards/rejected": -1.7540476322174072,
377
  "step": 230
378
  },
379
  {
380
+ "epoch": 0.5,
381
+ "learning_rate": 2.9181224366319943e-07,
382
+ "logits/chosen": -2.481611967086792,
383
+ "logits/rejected": -2.456181526184082,
384
+ "logps/chosen": -389.8509826660156,
385
+ "logps/rejected": -452.72149658203125,
386
+ "loss": 0.0771,
387
+ "rewards/accuracies": 0.6812499761581421,
388
+ "rewards/chosen": -1.3614814281463623,
389
+ "rewards/margins": 0.5804599523544312,
390
+ "rewards/rejected": -1.941941261291504,
391
  "step": 240
392
  },
393
  {
394
+ "epoch": 0.52,
395
+ "learning_rate": 2.7370891215954565e-07,
396
+ "logits/chosen": -2.4622182846069336,
397
+ "logits/rejected": -2.418468475341797,
398
+ "logps/chosen": -402.249755859375,
399
+ "logps/rejected": -437.009765625,
400
+ "loss": 0.0705,
401
+ "rewards/accuracies": 0.6937500238418579,
402
+ "rewards/chosen": -1.4332447052001953,
403
+ "rewards/margins": 0.5826060175895691,
404
+ "rewards/rejected": -2.015850782394409,
405
  "step": 250
406
  },
407
  {
408
+ "epoch": 0.54,
409
+ "learning_rate": 2.55479083351317e-07,
410
+ "logits/chosen": -2.4390504360198975,
411
+ "logits/rejected": -2.43849515914917,
412
+ "logps/chosen": -408.16400146484375,
413
+ "logps/rejected": -440.06536865234375,
414
+ "loss": 0.0761,
415
+ "rewards/accuracies": 0.6499999761581421,
416
+ "rewards/chosen": -1.3955520391464233,
417
+ "rewards/margins": 0.5204088687896729,
418
+ "rewards/rejected": -1.9159609079360962,
419
  "step": 260
420
  },
421
  {
422
+ "epoch": 0.56,
423
+ "learning_rate": 2.3722002126275822e-07,
424
+ "logits/chosen": -2.39439058303833,
425
+ "logits/rejected": -2.3895411491394043,
426
+ "logps/chosen": -426.22979736328125,
427
+ "logps/rejected": -473.0193786621094,
428
+ "loss": 0.067,
429
+ "rewards/accuracies": 0.768750011920929,
430
+ "rewards/chosen": -1.513763427734375,
431
+ "rewards/margins": 0.660820722579956,
432
+ "rewards/rejected": -2.174584150314331,
433
  "step": 270
434
  },
435
  {
436
+ "epoch": 0.59,
437
+ "learning_rate": 2.19029145890313e-07,
438
+ "logits/chosen": -2.3854241371154785,
439
+ "logits/rejected": -2.3567357063293457,
440
+ "logps/chosen": -406.0476379394531,
441
+ "logps/rejected": -449.47735595703125,
442
+ "loss": 0.0689,
443
+ "rewards/accuracies": 0.71875,
444
+ "rewards/chosen": -1.6464204788208008,
445
+ "rewards/margins": 0.5462825894355774,
446
+ "rewards/rejected": -2.1927027702331543,
447
  "step": 280
448
  },
449
  {
450
+ "epoch": 0.61,
451
+ "learning_rate": 2.0100351342479216e-07,
452
+ "logits/chosen": -2.3561172485351562,
453
+ "logits/rejected": -2.3167614936828613,
454
+ "logps/chosen": -395.48565673828125,
455
+ "logps/rejected": -440.73907470703125,
456
+ "loss": 0.0672,
457
+ "rewards/accuracies": 0.699999988079071,
458
+ "rewards/chosen": -1.4887441396713257,
459
+ "rewards/margins": 0.5762190818786621,
460
+ "rewards/rejected": -2.0649631023406982,
461
  "step": 290
462
  },
463
  {
464
+ "epoch": 0.63,
465
+ "learning_rate": 1.8323929841460178e-07,
466
+ "logits/chosen": -2.3569633960723877,
467
+ "logits/rejected": -2.326362371444702,
468
+ "logps/chosen": -421.4947204589844,
469
+ "logps/rejected": -477.8741760253906,
470
+ "loss": 0.0618,
471
+ "rewards/accuracies": 0.675000011920929,
472
+ "rewards/chosen": -1.6522613763809204,
473
+ "rewards/margins": 0.55296790599823,
474
+ "rewards/rejected": -2.2052292823791504,
475
+ "step": 300
476
+ },
477
+ {
478
+ "epoch": 0.63,
479
+ "eval_logits/chosen": -2.375107526779175,
480
+ "eval_logits/rejected": -2.359772205352783,
481
+ "eval_logps/chosen": -420.35418701171875,
482
+ "eval_logps/rejected": -492.0749206542969,
483
+ "eval_loss": 0.06571906059980392,
484
+ "eval_rewards/accuracies": 0.72265625,
485
+ "eval_rewards/chosen": -1.6331453323364258,
486
+ "eval_rewards/margins": 0.7140712738037109,
487
+ "eval_rewards/rejected": -2.3472166061401367,
488
+ "eval_runtime": 53.5158,
489
+ "eval_samples_per_second": 37.372,
490
+ "eval_steps_per_second": 0.598,
491
  "step": 300
492
  },
493
  {
494
+ "epoch": 0.65,
495
+ "learning_rate": 1.6583128063291573e-07,
496
+ "logits/chosen": -2.287219524383545,
497
+ "logits/rejected": -2.2284042835235596,
498
+ "logps/chosen": -444.0338439941406,
499
+ "logps/rejected": -484.8553771972656,
500
+ "loss": 0.0653,
501
+ "rewards/accuracies": 0.7875000238418579,
502
+ "rewards/chosen": -1.7085492610931396,
503
+ "rewards/margins": 0.6683012247085571,
504
+ "rewards/rejected": -2.3768503665924072,
505
  "step": 310
506
  },
507
  {
508
+ "epoch": 0.67,
509
+ "learning_rate": 1.488723393865766e-07,
510
+ "logits/chosen": -2.2944705486297607,
511
+ "logits/rejected": -2.2772765159606934,
512
+ "logps/chosen": -478.6187438964844,
513
+ "logps/rejected": -472.32958984375,
514
+ "loss": 0.0588,
515
+ "rewards/accuracies": 0.699999988079071,
516
+ "rewards/chosen": -1.8223636150360107,
517
+ "rewards/margins": 0.5468674898147583,
518
+ "rewards/rejected": -2.3692309856414795,
519
  "step": 320
520
  },
521
  {
522
+ "epoch": 0.69,
523
+ "learning_rate": 1.3245295796480788e-07,
524
+ "logits/chosen": -2.3150861263275146,
525
+ "logits/rejected": -2.2812893390655518,
526
+ "logps/chosen": -418.20782470703125,
527
+ "logps/rejected": -454.23974609375,
528
+ "loss": 0.0604,
529
+ "rewards/accuracies": 0.706250011920929,
530
+ "rewards/chosen": -1.6599462032318115,
531
+ "rewards/margins": 0.5461422801017761,
532
+ "rewards/rejected": -2.2060887813568115,
533
  "step": 330
534
  },
535
  {
536
+ "epoch": 0.71,
537
+ "learning_rate": 1.1666074087171627e-07,
538
+ "logits/chosen": -2.256033420562744,
539
+ "logits/rejected": -2.238840341567993,
540
+ "logps/chosen": -419.8775329589844,
541
+ "logps/rejected": -474.7325744628906,
542
+ "loss": 0.0681,
543
+ "rewards/accuracies": 0.699999988079071,
544
+ "rewards/chosen": -1.6863420009613037,
545
+ "rewards/margins": 0.6417552828788757,
546
+ "rewards/rejected": -2.328097343444824,
547
  "step": 340
548
  },
549
  {
550
+ "epoch": 0.73,
551
+ "learning_rate": 1.0157994641835734e-07,
552
+ "logits/chosen": -2.2696926593780518,
553
+ "logits/rejected": -2.240800380706787,
554
+ "logps/chosen": -401.9186706542969,
555
+ "logps/rejected": -481.47393798828125,
556
+ "loss": 0.0579,
557
+ "rewards/accuracies": 0.7250000238418579,
558
+ "rewards/chosen": -1.7596851587295532,
559
+ "rewards/margins": 0.6480830907821655,
560
+ "rewards/rejected": -2.4077677726745605,
561
  "step": 350
562
  },
563
  {
564
+ "epoch": 0.75,
565
+ "learning_rate": 8.729103716819111e-08,
566
+ "logits/chosen": -2.256950616836548,
567
+ "logits/rejected": -2.2109625339508057,
568
+ "logps/chosen": -444.003173828125,
569
+ "logps/rejected": -485.3224182128906,
570
+ "loss": 0.0574,
571
+ "rewards/accuracies": 0.6812499761581421,
572
+ "rewards/chosen": -1.7673498392105103,
573
+ "rewards/margins": 0.6924889087677002,
574
+ "rewards/rejected": -2.459838628768921,
575
  "step": 360
576
  },
577
  {
578
+ "epoch": 0.77,
579
+ "learning_rate": 7.387025063449081e-08,
580
+ "logits/chosen": -2.226203203201294,
581
+ "logits/rejected": -2.226749897003174,
582
+ "logps/chosen": -433.39007568359375,
583
+ "logps/rejected": -507.7019958496094,
584
+ "loss": 0.0557,
585
+ "rewards/accuracies": 0.7124999761581421,
586
+ "rewards/chosen": -1.781426191329956,
587
+ "rewards/margins": 0.7529138326644897,
588
+ "rewards/rejected": -2.5343403816223145,
589
  "step": 370
590
  },
591
  {
592
+ "epoch": 0.79,
593
+ "learning_rate": 6.138919252022435e-08,
594
+ "logits/chosen": -2.2622132301330566,
595
+ "logits/rejected": -2.2477574348449707,
596
+ "logps/chosen": -459.27008056640625,
597
+ "logps/rejected": -490.1805725097656,
598
+ "loss": 0.0554,
599
+ "rewards/accuracies": 0.737500011920929,
600
+ "rewards/chosen": -1.9344074726104736,
601
+ "rewards/margins": 0.5991317629814148,
602
+ "rewards/rejected": -2.5335395336151123,
603
  "step": 380
604
  },
605
  {
606
+ "epoch": 0.82,
607
+ "learning_rate": 4.991445467064689e-08,
608
+ "logits/chosen": -2.246565341949463,
609
+ "logits/rejected": -2.2325024604797363,
610
+ "logps/chosen": -434.74090576171875,
611
+ "logps/rejected": -499.4537658691406,
612
+ "loss": 0.0531,
613
+ "rewards/accuracies": 0.7250000238418579,
614
+ "rewards/chosen": -1.7754148244857788,
615
+ "rewards/margins": 0.7994632720947266,
616
+ "rewards/rejected": -2.574878215789795,
617
  "step": 390
618
  },
619
  {
620
+ "epoch": 0.84,
621
+ "learning_rate": 3.9507259776993954e-08,
622
+ "logits/chosen": -2.289376735687256,
623
+ "logits/rejected": -2.2795028686523438,
624
+ "logps/chosen": -482.32342529296875,
625
+ "logps/rejected": -517.7100219726562,
626
+ "loss": 0.0678,
627
+ "rewards/accuracies": 0.699999988079071,
628
+ "rewards/chosen": -1.821634292602539,
629
+ "rewards/margins": 0.6263509392738342,
630
+ "rewards/rejected": -2.4479854106903076,
631
  "step": 400
632
  },
633
+ {
634
+ "epoch": 0.84,
635
+ "eval_logits/chosen": -2.3071115016937256,
636
+ "eval_logits/rejected": -2.290386438369751,
637
+ "eval_logps/chosen": -430.7988586425781,
638
+ "eval_logps/rejected": -506.8580322265625,
639
+ "eval_loss": 0.06473570317029953,
640
+ "eval_rewards/accuracies": 0.72265625,
641
+ "eval_rewards/chosen": -1.7375926971435547,
642
+ "eval_rewards/margins": 0.7574552297592163,
643
+ "eval_rewards/rejected": -2.4950480461120605,
644
+ "eval_runtime": 53.4994,
645
+ "eval_samples_per_second": 37.384,
646
+ "eval_steps_per_second": 0.598,
647
+ "step": 400
648
+ },
649
+ {
650
+ "epoch": 0.86,
651
+ "learning_rate": 3.022313472693447e-08,
652
+ "logits/chosen": -2.2747280597686768,
653
+ "logits/rejected": -2.272991180419922,
654
+ "logps/chosen": -432.8849182128906,
655
+ "logps/rejected": -459.7318420410156,
656
+ "loss": 0.0597,
657
+ "rewards/accuracies": 0.625,
658
+ "rewards/chosen": -1.7819551229476929,
659
+ "rewards/margins": 0.49070867896080017,
660
+ "rewards/rejected": -2.2726640701293945,
661
+ "step": 410
662
+ },
663
+ {
664
+ "epoch": 0.88,
665
+ "learning_rate": 2.2111614344599684e-08,
666
+ "logits/chosen": -2.3252058029174805,
667
+ "logits/rejected": -2.283520221710205,
668
+ "logps/chosen": -446.5237731933594,
669
+ "logps/rejected": -498.34820556640625,
670
+ "loss": 0.0619,
671
+ "rewards/accuracies": 0.762499988079071,
672
+ "rewards/chosen": -1.688251256942749,
673
+ "rewards/margins": 0.7986016869544983,
674
+ "rewards/rejected": -2.4868531227111816,
675
+ "step": 420
676
+ },
677
+ {
678
+ "epoch": 0.9,
679
+ "learning_rate": 1.521597710086439e-08,
680
+ "logits/chosen": -2.3305728435516357,
681
+ "logits/rejected": -2.301042079925537,
682
+ "logps/chosen": -440.44195556640625,
683
+ "logps/rejected": -473.6935119628906,
684
+ "loss": 0.0677,
685
+ "rewards/accuracies": 0.6937500238418579,
686
+ "rewards/chosen": -1.6548879146575928,
687
+ "rewards/margins": 0.6255221962928772,
688
+ "rewards/rejected": -2.280410051345825,
689
+ "step": 430
690
+ },
691
+ {
692
+ "epoch": 0.92,
693
+ "learning_rate": 9.57301420397924e-09,
694
+ "logits/chosen": -2.2581725120544434,
695
+ "logits/rejected": -2.2572672367095947,
696
+ "logps/chosen": -451.09649658203125,
697
+ "logps/rejected": -473.00653076171875,
698
+ "loss": 0.0665,
699
+ "rewards/accuracies": 0.737500011920929,
700
+ "rewards/chosen": -1.7283083200454712,
701
+ "rewards/margins": 0.5989931225776672,
702
+ "rewards/rejected": -2.327301502227783,
703
+ "step": 440
704
+ },
705
+ {
706
+ "epoch": 0.94,
707
+ "learning_rate": 5.212833302556258e-09,
708
+ "logits/chosen": -2.2735867500305176,
709
+ "logits/rejected": -2.251678943634033,
710
+ "logps/chosen": -448.3414001464844,
711
+ "logps/rejected": -513.7218627929688,
712
+ "loss": 0.0621,
713
+ "rewards/accuracies": 0.6937500238418579,
714
+ "rewards/chosen": -1.7172386646270752,
715
+ "rewards/margins": 0.7713014483451843,
716
+ "rewards/rejected": -2.488539934158325,
717
+ "step": 450
718
+ },
719
+ {
720
+ "epoch": 0.96,
721
+ "learning_rate": 2.158697848236607e-09,
722
+ "logits/chosen": -2.2454590797424316,
723
+ "logits/rejected": -2.240736484527588,
724
+ "logps/chosen": -434.22247314453125,
725
+ "logps/rejected": -476.61578369140625,
726
+ "loss": 0.058,
727
+ "rewards/accuracies": 0.7250000238418579,
728
+ "rewards/chosen": -1.6892051696777344,
729
+ "rewards/margins": 0.6074910163879395,
730
+ "rewards/rejected": -2.296696186065674,
731
+ "step": 460
732
+ },
733
+ {
734
+ "epoch": 0.98,
735
+ "learning_rate": 4.269029751107489e-10,
736
+ "logits/chosen": -2.2940471172332764,
737
+ "logits/rejected": -2.265167713165283,
738
+ "logps/chosen": -451.9654846191406,
739
+ "logps/rejected": -475.11749267578125,
740
+ "loss": 0.0638,
741
+ "rewards/accuracies": 0.7250000238418579,
742
+ "rewards/chosen": -1.6610279083251953,
743
+ "rewards/margins": 0.7329179048538208,
744
+ "rewards/rejected": -2.3939454555511475,
745
+ "step": 470
746
+ },
747
  {
748
  "epoch": 1.0,
749
+ "step": 478,
750
  "total_flos": 0.0,
751
+ "train_loss": 0.10414429289526521,
752
+ "train_runtime": 3960.4871,
753
+ "train_samples_per_second": 15.436,
754
+ "train_steps_per_second": 0.121
755
  }
756
  ],
757
  "logging_steps": 10,
758
+ "max_steps": 478,
759
  "num_train_epochs": 1,
760
  "save_steps": 1000,
761
  "total_flos": 0.0,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f615665324c294412ab67ec01c797104bb1d270ada44fe46a66070866fae83f0
3
  size 5944
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a717ee85a879e33cf1276478869a3e9136952e476ad19ba8a64f3707bccd0c9
3
  size 5944