Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
Inference Endpoints
text-generation-inference
Model card Files Community
wzhouad commited on
Commit
5c3ce8e
•
1 Parent(s): d977275

Model save

Browse files
Files changed (9) hide show
  1. README.md +12 -13
  2. all_results.json +5 -5
  3. model-00001-of-00004.safetensors +1 -1
  4. model-00002-of-00004.safetensors +1 -1
  5. model-00003-of-00004.safetensors +1 -1
  6. model-00004-of-00004.safetensors +1 -1
  7. train_results.json +5 -5
  8. trainer_state.json +438 -622
  9. training_args.bin +1 -1
README.md CHANGED
@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
- - Loss: 0.0515
19
- - Rewards/chosen: -0.5698
20
- - Rewards/rejected: -1.0751
21
- - Rewards/accuracies: 0.7305
22
- - Rewards/margins: 0.5053
23
- - Logps/rejected: -473.2190
24
- - Logps/chosen: -416.4288
25
- - Logits/rejected: 0.8846
26
- - Logits/chosen: 0.7434
27
 
28
  ## Model description
29
 
@@ -60,10 +60,9 @@ The following hyperparameters were used during training:
60
 
61
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
62
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
63
- | 0.0993 | 0.21 | 100 | 0.1018 | -0.0745 | -0.2470 | 0.6875 | 0.1725 | -390.4119 | -366.9025 | 0.5707 | 0.5130 |
64
- | 0.0553 | 0.42 | 200 | 0.0540 | -0.5063 | -0.9361 | 0.7305 | 0.4298 | -459.3217 | -410.0835 | 0.8994 | 0.7755 |
65
- | 0.0503 | 0.63 | 300 | 0.0531 | -0.5353 | -1.0266 | 0.7461 | 0.4913 | -468.3688 | -412.9813 | 0.8095 | 0.6691 |
66
- | 0.0537 | 0.84 | 400 | 0.0515 | -0.5698 | -1.0751 | 0.7305 | 0.5053 | -473.2190 | -416.4288 | 0.8846 | 0.7434 |
67
 
68
 
69
  ### Framework versions
 
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
+ - Loss: 0.1258
19
+ - Rewards/chosen: 0.0938
20
+ - Rewards/rejected: 0.0434
21
+ - Rewards/accuracies: 0.5430
22
+ - Rewards/margins: 0.0504
23
+ - Logps/rejected: -361.3666
24
+ - Logps/chosen: -350.0670
25
+ - Logits/rejected: 0.4179
26
+ - Logits/chosen: 0.3310
27
 
28
  ## Model description
29
 
 
60
 
61
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
62
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
63
+ | 0.3022 | 0.28 | 100 | 0.1302 | 0.1319 | 0.1130 | 0.5078 | 0.0188 | -354.4073 | -346.2643 | 0.1904 | 0.1541 |
64
+ | 0.2907 | 0.56 | 200 | 0.1269 | 0.1422 | 0.0988 | 0.5430 | 0.0433 | -355.8251 | -345.2325 | 0.2916 | 0.2217 |
65
+ | 0.2824 | 0.84 | 300 | 0.1258 | 0.0938 | 0.0434 | 0.5430 | 0.0504 | -361.3666 | -350.0670 | 0.4179 | 0.3310 |
 
66
 
67
 
68
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.06614070340397472,
4
- "train_runtime": 4423.243,
5
- "train_samples": 61134,
6
- "train_samples_per_second": 13.821,
7
- "train_steps_per_second": 0.108
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.2852043948146734,
4
+ "train_runtime": 3316.6245,
5
+ "train_samples": 45548,
6
+ "train_samples_per_second": 13.733,
7
+ "train_steps_per_second": 0.107
8
  }
model-00001-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:38ba70e65ee6495d6142328650a81a3d99c446c7c25f7e9f8cac187fba978d69
3
  size 4976698672
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef19e2f36426a0eee7a52f50a2bac9e4cee8407115ac1702cfdca56669e8268d
3
  size 4976698672
model-00002-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dcdbaa7331284bffa3d618292207ea0cbbd9a63a92e4c00bf8ed4cbe607c5ed1
3
  size 4999802720
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5925052d9f2ff71b166e47505c840ed84a8d9d50c1f16615628fff43dd625236
3
  size 4999802720
model-00003-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4b5d969644264ab3fae956f1eb2d4bbdaf36aee0e6a96ece71c7e40e540b1640
3
  size 4915916176
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1e0765ba8a037cb4d3cd5b0fff7525ce4f1bb769f64d1ea7b9f3c8c5b9995d9
3
  size 4915916176
model-00004-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f13c9d26d250a584eded5cc5f110f766404c41b7baee91a2cc86e8a147bda71c
3
  size 1168138808
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5fbb19728d6f4bd28144d76c44b5ea58436d075fa47bd06b2f4ea7a03636c64
3
  size 1168138808
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.06614070340397472,
4
- "train_runtime": 4423.243,
5
- "train_samples": 61134,
6
- "train_samples_per_second": 13.821,
7
- "train_steps_per_second": 0.108
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.2852043948146734,
4
+ "train_runtime": 3316.6245,
5
+ "train_samples": 45548,
6
+ "train_samples_per_second": 13.733,
7
+ "train_steps_per_second": 0.107
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.9984301412872841,
5
  "eval_steps": 100,
6
- "global_step": 477,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 1.0416666666666666e-08,
14
- "logits/chosen": 0.07429073005914688,
15
- "logits/rejected": 0.11598015576601028,
16
- "logps/chosen": -422.86004638671875,
17
- "logps/rejected": -414.8901672363281,
18
- "loss": 0.1089,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -23,739 +23,555 @@
23
  "step": 1
24
  },
25
  {
26
- "epoch": 0.02,
27
- "learning_rate": 1.0416666666666667e-07,
28
- "logits/chosen": 0.30265143513679504,
29
- "logits/rejected": 0.37510383129119873,
30
- "logps/chosen": -376.6183166503906,
31
- "logps/rejected": -346.707275390625,
32
- "loss": 0.1038,
33
- "rewards/accuracies": 0.3958333432674408,
34
- "rewards/chosen": -0.0007193544879555702,
35
- "rewards/margins": -0.00018897267000284046,
36
- "rewards/rejected": -0.0005303816869854927,
37
  "step": 10
38
  },
39
- {
40
- "epoch": 0.04,
41
- "learning_rate": 2.0833333333333333e-07,
42
- "logits/chosen": 0.19012752175331116,
43
- "logits/rejected": 0.29147377610206604,
44
- "logps/chosen": -427.39190673828125,
45
- "logps/rejected": -371.12261962890625,
46
- "loss": 0.107,
47
- "rewards/accuracies": 0.5,
48
- "rewards/chosen": 0.0001767327485140413,
49
- "rewards/margins": 0.0014846216654404998,
50
- "rewards/rejected": -0.0013078890042379498,
51
- "step": 20
52
- },
53
  {
54
  "epoch": 0.06,
55
- "learning_rate": 3.1249999999999997e-07,
56
- "logits/chosen": 0.20171578228473663,
57
- "logits/rejected": 0.31732800602912903,
58
- "logps/chosen": -399.6023254394531,
59
- "logps/rejected": -374.3086853027344,
60
- "loss": 0.1094,
61
- "rewards/accuracies": 0.581250011920929,
62
- "rewards/chosen": -0.0014916183426976204,
63
- "rewards/margins": 0.001707876450382173,
64
- "rewards/rejected": -0.0031994946766644716,
65
- "step": 30
66
  },
67
  {
68
  "epoch": 0.08,
69
  "learning_rate": 4.1666666666666667e-07,
70
- "logits/chosen": 0.24505770206451416,
71
- "logits/rejected": 0.3264174163341522,
72
- "logps/chosen": -363.1432189941406,
73
- "logps/rejected": -371.2718200683594,
74
- "loss": 0.1094,
75
  "rewards/accuracies": 0.6499999761581421,
76
- "rewards/chosen": -0.0007695726817473769,
77
- "rewards/margins": 0.006053343415260315,
78
- "rewards/rejected": -0.006822916213423014,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  "step": 40
80
  },
81
  {
82
- "epoch": 0.1,
83
- "learning_rate": 4.999731868769026e-07,
84
- "logits/chosen": 0.26892322301864624,
85
- "logits/rejected": 0.34712380170822144,
86
- "logps/chosen": -378.1469421386719,
87
- "logps/rejected": -348.55352783203125,
88
- "loss": 0.105,
89
- "rewards/accuracies": 0.668749988079071,
90
- "rewards/chosen": -0.0007855228031985462,
91
- "rewards/margins": 0.022909890860319138,
92
- "rewards/rejected": -0.023695414885878563,
93
  "step": 50
94
  },
95
  {
96
- "epoch": 0.13,
97
- "learning_rate": 4.990353313429303e-07,
98
- "logits/chosen": 0.3134162127971649,
99
- "logits/rejected": 0.36972466111183167,
100
- "logps/chosen": -317.57965087890625,
101
- "logps/rejected": -317.5392761230469,
102
- "loss": 0.1049,
103
- "rewards/accuracies": 0.65625,
104
- "rewards/chosen": -0.008349133655428886,
105
- "rewards/margins": 0.035110391676425934,
106
- "rewards/rejected": -0.04345952719449997,
107
  "step": 60
108
  },
109
  {
110
- "epoch": 0.15,
111
- "learning_rate": 4.967625656594781e-07,
112
- "logits/chosen": 0.23508644104003906,
113
- "logits/rejected": 0.30452126264572144,
114
- "logps/chosen": -359.3031311035156,
115
- "logps/rejected": -345.0860900878906,
116
- "loss": 0.1082,
117
- "rewards/accuracies": 0.625,
118
- "rewards/chosen": -0.01529479306191206,
119
- "rewards/margins": 0.044887155294418335,
120
- "rewards/rejected": -0.06018194556236267,
121
  "step": 70
122
  },
123
  {
124
- "epoch": 0.17,
125
- "learning_rate": 4.93167072587771e-07,
126
- "logits/chosen": 0.33254843950271606,
127
- "logits/rejected": 0.3818221688270569,
128
- "logps/chosen": -367.01129150390625,
129
- "logps/rejected": -355.1100158691406,
130
- "loss": 0.1028,
131
- "rewards/accuracies": 0.762499988079071,
132
- "rewards/chosen": -0.01488255150616169,
133
- "rewards/margins": 0.08553680032491684,
134
- "rewards/rejected": -0.10041935741901398,
135
  "step": 80
136
  },
137
  {
138
- "epoch": 0.19,
139
- "learning_rate": 4.882681251368548e-07,
140
- "logits/chosen": 0.3798341751098633,
141
- "logits/rejected": 0.4554978311061859,
142
- "logps/chosen": -369.4306640625,
143
- "logps/rejected": -350.7769470214844,
144
- "loss": 0.1031,
145
- "rewards/accuracies": 0.668749988079071,
146
- "rewards/chosen": -0.04546991363167763,
147
- "rewards/margins": 0.09270831197500229,
148
- "rewards/rejected": -0.1381782442331314,
149
  "step": 90
150
  },
151
  {
152
- "epoch": 0.21,
153
- "learning_rate": 4.820919832540181e-07,
154
- "logits/chosen": 0.4522795081138611,
155
- "logits/rejected": 0.5922520756721497,
156
- "logps/chosen": -379.4673767089844,
157
- "logps/rejected": -347.5188903808594,
158
- "loss": 0.0993,
159
- "rewards/accuracies": 0.6625000238418579,
160
- "rewards/chosen": -0.062339335680007935,
161
- "rewards/margins": 0.1438979208469391,
162
- "rewards/rejected": -0.20623722672462463,
163
  "step": 100
164
  },
165
  {
166
- "epoch": 0.21,
167
- "eval_logits/chosen": 0.5129754543304443,
168
- "eval_logits/rejected": 0.5706519484519958,
169
- "eval_logps/chosen": -366.9024963378906,
170
- "eval_logps/rejected": -390.411865234375,
171
- "eval_loss": 0.10182792693376541,
172
- "eval_rewards/accuracies": 0.6875,
173
- "eval_rewards/chosen": -0.07451306283473969,
174
- "eval_rewards/margins": 0.17251166701316833,
175
- "eval_rewards/rejected": -0.2470247447490692,
176
- "eval_runtime": 75.2158,
177
- "eval_samples_per_second": 26.59,
178
- "eval_steps_per_second": 0.425,
179
  "step": 100
180
  },
181
  {
182
- "epoch": 0.23,
183
- "learning_rate": 4.7467175306295647e-07,
184
- "logits/chosen": 0.47333478927612305,
185
- "logits/rejected": 0.5572434663772583,
186
- "logps/chosen": -408.17938232421875,
187
- "logps/rejected": -431.2904357910156,
188
- "loss": 0.0955,
189
- "rewards/accuracies": 0.731249988079071,
190
- "rewards/chosen": -0.12534171342849731,
191
- "rewards/margins": 0.16846705973148346,
192
- "rewards/rejected": -0.2938087582588196,
193
  "step": 110
194
  },
195
  {
196
- "epoch": 0.25,
197
- "learning_rate": 4.6604720940421207e-07,
198
- "logits/chosen": 0.49355560541152954,
199
- "logits/rejected": 0.625954270362854,
200
- "logps/chosen": -393.81842041015625,
201
- "logps/rejected": -401.51202392578125,
202
- "loss": 0.0822,
203
- "rewards/accuracies": 0.7124999761581421,
204
- "rewards/chosen": -0.14783725142478943,
205
- "rewards/margins": 0.23524054884910583,
206
- "rewards/rejected": -0.38307780027389526,
207
  "step": 120
208
  },
209
  {
210
- "epoch": 0.27,
211
- "learning_rate": 4.5626458262912735e-07,
212
- "logits/chosen": 0.5241672396659851,
213
- "logits/rejected": 0.8548004031181335,
214
- "logps/chosen": -435.3441467285156,
215
- "logps/rejected": -357.9606018066406,
216
- "loss": 0.0797,
217
- "rewards/accuracies": 0.606249988079071,
218
- "rewards/chosen": -0.31069421768188477,
219
- "rewards/margins": 0.1076047271490097,
220
- "rewards/rejected": -0.41829895973205566,
221
  "step": 130
222
  },
223
  {
224
- "epoch": 0.29,
225
- "learning_rate": 4.453763107901675e-07,
226
- "logits/chosen": 0.5490397810935974,
227
- "logits/rejected": 0.7539095282554626,
228
- "logps/chosen": -413.84002685546875,
229
- "logps/rejected": -386.32379150390625,
230
- "loss": 0.0682,
231
- "rewards/accuracies": 0.625,
232
- "rewards/chosen": -0.3854388892650604,
233
- "rewards/margins": 0.1845283955335617,
234
- "rewards/rejected": -0.5699672698974609,
235
  "step": 140
236
  },
237
  {
238
- "epoch": 0.31,
239
- "learning_rate": 4.3344075855595097e-07,
240
- "logits/chosen": 0.6215056777000427,
241
- "logits/rejected": 0.7831140756607056,
242
- "logps/chosen": -419.662841796875,
243
- "logps/rejected": -391.9927673339844,
244
- "loss": 0.0704,
245
- "rewards/accuracies": 0.65625,
246
- "rewards/chosen": -0.35133233666419983,
247
- "rewards/margins": 0.202161505818367,
248
- "rewards/rejected": -0.553493857383728,
249
  "step": 150
250
  },
251
  {
252
- "epoch": 0.33,
253
- "learning_rate": 4.2052190435769554e-07,
254
- "logits/chosen": 0.7069424390792847,
255
- "logits/rejected": 0.8614177703857422,
256
- "logps/chosen": -384.68463134765625,
257
- "logps/rejected": -374.79144287109375,
258
- "loss": 0.0684,
259
- "rewards/accuracies": 0.668749988079071,
260
- "rewards/chosen": -0.2964687943458557,
261
- "rewards/margins": 0.2469368427991867,
262
- "rewards/rejected": -0.5434056520462036,
263
  "step": 160
264
  },
265
  {
266
- "epoch": 0.36,
267
- "learning_rate": 4.0668899744407567e-07,
268
- "logits/chosen": 0.6795724034309387,
269
- "logits/rejected": 0.788532555103302,
270
- "logps/chosen": -407.76092529296875,
271
- "logps/rejected": -429.6249084472656,
272
- "loss": 0.065,
273
- "rewards/accuracies": 0.6875,
274
- "rewards/chosen": -0.38756096363067627,
275
- "rewards/margins": 0.3297034204006195,
276
- "rewards/rejected": -0.7172644734382629,
277
  "step": 170
278
  },
279
  {
280
- "epoch": 0.38,
281
- "learning_rate": 3.920161866827889e-07,
282
- "logits/chosen": 0.6546531319618225,
283
- "logits/rejected": 0.8610761761665344,
284
- "logps/chosen": -399.92608642578125,
285
- "logps/rejected": -425.089599609375,
286
- "loss": 0.059,
287
- "rewards/accuracies": 0.699999988079071,
288
- "rewards/chosen": -0.45196419954299927,
289
- "rewards/margins": 0.27189016342163086,
290
- "rewards/rejected": -0.7238543033599854,
291
  "step": 180
292
  },
293
  {
294
- "epoch": 0.4,
295
- "learning_rate": 3.765821230985757e-07,
296
- "logits/chosen": 0.694263756275177,
297
- "logits/rejected": 0.9027630090713501,
298
- "logps/chosen": -411.06268310546875,
299
- "logps/rejected": -397.81097412109375,
300
- "loss": 0.0616,
301
- "rewards/accuracies": 0.625,
302
- "rewards/chosen": -0.46890345215797424,
303
- "rewards/margins": 0.21999216079711914,
304
- "rewards/rejected": -0.688895583152771,
305
  "step": 190
306
  },
307
  {
308
- "epoch": 0.42,
309
- "learning_rate": 3.604695382782159e-07,
310
- "logits/chosen": 0.6648334264755249,
311
- "logits/rejected": 0.9266846776008606,
312
- "logps/chosen": -450.61871337890625,
313
- "logps/rejected": -451.17596435546875,
314
- "loss": 0.0553,
315
- "rewards/accuracies": 0.71875,
316
- "rewards/chosen": -0.5313616394996643,
317
- "rewards/margins": 0.4010355472564697,
318
- "rewards/rejected": -0.9323970675468445,
319
  "step": 200
320
  },
321
  {
322
- "epoch": 0.42,
323
- "eval_logits/chosen": 0.7755414247512817,
324
- "eval_logits/rejected": 0.8994145393371582,
325
- "eval_logps/chosen": -410.08349609375,
326
- "eval_logps/rejected": -459.32171630859375,
327
- "eval_loss": 0.05395492538809776,
328
- "eval_rewards/accuracies": 0.73046875,
329
- "eval_rewards/chosen": -0.5063228607177734,
330
- "eval_rewards/margins": 0.42980003356933594,
331
- "eval_rewards/rejected": -0.9361228942871094,
332
- "eval_runtime": 75.0793,
333
- "eval_samples_per_second": 26.638,
334
- "eval_steps_per_second": 0.426,
335
  "step": 200
336
  },
337
  {
338
- "epoch": 0.44,
339
- "learning_rate": 3.4376480090239047e-07,
340
- "logits/chosen": 0.740880012512207,
341
- "logits/rejected": 0.9735954999923706,
342
- "logps/chosen": -407.00970458984375,
343
- "logps/rejected": -405.5915832519531,
344
- "loss": 0.0517,
345
- "rewards/accuracies": 0.6187499761581421,
346
- "rewards/chosen": -0.6327391266822815,
347
- "rewards/margins": 0.24419736862182617,
348
- "rewards/rejected": -0.8769365549087524,
349
  "step": 210
350
  },
351
  {
352
- "epoch": 0.46,
353
- "learning_rate": 3.265574537815398e-07,
354
- "logits/chosen": 0.5885998606681824,
355
- "logits/rejected": 0.8313006162643433,
356
- "logps/chosen": -449.39251708984375,
357
- "logps/rejected": -427.76641845703125,
358
- "loss": 0.0537,
359
- "rewards/accuracies": 0.7124999761581421,
360
- "rewards/chosen": -0.5039619207382202,
361
- "rewards/margins": 0.33774858713150024,
362
- "rewards/rejected": -0.8417104482650757,
363
  "step": 220
364
  },
365
  {
366
- "epoch": 0.48,
367
- "learning_rate": 3.0893973387735683e-07,
368
- "logits/chosen": 0.5277591347694397,
369
- "logits/rejected": 0.7385606169700623,
370
- "logps/chosen": -428.235595703125,
371
- "logps/rejected": -417.846923828125,
372
- "loss": 0.054,
373
- "rewards/accuracies": 0.643750011920929,
374
- "rewards/chosen": -0.5545467138290405,
375
- "rewards/margins": 0.3285568654537201,
376
- "rewards/rejected": -0.8831035494804382,
377
  "step": 230
378
  },
379
  {
380
- "epoch": 0.5,
381
- "learning_rate": 2.910060778827554e-07,
382
- "logits/chosen": 0.5636624693870544,
383
- "logits/rejected": 0.7239071130752563,
384
- "logps/chosen": -410.8172302246094,
385
- "logps/rejected": -453.8780822753906,
386
- "loss": 0.0517,
387
- "rewards/accuracies": 0.625,
388
- "rewards/chosen": -0.6257103085517883,
389
- "rewards/margins": 0.3102150559425354,
390
- "rewards/rejected": -0.9359253644943237,
391
  "step": 240
392
  },
393
  {
394
- "epoch": 0.52,
395
- "learning_rate": 2.7285261601056697e-07,
396
- "logits/chosen": 0.6507786512374878,
397
- "logits/rejected": 0.7937926054000854,
398
- "logps/chosen": -399.25531005859375,
399
- "logps/rejected": -428.21173095703125,
400
- "loss": 0.0482,
401
- "rewards/accuracies": 0.71875,
402
- "rewards/chosen": -0.5876737236976624,
403
- "rewards/margins": 0.42669692635536194,
404
- "rewards/rejected": -1.0143706798553467,
405
  "step": 250
406
  },
407
  {
408
- "epoch": 0.54,
409
- "learning_rate": 2.5457665670441937e-07,
410
- "logits/chosen": 0.7192268967628479,
411
- "logits/rejected": 0.9365060925483704,
412
- "logps/chosen": -442.73931884765625,
413
- "logps/rejected": -429.482421875,
414
- "loss": 0.0515,
415
- "rewards/accuracies": 0.606249988079071,
416
- "rewards/chosen": -0.7237201929092407,
417
- "rewards/margins": 0.23805689811706543,
418
- "rewards/rejected": -0.9617769122123718,
419
  "step": 260
420
  },
421
  {
422
- "epoch": 0.57,
423
- "learning_rate": 2.3627616503391812e-07,
424
- "logits/chosen": 0.8510295748710632,
425
- "logits/rejected": 1.0050238370895386,
426
- "logps/chosen": -445.4356384277344,
427
- "logps/rejected": -501.38287353515625,
428
- "loss": 0.0505,
429
- "rewards/accuracies": 0.6499999761581421,
430
- "rewards/chosen": -0.6589037179946899,
431
- "rewards/margins": 0.45235753059387207,
432
- "rewards/rejected": -1.111261248588562,
433
  "step": 270
434
  },
435
  {
436
- "epoch": 0.59,
437
- "learning_rate": 2.1804923757009882e-07,
438
- "logits/chosen": 0.7827587723731995,
439
- "logits/rejected": 0.8751026391983032,
440
- "logps/chosen": -394.4527282714844,
441
- "logps/rejected": -438.7657165527344,
442
- "loss": 0.0514,
443
- "rewards/accuracies": 0.6937500238418579,
444
- "rewards/chosen": -0.6062650084495544,
445
- "rewards/margins": 0.3552294373512268,
446
- "rewards/rejected": -0.9614945650100708,
447
  "step": 280
448
  },
449
  {
450
- "epoch": 0.61,
451
- "learning_rate": 1.9999357655598891e-07,
452
- "logits/chosen": 0.6437442898750305,
453
- "logits/rejected": 0.9078620672225952,
454
- "logps/chosen": -437.314697265625,
455
- "logps/rejected": -448.2724609375,
456
- "loss": 0.0493,
457
- "rewards/accuracies": 0.675000011920929,
458
- "rewards/chosen": -0.6567361950874329,
459
- "rewards/margins": 0.38017868995666504,
460
- "rewards/rejected": -1.0369149446487427,
461
  "step": 290
462
  },
463
  {
464
- "epoch": 0.63,
465
- "learning_rate": 1.8220596619089573e-07,
466
- "logits/chosen": 0.6364871263504028,
467
- "logits/rejected": 0.8500644564628601,
468
- "logps/chosen": -384.7347412109375,
469
- "logps/rejected": -401.3032531738281,
470
- "loss": 0.0503,
471
- "rewards/accuracies": 0.6625000238418579,
472
- "rewards/chosen": -0.5907383561134338,
473
- "rewards/margins": 0.31186026334762573,
474
- "rewards/rejected": -0.9025986790657043,
475
  "step": 300
476
  },
477
  {
478
- "epoch": 0.63,
479
- "eval_logits/chosen": 0.6691160202026367,
480
- "eval_logits/rejected": 0.8094965219497681,
481
- "eval_logps/chosen": -412.98126220703125,
482
- "eval_logps/rejected": -468.3688049316406,
483
- "eval_loss": 0.05308157950639725,
484
- "eval_rewards/accuracies": 0.74609375,
485
- "eval_rewards/chosen": -0.5353004932403564,
486
- "eval_rewards/margins": 0.4912930727005005,
487
- "eval_rewards/rejected": -1.0265936851501465,
488
- "eval_runtime": 75.2339,
489
- "eval_samples_per_second": 26.584,
490
- "eval_steps_per_second": 0.425,
491
  "step": 300
492
  },
493
  {
494
- "epoch": 0.65,
495
- "learning_rate": 1.647817538357072e-07,
496
- "logits/chosen": 0.5804156064987183,
497
- "logits/rejected": 0.8023425936698914,
498
- "logps/chosen": -431.28790283203125,
499
- "logps/rejected": -459.8777770996094,
500
- "loss": 0.0547,
501
- "rewards/accuracies": 0.699999988079071,
502
- "rewards/chosen": -0.5415506958961487,
503
- "rewards/margins": 0.43213915824890137,
504
- "rewards/rejected": -0.97368985414505,
505
  "step": 310
506
  },
507
  {
508
- "epoch": 0.67,
509
- "learning_rate": 1.478143389201113e-07,
510
- "logits/chosen": 0.5641556978225708,
511
- "logits/rejected": 0.7904598712921143,
512
- "logps/chosen": -419.69329833984375,
513
- "logps/rejected": -420.63629150390625,
514
- "loss": 0.0512,
515
- "rewards/accuracies": 0.6812499761581421,
516
- "rewards/chosen": -0.5348724126815796,
517
- "rewards/margins": 0.3764938712120056,
518
- "rewards/rejected": -0.9113662838935852,
519
  "step": 320
520
  },
521
  {
522
- "epoch": 0.69,
523
- "learning_rate": 1.3139467229135998e-07,
524
- "logits/chosen": 0.7080197334289551,
525
- "logits/rejected": 0.8454129099845886,
526
- "logps/chosen": -395.1232604980469,
527
- "logps/rejected": -457.27978515625,
528
- "loss": 0.0486,
529
- "rewards/accuracies": 0.6625000238418579,
530
- "rewards/chosen": -0.5784841775894165,
531
- "rewards/margins": 0.39547181129455566,
532
- "rewards/rejected": -0.9739561080932617,
533
  "step": 330
534
  },
535
- {
536
- "epoch": 0.71,
537
- "learning_rate": 1.1561076868822755e-07,
538
- "logits/chosen": 0.7032380700111389,
539
- "logits/rejected": 0.8845696449279785,
540
- "logps/chosen": -421.2705078125,
541
- "logps/rejected": -432.5623474121094,
542
- "loss": 0.0532,
543
- "rewards/accuracies": 0.699999988079071,
544
- "rewards/chosen": -0.6043194532394409,
545
- "rewards/margins": 0.37815985083580017,
546
- "rewards/rejected": -0.9824793934822083,
547
- "step": 340
548
- },
549
- {
550
- "epoch": 0.73,
551
- "learning_rate": 1.0054723495346482e-07,
552
- "logits/chosen": 0.7004996538162231,
553
- "logits/rejected": 0.8182412981987,
554
- "logps/chosen": -392.1104431152344,
555
- "logps/rejected": -416.94415283203125,
556
- "loss": 0.0467,
557
- "rewards/accuracies": 0.637499988079071,
558
- "rewards/chosen": -0.6616944074630737,
559
- "rewards/margins": 0.3355887532234192,
560
- "rewards/rejected": -0.9972831606864929,
561
- "step": 350
562
- },
563
- {
564
- "epoch": 0.75,
565
- "learning_rate": 8.628481651367875e-08,
566
- "logits/chosen": 0.6653555631637573,
567
- "logits/rejected": 0.8546744585037231,
568
- "logps/chosen": -370.65008544921875,
569
- "logps/rejected": -413.5162048339844,
570
- "loss": 0.0484,
571
- "rewards/accuracies": 0.6937500238418579,
572
- "rewards/chosen": -0.529220461845398,
573
- "rewards/margins": 0.42656245827674866,
574
- "rewards/rejected": -0.9557830095291138,
575
- "step": 360
576
- },
577
- {
578
- "epoch": 0.77,
579
- "learning_rate": 7.289996455765748e-08,
580
- "logits/chosen": 0.6534770131111145,
581
- "logits/rejected": 0.7723512053489685,
582
- "logps/chosen": -436.49346923828125,
583
- "logps/rejected": -463.48577880859375,
584
- "loss": 0.0486,
585
- "rewards/accuracies": 0.71875,
586
- "rewards/chosen": -0.5268007516860962,
587
- "rewards/margins": 0.5095833539962769,
588
- "rewards/rejected": -1.0363839864730835,
589
- "step": 370
590
- },
591
- {
592
- "epoch": 0.8,
593
- "learning_rate": 6.046442623320145e-08,
594
- "logits/chosen": 0.7045032978057861,
595
- "logits/rejected": 0.7946165800094604,
596
- "logps/chosen": -398.9028625488281,
597
- "logps/rejected": -436.78515625,
598
- "loss": 0.0499,
599
- "rewards/accuracies": 0.7250000238418579,
600
- "rewards/chosen": -0.5992251634597778,
601
- "rewards/margins": 0.39836931228637695,
602
- "rewards/rejected": -0.9975944757461548,
603
- "step": 380
604
- },
605
- {
606
- "epoch": 0.82,
607
- "learning_rate": 4.904486005914027e-08,
608
- "logits/chosen": 0.5793179869651794,
609
- "logits/rejected": 0.7902762293815613,
610
- "logps/chosen": -427.8744201660156,
611
- "logps/rejected": -456.980224609375,
612
- "loss": 0.0472,
613
- "rewards/accuracies": 0.7250000238418579,
614
- "rewards/chosen": -0.541319727897644,
615
- "rewards/margins": 0.55488520860672,
616
- "rewards/rejected": -1.0962049961090088,
617
- "step": 390
618
- },
619
- {
620
- "epoch": 0.84,
621
- "learning_rate": 3.8702478614051345e-08,
622
- "logits/chosen": 0.6567720174789429,
623
- "logits/rejected": 0.8759796023368835,
624
- "logps/chosen": -413.21124267578125,
625
- "logps/rejected": -466.8826599121094,
626
- "loss": 0.0537,
627
- "rewards/accuracies": 0.668749988079071,
628
- "rewards/chosen": -0.5818170309066772,
629
- "rewards/margins": 0.48379120230674744,
630
- "rewards/rejected": -1.065608263015747,
631
- "step": 400
632
- },
633
- {
634
- "epoch": 0.84,
635
- "eval_logits/chosen": 0.7434472441673279,
636
- "eval_logits/rejected": 0.8845732808113098,
637
- "eval_logps/chosen": -416.42877197265625,
638
- "eval_logps/rejected": -473.2190246582031,
639
- "eval_loss": 0.05153993144631386,
640
- "eval_rewards/accuracies": 0.73046875,
641
- "eval_rewards/chosen": -0.5697759985923767,
642
- "eval_rewards/margins": 0.5053197145462036,
643
- "eval_rewards/rejected": -1.075095772743225,
644
- "eval_runtime": 76.6604,
645
- "eval_samples_per_second": 26.089,
646
- "eval_steps_per_second": 0.417,
647
- "step": 400
648
- },
649
- {
650
- "epoch": 0.86,
651
- "learning_rate": 2.9492720416985e-08,
652
- "logits/chosen": 0.6941822171211243,
653
- "logits/rejected": 0.8047625422477722,
654
- "logps/chosen": -401.1028137207031,
655
- "logps/rejected": -427.03302001953125,
656
- "loss": 0.0478,
657
- "rewards/accuracies": 0.6625000238418579,
658
- "rewards/chosen": -0.576189398765564,
659
- "rewards/margins": 0.39370283484458923,
660
- "rewards/rejected": -0.969892144203186,
661
- "step": 410
662
- },
663
- {
664
- "epoch": 0.88,
665
- "learning_rate": 2.1464952759020856e-08,
666
- "logits/chosen": 0.7099225521087646,
667
- "logits/rejected": 0.8760135769844055,
668
- "logps/chosen": -388.6665954589844,
669
- "logps/rejected": -421.422607421875,
670
- "loss": 0.0477,
671
- "rewards/accuracies": 0.6625000238418579,
672
- "rewards/chosen": -0.5949358344078064,
673
- "rewards/margins": 0.3995293378829956,
674
- "rewards/rejected": -0.9944650530815125,
675
- "step": 420
676
- },
677
- {
678
- "epoch": 0.9,
679
- "learning_rate": 1.4662207078575684e-08,
680
- "logits/chosen": 0.6152527332305908,
681
- "logits/rejected": 0.7688241600990295,
682
- "logps/chosen": -453.6767578125,
683
- "logps/rejected": -463.40863037109375,
684
- "loss": 0.0527,
685
- "rewards/accuracies": 0.699999988079071,
686
- "rewards/chosen": -0.613703727722168,
687
- "rewards/margins": 0.43121272325515747,
688
- "rewards/rejected": -1.0449163913726807,
689
- "step": 430
690
- },
691
- {
692
- "epoch": 0.92,
693
- "learning_rate": 9.12094829893642e-09,
694
- "logits/chosen": 0.7038220167160034,
695
- "logits/rejected": 0.8598071932792664,
696
- "logps/chosen": -436.53082275390625,
697
- "logps/rejected": -434.2837829589844,
698
- "loss": 0.0484,
699
- "rewards/accuracies": 0.6937500238418579,
700
- "rewards/chosen": -0.6220607161521912,
701
- "rewards/margins": 0.44049739837646484,
702
- "rewards/rejected": -1.0625581741333008,
703
- "step": 440
704
- },
705
- {
706
- "epoch": 0.94,
707
- "learning_rate": 4.8708793644441086e-09,
708
- "logits/chosen": 0.6375268697738647,
709
- "logits/rejected": 0.8610206842422485,
710
- "logps/chosen": -462.42449951171875,
711
- "logps/rejected": -464.8294372558594,
712
- "loss": 0.0474,
713
- "rewards/accuracies": 0.7124999761581421,
714
- "rewards/chosen": -0.685294508934021,
715
- "rewards/margins": 0.4343951344490051,
716
- "rewards/rejected": -1.119689702987671,
717
- "step": 450
718
- },
719
  {
720
  "epoch": 0.96,
721
- "learning_rate": 1.9347820230782295e-09,
722
- "logits/chosen": 0.6665740013122559,
723
- "logits/rejected": 0.8705039024353027,
724
- "logps/chosen": -409.6141662597656,
725
- "logps/rejected": -419.2763671875,
726
- "loss": 0.0457,
727
- "rewards/accuracies": 0.6812499761581421,
728
- "rewards/chosen": -0.5975238084793091,
729
- "rewards/margins": 0.3504991829395294,
730
- "rewards/rejected": -0.9480230212211609,
731
- "step": 460
732
  },
733
  {
734
  "epoch": 0.98,
735
- "learning_rate": 3.2839470889836627e-10,
736
- "logits/chosen": 0.5994333028793335,
737
- "logits/rejected": 0.8693232536315918,
738
- "logps/chosen": -446.9803771972656,
739
- "logps/rejected": -442.191650390625,
740
- "loss": 0.0495,
741
- "rewards/accuracies": 0.706250011920929,
742
- "rewards/chosen": -0.5732464790344238,
743
- "rewards/margins": 0.4613564610481262,
744
- "rewards/rejected": -1.0346029996871948,
745
- "step": 470
746
  },
747
  {
748
  "epoch": 1.0,
749
- "step": 477,
750
  "total_flos": 0.0,
751
- "train_loss": 0.06614070340397472,
752
- "train_runtime": 4423.243,
753
- "train_samples_per_second": 13.821,
754
- "train_steps_per_second": 0.108
755
  }
756
  ],
757
  "logging_steps": 10,
758
- "max_steps": 477,
759
  "num_train_epochs": 1,
760
  "save_steps": 1000,
761
  "total_flos": 0.0,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
  "eval_steps": 100,
6
+ "global_step": 356,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.3888888888888887e-08,
14
+ "logits/chosen": -0.07916320115327835,
15
+ "logits/rejected": 0.0910365954041481,
16
+ "logps/chosen": -527.0689697265625,
17
+ "logps/rejected": -183.7891082763672,
18
+ "loss": 0.2596,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
23
  "step": 1
24
  },
25
  {
26
+ "epoch": 0.03,
27
+ "learning_rate": 1.3888888888888888e-07,
28
+ "logits/chosen": 0.0035775601863861084,
29
+ "logits/rejected": 0.04552823305130005,
30
+ "logps/chosen": -360.2843933105469,
31
+ "logps/rejected": -270.2789611816406,
32
+ "loss": 0.2699,
33
+ "rewards/accuracies": 0.4166666567325592,
34
+ "rewards/chosen": -0.0005851028254255652,
35
+ "rewards/margins": -0.0009591608541086316,
36
+ "rewards/rejected": 0.0003740580577868968,
37
  "step": 10
38
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  {
40
  "epoch": 0.06,
41
+ "learning_rate": 2.7777777777777776e-07,
42
+ "logits/chosen": 0.058228444308042526,
43
+ "logits/rejected": 0.11480607837438583,
44
+ "logps/chosen": -348.0200500488281,
45
+ "logps/rejected": -243.383056640625,
46
+ "loss": 0.2676,
47
+ "rewards/accuracies": 0.512499988079071,
48
+ "rewards/chosen": 0.0004963463870808482,
49
+ "rewards/margins": 0.0019038517493754625,
50
+ "rewards/rejected": -0.0014075052458792925,
51
+ "step": 20
52
  },
53
  {
54
  "epoch": 0.08,
55
  "learning_rate": 4.1666666666666667e-07,
56
+ "logits/chosen": 0.0972701907157898,
57
+ "logits/rejected": 0.13563457131385803,
58
+ "logps/chosen": -369.4275207519531,
59
+ "logps/rejected": -250.97119140625,
60
+ "loss": 0.2664,
61
  "rewards/accuracies": 0.6499999761581421,
62
+ "rewards/chosen": 0.00677915196865797,
63
+ "rewards/margins": 0.0182232316583395,
64
+ "rewards/rejected": -0.01144408155232668,
65
+ "step": 30
66
+ },
67
+ {
68
+ "epoch": 0.11,
69
+ "learning_rate": 4.998072590601808e-07,
70
+ "logits/chosen": 0.04347607493400574,
71
+ "logits/rejected": 0.10795494168996811,
72
+ "logps/chosen": -327.04644775390625,
73
+ "logps/rejected": -232.80764770507812,
74
+ "loss": 0.2733,
75
+ "rewards/accuracies": 0.550000011920929,
76
+ "rewards/chosen": 0.0005852997419424355,
77
+ "rewards/margins": 0.027946794405579567,
78
+ "rewards/rejected": -0.027361497282981873,
79
  "step": 40
80
  },
81
  {
82
+ "epoch": 0.14,
83
+ "learning_rate": 4.976423351108942e-07,
84
+ "logits/chosen": 0.04018152505159378,
85
+ "logits/rejected": 0.10748956352472305,
86
+ "logps/chosen": -327.4648742675781,
87
+ "logps/rejected": -238.63186645507812,
88
+ "loss": 0.2772,
89
+ "rewards/accuracies": 0.574999988079071,
90
+ "rewards/chosen": -0.02821285091340542,
91
+ "rewards/margins": 0.05053957551717758,
92
+ "rewards/rejected": -0.07875243574380875,
93
  "step": 50
94
  },
95
  {
96
+ "epoch": 0.17,
97
+ "learning_rate": 4.930924800994191e-07,
98
+ "logits/chosen": -0.03577030822634697,
99
+ "logits/rejected": 0.03312467411160469,
100
+ "logps/chosen": -385.00408935546875,
101
+ "logps/rejected": -272.11126708984375,
102
+ "loss": 0.286,
103
+ "rewards/accuracies": 0.612500011920929,
104
+ "rewards/chosen": -0.06713319569826126,
105
+ "rewards/margins": 0.11954391002655029,
106
+ "rewards/rejected": -0.18667711317539215,
107
  "step": 60
108
  },
109
  {
110
+ "epoch": 0.2,
111
+ "learning_rate": 4.862015116167195e-07,
112
+ "logits/chosen": 0.007775746285915375,
113
+ "logits/rejected": 0.06914432346820831,
114
+ "logps/chosen": -410.8778381347656,
115
+ "logps/rejected": -264.062255859375,
116
+ "loss": 0.2833,
117
+ "rewards/accuracies": 0.5874999761581421,
118
+ "rewards/chosen": -0.06701745092868805,
119
+ "rewards/margins": 0.1522327959537506,
120
+ "rewards/rejected": -0.21925024688243866,
121
  "step": 70
122
  },
123
  {
124
+ "epoch": 0.22,
125
+ "learning_rate": 4.770357934562704e-07,
126
+ "logits/chosen": -0.05758536607027054,
127
+ "logits/rejected": 0.001774595701135695,
128
+ "logps/chosen": -347.01800537109375,
129
+ "logps/rejected": -270.14691162109375,
130
+ "loss": 0.2877,
131
+ "rewards/accuracies": 0.637499988079071,
132
+ "rewards/chosen": -0.08113797008991241,
133
+ "rewards/margins": 0.11941523849964142,
134
+ "rewards/rejected": -0.20055320858955383,
135
  "step": 80
136
  },
137
  {
138
+ "epoch": 0.25,
139
+ "learning_rate": 4.6568359649444796e-07,
140
+ "logits/chosen": -0.029236888512969017,
141
+ "logits/rejected": 0.007240456994622946,
142
+ "logps/chosen": -387.5735778808594,
143
+ "logps/rejected": -295.7844543457031,
144
+ "loss": 0.2978,
145
+ "rewards/accuracies": 0.6187499761581421,
146
+ "rewards/chosen": -0.08084937185049057,
147
+ "rewards/margins": 0.1757977455854416,
148
+ "rewards/rejected": -0.25664713978767395,
149
  "step": 90
150
  },
151
  {
152
+ "epoch": 0.28,
153
+ "learning_rate": 4.5225424859373684e-07,
154
+ "logits/chosen": -0.04320114105939865,
155
+ "logits/rejected": 0.019383195787668228,
156
+ "logps/chosen": -370.8349304199219,
157
+ "logps/rejected": -264.00286865234375,
158
+ "loss": 0.3022,
159
+ "rewards/accuracies": 0.612500011920929,
160
+ "rewards/chosen": -0.008325784467160702,
161
+ "rewards/margins": 0.18024340271949768,
162
+ "rewards/rejected": -0.18856915831565857,
163
  "step": 100
164
  },
165
  {
166
+ "epoch": 0.28,
167
+ "eval_logits/chosen": 0.15412691235542297,
168
+ "eval_logits/rejected": 0.1903735250234604,
169
+ "eval_logps/chosen": -346.26434326171875,
170
+ "eval_logps/rejected": -354.40728759765625,
171
+ "eval_loss": 0.13015557825565338,
172
+ "eval_rewards/accuracies": 0.5078125,
173
+ "eval_rewards/chosen": 0.1318686306476593,
174
+ "eval_rewards/margins": 0.018847040832042694,
175
+ "eval_rewards/rejected": 0.11302158236503601,
176
+ "eval_runtime": 76.3276,
177
+ "eval_samples_per_second": 26.203,
178
+ "eval_steps_per_second": 0.419,
179
  "step": 100
180
  },
181
  {
182
+ "epoch": 0.31,
183
+ "learning_rate": 4.3687708171564917e-07,
184
+ "logits/chosen": -0.06751509010791779,
185
+ "logits/rejected": 0.007752983830869198,
186
+ "logps/chosen": -325.31787109375,
187
+ "logps/rejected": -287.33306884765625,
188
+ "loss": 0.2905,
189
+ "rewards/accuracies": 0.5874999761581421,
190
+ "rewards/chosen": -0.07388351857662201,
191
+ "rewards/margins": 0.07556183636188507,
192
+ "rewards/rejected": -0.14944536983966827,
193
  "step": 110
194
  },
195
  {
196
+ "epoch": 0.34,
197
+ "learning_rate": 4.1970018638323547e-07,
198
+ "logits/chosen": -0.0013766803313046694,
199
+ "logits/rejected": 0.042307257652282715,
200
+ "logps/chosen": -313.0906066894531,
201
+ "logps/rejected": -243.065673828125,
202
+ "loss": 0.2898,
203
+ "rewards/accuracies": 0.6312500238418579,
204
+ "rewards/chosen": 0.0047665126621723175,
205
+ "rewards/margins": 0.10345210880041122,
206
+ "rewards/rejected": -0.0986856073141098,
207
  "step": 120
208
  },
209
  {
210
+ "epoch": 0.37,
211
+ "learning_rate": 4.0088898548839285e-07,
212
+ "logits/chosen": -0.05491367727518082,
213
+ "logits/rejected": 0.013940325006842613,
214
+ "logps/chosen": -400.29022216796875,
215
+ "logps/rejected": -242.8967742919922,
216
+ "loss": 0.2966,
217
+ "rewards/accuracies": 0.6812499761581421,
218
+ "rewards/chosen": 0.13614816963672638,
219
+ "rewards/margins": 0.25932490825653076,
220
+ "rewards/rejected": -0.1231766939163208,
221
  "step": 130
222
  },
223
  {
224
+ "epoch": 0.39,
225
+ "learning_rate": 3.806246411789872e-07,
226
+ "logits/chosen": -0.06341247260570526,
227
+ "logits/rejected": -0.00459135789424181,
228
+ "logps/chosen": -322.44781494140625,
229
+ "logps/rejected": -244.84060668945312,
230
+ "loss": 0.2881,
231
+ "rewards/accuracies": 0.5562499761581421,
232
+ "rewards/chosen": 0.04705403372645378,
233
+ "rewards/margins": 0.11207260191440582,
234
+ "rewards/rejected": -0.06501854956150055,
235
  "step": 140
236
  },
237
  {
238
+ "epoch": 0.42,
239
+ "learning_rate": 3.5910231016833546e-07,
240
+ "logits/chosen": -0.070883609354496,
241
+ "logits/rejected": -0.01928647793829441,
242
+ "logps/chosen": -339.30511474609375,
243
+ "logps/rejected": -266.8258361816406,
244
+ "loss": 0.2982,
245
+ "rewards/accuracies": 0.53125,
246
+ "rewards/chosen": 0.04272865131497383,
247
+ "rewards/margins": 0.11490585654973984,
248
+ "rewards/rejected": -0.0721772164106369,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.45,
253
+ "learning_rate": 3.3652926426937325e-07,
254
+ "logits/chosen": -0.0023407593835145235,
255
+ "logits/rejected": 0.09304739534854889,
256
+ "logps/chosen": -361.07958984375,
257
+ "logps/rejected": -255.61373901367188,
258
+ "loss": 0.2943,
259
+ "rewards/accuracies": 0.643750011920929,
260
+ "rewards/chosen": 0.0948086529970169,
261
+ "rewards/margins": 0.16468903422355652,
262
+ "rewards/rejected": -0.06988037377595901,
263
  "step": 160
264
  },
265
  {
266
+ "epoch": 0.48,
267
+ "learning_rate": 3.1312289425378944e-07,
268
+ "logits/chosen": 0.053202610462903976,
269
+ "logits/rejected": 0.09878319501876831,
270
+ "logps/chosen": -324.02178955078125,
271
+ "logps/rejected": -254.9729461669922,
272
+ "loss": 0.2868,
273
+ "rewards/accuracies": 0.5562499761581421,
274
+ "rewards/chosen": 0.060007404536008835,
275
+ "rewards/margins": 0.15731558203697205,
276
+ "rewards/rejected": -0.09730817377567291,
277
  "step": 170
278
  },
279
  {
280
+ "epoch": 0.51,
281
+ "learning_rate": 2.8910861626005773e-07,
282
+ "logits/chosen": -0.010406618937849998,
283
+ "logits/rejected": 0.06127941608428955,
284
+ "logps/chosen": -340.56890869140625,
285
+ "logps/rejected": -286.2023010253906,
286
+ "loss": 0.2884,
287
+ "rewards/accuracies": 0.550000011920929,
288
+ "rewards/chosen": 0.016947364434599876,
289
+ "rewards/margins": 0.11571431159973145,
290
+ "rewards/rejected": -0.09876694530248642,
291
  "step": 180
292
  },
293
  {
294
+ "epoch": 0.53,
295
+ "learning_rate": 2.647177009127972e-07,
296
+ "logits/chosen": -0.0076253474690020084,
297
+ "logits/rejected": 0.09804458916187286,
298
+ "logps/chosen": -349.88885498046875,
299
+ "logps/rejected": -273.30108642578125,
300
+ "loss": 0.2811,
301
+ "rewards/accuracies": 0.606249988079071,
302
+ "rewards/chosen": 0.05855409428477287,
303
+ "rewards/margins": 0.16944967210292816,
304
+ "rewards/rejected": -0.11089557409286499,
305
  "step": 190
306
  },
307
  {
308
+ "epoch": 0.56,
309
+ "learning_rate": 2.401850460602329e-07,
310
+ "logits/chosen": -0.0508098229765892,
311
+ "logits/rejected": 0.060315102338790894,
312
+ "logps/chosen": -366.349365234375,
313
+ "logps/rejected": -242.5937042236328,
314
+ "loss": 0.2907,
315
+ "rewards/accuracies": 0.606249988079071,
316
+ "rewards/chosen": 0.08607779443264008,
317
+ "rewards/margins": 0.2081635296344757,
318
+ "rewards/rejected": -0.12208573520183563,
319
  "step": 200
320
  },
321
  {
322
+ "epoch": 0.56,
323
+ "eval_logits/chosen": 0.22172768414020538,
324
+ "eval_logits/rejected": 0.29156258702278137,
325
+ "eval_logps/chosen": -345.2325439453125,
326
+ "eval_logps/rejected": -355.8251037597656,
327
+ "eval_loss": 0.12694023549556732,
328
+ "eval_rewards/accuracies": 0.54296875,
329
+ "eval_rewards/chosen": 0.14218640327453613,
330
+ "eval_rewards/margins": 0.043343350291252136,
331
+ "eval_rewards/rejected": 0.0988430380821228,
332
+ "eval_runtime": 75.2765,
333
+ "eval_samples_per_second": 26.569,
334
+ "eval_steps_per_second": 0.425,
335
  "step": 200
336
  },
337
  {
338
+ "epoch": 0.59,
339
+ "learning_rate": 2.1574691457950803e-07,
340
+ "logits/chosen": -0.06948701292276382,
341
+ "logits/rejected": 0.025958305224776268,
342
+ "logps/chosen": -422.06103515625,
343
+ "logps/rejected": -274.491943359375,
344
+ "loss": 0.2874,
345
+ "rewards/accuracies": 0.606249988079071,
346
+ "rewards/chosen": 0.14031639695167542,
347
+ "rewards/margins": 0.2419450730085373,
348
+ "rewards/rejected": -0.10162866115570068,
349
  "step": 210
350
  },
351
  {
352
+ "epoch": 0.62,
353
+ "learning_rate": 1.9163865903602372e-07,
354
+ "logits/chosen": 0.019119512289762497,
355
+ "logits/rejected": 0.10009634494781494,
356
+ "logps/chosen": -378.73602294921875,
357
+ "logps/rejected": -259.0059814453125,
358
+ "loss": 0.2888,
359
+ "rewards/accuracies": 0.668749988079071,
360
+ "rewards/chosen": 0.09178078174591064,
361
+ "rewards/margins": 0.21833208203315735,
362
+ "rewards/rejected": -0.12655127048492432,
363
  "step": 220
364
  },
365
  {
366
+ "epoch": 0.65,
367
+ "learning_rate": 1.6809245510957666e-07,
368
+ "logits/chosen": 0.03617783635854721,
369
+ "logits/rejected": 0.12572622299194336,
370
+ "logps/chosen": -347.8811950683594,
371
+ "logps/rejected": -265.3426208496094,
372
+ "loss": 0.2745,
373
+ "rewards/accuracies": 0.612500011920929,
374
+ "rewards/chosen": 0.05431933328509331,
375
+ "rewards/margins": 0.1574307382106781,
376
+ "rewards/rejected": -0.1031113862991333,
377
  "step": 230
378
  },
379
  {
380
+ "epoch": 0.67,
381
+ "learning_rate": 1.4533506561564305e-07,
382
+ "logits/chosen": -0.04639708250761032,
383
+ "logits/rejected": 0.06410099565982819,
384
+ "logps/chosen": -371.84979248046875,
385
+ "logps/rejected": -256.31402587890625,
386
+ "loss": 0.2709,
387
+ "rewards/accuracies": 0.606249988079071,
388
+ "rewards/chosen": 0.05453202873468399,
389
+ "rewards/margins": 0.18710599839687347,
390
+ "rewards/rejected": -0.13257397711277008,
391
  "step": 240
392
  },
393
  {
394
+ "epoch": 0.7,
395
+ "learning_rate": 1.2358565665550387e-07,
396
+ "logits/chosen": 0.0129060298204422,
397
+ "logits/rejected": 0.13165046274662018,
398
+ "logps/chosen": -305.75457763671875,
399
+ "logps/rejected": -239.74130249023438,
400
+ "loss": 0.2815,
401
+ "rewards/accuracies": 0.606249988079071,
402
+ "rewards/chosen": 0.016258252784609795,
403
+ "rewards/margins": 0.1473822295665741,
404
+ "rewards/rejected": -0.13112398982048035,
405
  "step": 250
406
  },
407
  {
408
+ "epoch": 0.73,
409
+ "learning_rate": 1.0305368692688174e-07,
410
+ "logits/chosen": -0.16727735102176666,
411
+ "logits/rejected": -0.08288873732089996,
412
+ "logps/chosen": -390.76519775390625,
413
+ "logps/rejected": -272.0246887207031,
414
+ "loss": 0.284,
415
+ "rewards/accuracies": 0.637499988079071,
416
+ "rewards/chosen": 0.08555278927087784,
417
+ "rewards/margins": 0.22741886973381042,
418
+ "rewards/rejected": -0.141866073012352,
419
  "step": 260
420
  },
421
  {
422
+ "epoch": 0.76,
423
+ "learning_rate": 8.393689052217964e-08,
424
+ "logits/chosen": -0.04847250133752823,
425
+ "logits/rejected": 0.006302345544099808,
426
+ "logps/chosen": -329.28369140625,
427
+ "logps/rejected": -255.73458862304688,
428
+ "loss": 0.2715,
429
+ "rewards/accuracies": 0.612500011920929,
430
+ "rewards/chosen": 0.0007442310452461243,
431
+ "rewards/margins": 0.1538374274969101,
432
+ "rewards/rejected": -0.15309318900108337,
433
  "step": 270
434
  },
435
  {
436
+ "epoch": 0.79,
437
+ "learning_rate": 6.641937264107867e-08,
438
+ "logits/chosen": -0.02019808068871498,
439
+ "logits/rejected": 0.09664047509431839,
440
+ "logps/chosen": -372.004150390625,
441
+ "logps/rejected": -239.39230346679688,
442
+ "loss": 0.2873,
443
+ "rewards/accuracies": 0.606249988079071,
444
+ "rewards/chosen": 0.05372091382741928,
445
+ "rewards/margins": 0.2529592514038086,
446
+ "rewards/rejected": -0.1992383599281311,
447
  "step": 280
448
  },
449
  {
450
+ "epoch": 0.81,
451
+ "learning_rate": 5.066983655682325e-08,
452
+ "logits/chosen": 0.012843111529946327,
453
+ "logits/rejected": 0.14069826900959015,
454
+ "logps/chosen": -388.0031433105469,
455
+ "logps/rejected": -264.6817626953125,
456
+ "loss": 0.282,
457
+ "rewards/accuracies": 0.6187499761581421,
458
+ "rewards/chosen": 0.051469434052705765,
459
+ "rewards/margins": 0.20394675433635712,
460
+ "rewards/rejected": -0.15247733891010284,
461
  "step": 290
462
  },
463
  {
464
+ "epoch": 0.84,
465
+ "learning_rate": 3.683995891147695e-08,
466
+ "logits/chosen": 0.0027360408566892147,
467
+ "logits/rejected": 0.08911313861608505,
468
+ "logps/chosen": -389.00469970703125,
469
+ "logps/rejected": -265.6505432128906,
470
+ "loss": 0.2824,
471
+ "rewards/accuracies": 0.581250011920929,
472
+ "rewards/chosen": 0.06350895017385483,
473
+ "rewards/margins": 0.215250164270401,
474
+ "rewards/rejected": -0.15174119174480438,
475
  "step": 300
476
  },
477
  {
478
+ "epoch": 0.84,
479
+ "eval_logits/chosen": 0.33101534843444824,
480
+ "eval_logits/rejected": 0.41789910197257996,
481
+ "eval_logps/chosen": -350.06695556640625,
482
+ "eval_logps/rejected": -361.3666076660156,
483
+ "eval_loss": 0.1257748305797577,
484
+ "eval_rewards/accuracies": 0.54296875,
485
+ "eval_rewards/chosen": 0.09384238719940186,
486
+ "eval_rewards/margins": 0.050414081662893295,
487
+ "eval_rewards/rejected": 0.04342830926179886,
488
+ "eval_runtime": 75.6409,
489
+ "eval_samples_per_second": 26.441,
490
+ "eval_steps_per_second": 0.423,
491
  "step": 300
492
  },
493
  {
494
+ "epoch": 0.87,
495
+ "learning_rate": 2.5062928986944676e-08,
496
+ "logits/chosen": 0.008039293810725212,
497
+ "logits/rejected": 0.10899440944194794,
498
+ "logps/chosen": -319.66778564453125,
499
+ "logps/rejected": -248.84130859375,
500
+ "loss": 0.2817,
501
+ "rewards/accuracies": 0.6000000238418579,
502
+ "rewards/chosen": 0.021648388355970383,
503
+ "rewards/margins": 0.19138005375862122,
504
+ "rewards/rejected": -0.16973166167736053,
505
  "step": 310
506
  },
507
  {
508
+ "epoch": 0.9,
509
+ "learning_rate": 1.5452166019378987e-08,
510
+ "logits/chosen": 0.03970743715763092,
511
+ "logits/rejected": 0.1318010538816452,
512
+ "logps/chosen": -340.17822265625,
513
+ "logps/rejected": -257.47357177734375,
514
+ "loss": 0.2829,
515
+ "rewards/accuracies": 0.643750011920929,
516
+ "rewards/chosen": 0.01912551559507847,
517
+ "rewards/margins": 0.16330552101135254,
518
+ "rewards/rejected": -0.14417998492717743,
519
  "step": 320
520
  },
521
  {
522
+ "epoch": 0.93,
523
+ "learning_rate": 8.100226909935059e-09,
524
+ "logits/chosen": 0.0007024571532383561,
525
+ "logits/rejected": 0.1095796599984169,
526
+ "logps/chosen": -362.5918884277344,
527
+ "logps/rejected": -277.5724792480469,
528
+ "loss": 0.2791,
529
+ "rewards/accuracies": 0.5562499761581421,
530
+ "rewards/chosen": -0.006788396742194891,
531
+ "rewards/margins": 0.13104668259620667,
532
+ "rewards/rejected": -0.13783510029315948,
533
  "step": 330
534
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
535
  {
536
  "epoch": 0.96,
537
+ "learning_rate": 3.077914851215585e-09,
538
+ "logits/chosen": 0.02732967771589756,
539
+ "logits/rejected": 0.11049536615610123,
540
+ "logps/chosen": -334.34368896484375,
541
+ "logps/rejected": -257.8246154785156,
542
+ "loss": 0.2882,
543
+ "rewards/accuracies": 0.6625000238418579,
544
+ "rewards/chosen": -0.023633623495697975,
545
+ "rewards/margins": 0.15649138391017914,
546
+ "rewards/rejected": -0.18012499809265137,
547
+ "step": 340
548
  },
549
  {
550
  "epoch": 0.98,
551
+ "learning_rate": 4.3359745382104405e-10,
552
+ "logits/chosen": -0.038154929876327515,
553
+ "logits/rejected": 0.05033816024661064,
554
+ "logps/chosen": -377.3148498535156,
555
+ "logps/rejected": -248.7563934326172,
556
+ "loss": 0.2908,
557
+ "rewards/accuracies": 0.6937500238418579,
558
+ "rewards/chosen": 0.060918308794498444,
559
+ "rewards/margins": 0.26015806198120117,
560
+ "rewards/rejected": -0.19923976063728333,
561
+ "step": 350
562
  },
563
  {
564
  "epoch": 1.0,
565
+ "step": 356,
566
  "total_flos": 0.0,
567
+ "train_loss": 0.2852043948146734,
568
+ "train_runtime": 3316.6245,
569
+ "train_samples_per_second": 13.733,
570
+ "train_steps_per_second": 0.107
571
  }
572
  ],
573
  "logging_steps": 10,
574
+ "max_steps": 356,
575
  "num_train_epochs": 1,
576
  "save_steps": 1000,
577
  "total_flos": 0.0,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c78334661030348bcfc61a245be57890af303020f8351feceea1b36f368cb134
3
  size 5944
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d403be7dca0d49ce709616288823cff1c4250954055d6d447a09cc88935e23b
3
  size 5944