wzhouad commited on
Commit
0d4f8c7
1 Parent(s): 949ac1a

Model save

Browse files
README.md CHANGED
@@ -32,7 +32,7 @@ More information needed
32
  ### Training hyperparameters
33
 
34
  The following hyperparameters were used during training:
35
- - learning_rate: 2e-06
36
  - train_batch_size: 2
37
  - eval_batch_size: 8
38
  - seed: 1
 
32
  ### Training hyperparameters
33
 
34
  The following hyperparameters were used during training:
35
+ - learning_rate: 1e-06
36
  - train_batch_size: 2
37
  - eval_batch_size: 8
38
  - seed: 1
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 2.0,
3
- "train_loss": 0.18113358499298632,
4
- "train_runtime": 8124.3794,
5
  "train_samples": 45548,
6
- "train_samples_per_second": 11.213,
7
- "train_steps_per_second": 0.087
8
  }
 
1
  {
2
  "epoch": 2.0,
3
+ "train_loss": 0.25192113166124047,
4
+ "train_runtime": 8113.4503,
5
  "train_samples": 45548,
6
+ "train_samples_per_second": 11.228,
7
+ "train_steps_per_second": 0.088
8
  }
model-00001-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5e0f852f610b5179bc0d46ffe89d5644556da8841082b008cdae3c65d2a759bb
3
  size 4976698672
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3402aa3938e1c31d822d0e375cf01dcc5f8377b7a31a3b6cf0575f1fb7953d4a
3
  size 4976698672
model-00002-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c2b594dde7a8f834fedff85680da6838d81c3edbfb2b21c505fa6c3c19bf968f
3
  size 4999802720
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:655f95ca04b0961ed4b515a7348c05cf72428837d0d1656bb20de7415dd6d23b
3
  size 4999802720
model-00003-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:efd9bdaaa5dc9e6569743b905fb69b653c0deaca599081e9a0b714570bd74b7b
3
  size 4915916176
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8c376df770dfdaedca5de952735c527acff55b46dffecb9de5c18634541d6e2
3
  size 4915916176
model-00004-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4291b6d92a0c55352fc37c06c9f74edafeb0cfa5b292a4fa380c79ded3be2a05
3
  size 1168138808
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a996e33d039658928f6a70a590d6caec1f6957e1db80bebabe1c6f9186fd855d
3
  size 1168138808
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 2.0,
3
- "train_loss": 0.18113358499298632,
4
- "train_runtime": 8124.3794,
5
  "train_samples": 45548,
6
- "train_samples_per_second": 11.213,
7
- "train_steps_per_second": 0.087
8
  }
 
1
  {
2
  "epoch": 2.0,
3
+ "train_loss": 0.25192113166124047,
4
+ "train_runtime": 8113.4503,
5
  "train_samples": 45548,
6
+ "train_samples_per_second": 11.228,
7
+ "train_steps_per_second": 0.088
8
  }
trainer_state.json CHANGED
@@ -10,7 +10,7 @@
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 2.816901408450704e-08,
14
  "logits/chosen": -0.20257116854190826,
15
  "logits/rejected": -0.07568661123514175,
16
  "logps/chosen": -521.0269775390625,
@@ -24,1006 +24,1006 @@
24
  },
25
  {
26
  "epoch": 0.03,
27
- "learning_rate": 2.8169014084507043e-07,
28
- "logits/chosen": -0.13755783438682556,
29
- "logits/rejected": -0.08772192895412445,
30
- "logps/chosen": -345.8690185546875,
31
- "logps/rejected": -280.64691162109375,
32
  "loss": 0.3323,
33
- "rewards/accuracies": 0.4513888955116272,
34
- "rewards/chosen": 4.912400618195534e-05,
35
- "rewards/margins": -0.00030259901541285217,
36
- "rewards/rejected": 0.0003517230798024684,
37
  "step": 10
38
  },
39
  {
40
  "epoch": 0.06,
41
- "learning_rate": 5.633802816901409e-07,
42
- "logits/chosen": -0.0957806184887886,
43
- "logits/rejected": -0.057603687047958374,
44
- "logps/chosen": -331.79766845703125,
45
- "logps/rejected": -241.81185913085938,
46
- "loss": 0.329,
47
- "rewards/accuracies": 0.550000011920929,
48
- "rewards/chosen": 0.0004024081281386316,
49
- "rewards/margins": 0.00799016747623682,
50
- "rewards/rejected": -0.007587759289890528,
51
  "step": 20
52
  },
53
  {
54
  "epoch": 0.08,
55
- "learning_rate": 8.450704225352112e-07,
56
- "logits/chosen": -0.05522637814283371,
57
- "logits/rejected": 0.0007691409555263817,
58
- "logps/chosen": -387.5187072753906,
59
- "logps/rejected": -256.33380126953125,
60
- "loss": 0.3393,
61
- "rewards/accuracies": 0.606249988079071,
62
- "rewards/chosen": 0.0015848552575334907,
63
- "rewards/margins": 0.056810565292835236,
64
- "rewards/rejected": -0.05522570759057999,
65
  "step": 30
66
  },
67
  {
68
  "epoch": 0.11,
69
- "learning_rate": 1.1267605633802817e-06,
70
- "logits/chosen": -0.07449092715978622,
71
- "logits/rejected": -0.021578723564743996,
72
- "logps/chosen": -341.57391357421875,
73
- "logps/rejected": -272.60400390625,
74
- "loss": 0.3699,
75
- "rewards/accuracies": 0.518750011920929,
76
- "rewards/chosen": -0.0639580637216568,
77
- "rewards/margins": 0.056426145136356354,
78
- "rewards/rejected": -0.12038421630859375,
79
  "step": 40
80
  },
81
  {
82
  "epoch": 0.14,
83
- "learning_rate": 1.408450704225352e-06,
84
- "logits/chosen": -0.06206550449132919,
85
- "logits/rejected": -0.011999653652310371,
86
- "logps/chosen": -344.6331787109375,
87
- "logps/rejected": -287.69635009765625,
88
- "loss": 0.3883,
89
- "rewards/accuracies": 0.6000000238418579,
90
- "rewards/chosen": -0.10917987674474716,
91
- "rewards/margins": 0.14197741448879242,
92
- "rewards/rejected": -0.251157283782959,
93
  "step": 50
94
  },
95
  {
96
  "epoch": 0.17,
97
- "learning_rate": 1.6901408450704225e-06,
98
- "logits/chosen": -0.17808644473552704,
99
- "logits/rejected": -0.11181376129388809,
100
- "logps/chosen": -425.6441345214844,
101
- "logps/rejected": -281.8149108886719,
102
- "loss": 0.3986,
103
  "rewards/accuracies": 0.6875,
104
- "rewards/chosen": -0.03321179002523422,
105
- "rewards/margins": 0.33166056871414185,
106
- "rewards/rejected": -0.36487236618995667,
107
  "step": 60
108
  },
109
  {
110
  "epoch": 0.2,
111
- "learning_rate": 1.971830985915493e-06,
112
- "logits/chosen": -0.17757034301757812,
113
- "logits/rejected": -0.054698117077350616,
114
- "logps/chosen": -403.33819580078125,
115
- "logps/rejected": -279.6980895996094,
116
- "loss": 0.3837,
117
- "rewards/accuracies": 0.6312500238418579,
118
- "rewards/chosen": 0.03798174858093262,
119
- "rewards/margins": 0.2265440672636032,
120
- "rewards/rejected": -0.1885623335838318,
121
  "step": 70
122
  },
123
  {
124
  "epoch": 0.22,
125
- "learning_rate": 1.9990212265199736e-06,
126
- "logits/chosen": -0.1993887722492218,
127
- "logits/rejected": -0.19502341747283936,
128
- "logps/chosen": -284.59423828125,
129
- "logps/rejected": -219.568359375,
130
- "loss": 0.3918,
131
- "rewards/accuracies": 0.6499999761581421,
132
- "rewards/chosen": 0.027263695374131203,
133
- "rewards/margins": 0.18515612185001373,
134
- "rewards/rejected": -0.1578924059867859,
135
  "step": 80
136
  },
137
  {
138
  "epoch": 0.25,
139
- "learning_rate": 1.995640271796129e-06,
140
- "logits/chosen": -0.3508986830711365,
141
- "logits/rejected": -0.2989083230495453,
142
- "logps/chosen": -388.96124267578125,
143
- "logps/rejected": -249.9578094482422,
144
- "loss": 0.3972,
145
  "rewards/accuracies": 0.7124999761581421,
146
- "rewards/chosen": 0.1189202070236206,
147
- "rewards/margins": 0.3718037009239197,
148
- "rewards/rejected": -0.25288352370262146,
149
  "step": 90
150
  },
151
  {
152
  "epoch": 0.28,
153
- "learning_rate": 1.9898532207817787e-06,
154
- "logits/chosen": -0.13028475642204285,
155
- "logits/rejected": -0.12288811057806015,
156
- "logps/chosen": -375.6239929199219,
157
- "logps/rejected": -300.58013916015625,
158
- "loss": 0.3681,
159
- "rewards/accuracies": 0.612500011920929,
160
- "rewards/chosen": -0.078019879758358,
161
- "rewards/margins": 0.1508217304944992,
162
- "rewards/rejected": -0.2288416177034378,
163
  "step": 100
164
  },
165
  {
166
  "epoch": 0.31,
167
- "learning_rate": 1.9816740586504575e-06,
168
- "logits/chosen": -0.09742162376642227,
169
- "logits/rejected": -0.04224073141813278,
170
- "logps/chosen": -351.0619201660156,
171
- "logps/rejected": -310.95257568359375,
172
- "loss": 0.3116,
173
- "rewards/accuracies": 0.53125,
174
- "rewards/chosen": -0.23143887519836426,
175
- "rewards/margins": 0.0850377082824707,
176
- "rewards/rejected": -0.3164765536785126,
177
  "step": 110
178
  },
179
  {
180
  "epoch": 0.34,
181
- "learning_rate": 1.971122551428331e-06,
182
- "logits/chosen": -0.10740338265895844,
183
- "logits/rejected": -0.07498879730701447,
184
- "logps/chosen": -354.75677490234375,
185
- "logps/rejected": -261.3316955566406,
186
- "loss": 0.3164,
187
  "rewards/accuracies": 0.6312500238418579,
188
- "rewards/chosen": -0.22371773421764374,
189
- "rewards/margins": 0.21556267142295837,
190
- "rewards/rejected": -0.4392804205417633,
191
  "step": 120
192
  },
193
  {
194
  "epoch": 0.37,
195
- "learning_rate": 1.9582241982269803e-06,
196
- "logits/chosen": -0.090018130838871,
197
- "logits/rejected": -0.013507525436580181,
198
- "logps/chosen": -415.02020263671875,
199
- "logps/rejected": -290.92144775390625,
200
- "loss": 0.3473,
201
- "rewards/accuracies": 0.6875,
202
- "rewards/chosen": -0.28872472047805786,
203
- "rewards/margins": 0.3263753056526184,
204
- "rewards/rejected": -0.6151000261306763,
205
  "step": 130
206
  },
207
  {
208
  "epoch": 0.39,
209
- "learning_rate": 1.9430101696214336e-06,
210
- "logits/chosen": -0.1004706472158432,
211
- "logits/rejected": -0.08849872648715973,
212
- "logps/chosen": -363.7771301269531,
213
- "logps/rejected": -290.38299560546875,
214
- "loss": 0.3338,
215
- "rewards/accuracies": 0.581250011920929,
216
- "rewards/chosen": -0.2705627381801605,
217
- "rewards/margins": 0.14047202467918396,
218
- "rewards/rejected": -0.4110347628593445,
219
  "step": 140
220
  },
221
  {
222
  "epoch": 0.42,
223
- "learning_rate": 1.9255172323223463e-06,
224
- "logits/chosen": -0.04992828518152237,
225
- "logits/rejected": 0.011964783072471619,
226
- "logps/chosen": -336.70928955078125,
227
- "logps/rejected": -298.07611083984375,
228
- "loss": 0.3193,
229
- "rewards/accuracies": 0.5562499761581421,
230
- "rewards/chosen": -0.31615492701530457,
231
- "rewards/margins": 0.1844826191663742,
232
- "rewards/rejected": -0.5006375312805176,
233
  "step": 150
234
  },
235
  {
236
  "epoch": 0.45,
237
- "learning_rate": 1.905787660324391e-06,
238
- "logits/chosen": -0.011503048241138458,
239
- "logits/rejected": 0.1220928207039833,
240
- "logps/chosen": -424.7059631347656,
241
- "logps/rejected": -320.0885009765625,
242
- "loss": 0.3057,
243
- "rewards/accuracies": 0.612500011920929,
244
- "rewards/chosen": -0.3887358605861664,
245
- "rewards/margins": 0.23610559105873108,
246
- "rewards/rejected": -0.6248414516448975,
247
  "step": 160
248
  },
249
  {
250
  "epoch": 0.48,
251
- "learning_rate": 1.8838691327455609e-06,
252
- "logits/chosen": -0.026822054758667946,
253
- "logits/rejected": 0.04267163202166557,
254
- "logps/chosen": -369.6314392089844,
255
- "logps/rejected": -326.5650329589844,
256
- "loss": 0.3049,
257
- "rewards/accuracies": 0.612500011920929,
258
- "rewards/chosen": -0.37862712144851685,
259
- "rewards/margins": 0.2466701716184616,
260
- "rewards/rejected": -0.6252972483634949,
261
  "step": 170
262
  },
263
  {
264
  "epoch": 0.51,
265
- "learning_rate": 1.8598146186042808e-06,
266
- "logits/chosen": -0.2679498791694641,
267
- "logits/rejected": -0.21142823994159698,
268
- "logps/chosen": -383.9067687988281,
269
- "logps/rejected": -319.5235900878906,
270
- "loss": 0.3293,
271
- "rewards/accuracies": 0.612500011920929,
272
- "rewards/chosen": -0.4264276623725891,
273
- "rewards/margins": 0.24799779057502747,
274
- "rewards/rejected": -0.674425482749939,
275
  "step": 180
276
  },
277
  {
278
  "epoch": 0.53,
279
- "learning_rate": 1.8336822488127723e-06,
280
- "logits/chosen": -0.05206901952624321,
281
- "logits/rejected": 0.03507734090089798,
282
- "logps/chosen": -408.74798583984375,
283
- "logps/rejected": -323.0229797363281,
284
- "loss": 0.284,
285
  "rewards/accuracies": 0.643750011920929,
286
- "rewards/chosen": -0.5230687260627747,
287
- "rewards/margins": 0.4133565425872803,
288
- "rewards/rejected": -0.9364253282546997,
289
  "step": 190
290
  },
291
  {
292
  "epoch": 0.56,
293
- "learning_rate": 1.805535175696026e-06,
294
- "logits/chosen": -0.026757067069411278,
295
- "logits/rejected": 0.11062588542699814,
296
- "logps/chosen": -406.74176025390625,
297
- "logps/rejected": -343.10546875,
298
- "loss": 0.2943,
299
- "rewards/accuracies": 0.6000000238418579,
300
- "rewards/chosen": -0.7670020461082458,
301
- "rewards/margins": 0.22102268040180206,
302
- "rewards/rejected": -0.9880247116088867,
303
  "step": 200
304
  },
305
  {
306
  "epoch": 0.59,
307
- "learning_rate": 1.7754414203758602e-06,
308
- "logits/chosen": 0.09901715815067291,
309
- "logits/rejected": 0.19180616736412048,
310
- "logps/chosen": -463.052734375,
311
- "logps/rejected": -346.2613220214844,
312
- "loss": 0.2641,
313
- "rewards/accuracies": 0.6312500238418579,
314
- "rewards/chosen": -0.6034746766090393,
315
- "rewards/margins": 0.1767103672027588,
316
- "rewards/rejected": -0.7801850438117981,
317
  "step": 210
318
  },
319
  {
320
  "epoch": 0.62,
321
- "learning_rate": 1.7434737083888904e-06,
322
- "logits/chosen": 0.036244794726371765,
323
- "logits/rejected": 0.10729759931564331,
324
- "logps/chosen": -457.8307189941406,
325
- "logps/rejected": -356.2217712402344,
326
- "loss": 0.2726,
327
- "rewards/accuracies": 0.6812499761581421,
328
- "rewards/chosen": -0.6712073087692261,
329
- "rewards/margins": 0.2408912628889084,
330
- "rewards/rejected": -0.9120985865592957,
331
  "step": 220
332
  },
333
  {
334
  "epoch": 0.65,
335
- "learning_rate": 1.709709293935662e-06,
336
- "logits/chosen": 0.09133528172969818,
337
- "logits/rejected": 0.2221805602312088,
338
- "logps/chosen": -408.2891540527344,
339
- "logps/rejected": -331.88568115234375,
340
- "loss": 0.2977,
341
  "rewards/accuracies": 0.612500011920929,
342
- "rewards/chosen": -0.5791844129562378,
343
- "rewards/margins": 0.30291682481765747,
344
- "rewards/rejected": -0.8821012377738953,
345
  "step": 230
346
  },
347
  {
348
  "epoch": 0.67,
349
- "learning_rate": 1.6742297731856636e-06,
350
- "logits/chosen": -0.13875767588615417,
351
- "logits/rejected": -0.030141670256853104,
352
- "logps/chosen": -401.0727233886719,
353
- "logps/rejected": -305.03875732421875,
354
- "loss": 0.301,
355
- "rewards/accuracies": 0.65625,
356
- "rewards/chosen": -0.4225517213344574,
357
- "rewards/margins": 0.2361893206834793,
358
- "rewards/rejected": -0.6587410569190979,
359
  "step": 240
360
  },
361
  {
362
  "epoch": 0.7,
363
- "learning_rate": 1.6371208870894001e-06,
364
- "logits/chosen": -0.11314062029123306,
365
- "logits/rejected": -0.028332043439149857,
366
- "logps/chosen": -430.2290954589844,
367
- "logps/rejected": -321.7243957519531,
368
- "loss": 0.3146,
369
- "rewards/accuracies": 0.6625000238418579,
370
- "rewards/chosen": -0.48336172103881836,
371
- "rewards/margins": 0.32641273736953735,
372
- "rewards/rejected": -0.8097743988037109,
373
  "step": 250
374
  },
375
  {
376
  "epoch": 0.73,
377
- "learning_rate": 1.5984723141740574e-06,
378
- "logits/chosen": -0.060978569090366364,
379
- "logits/rejected": 0.013845810666680336,
380
- "logps/chosen": -429.4888610839844,
381
- "logps/rejected": -350.95147705078125,
382
- "loss": 0.3233,
383
- "rewards/accuracies": 0.6499999761581421,
384
- "rewards/chosen": -0.6816853880882263,
385
- "rewards/margins": 0.37134552001953125,
386
- "rewards/rejected": -1.0530308485031128,
387
  "step": 260
388
  },
389
  {
390
  "epoch": 0.76,
391
- "learning_rate": 1.5583774538234882e-06,
392
- "logits/chosen": 0.009507184848189354,
393
- "logits/rejected": 0.053323499858379364,
394
- "logps/chosen": -420.3421325683594,
395
- "logps/rejected": -358.8752136230469,
396
- "loss": 0.2801,
397
- "rewards/accuracies": 0.606249988079071,
398
- "rewards/chosen": -0.8152508735656738,
399
- "rewards/margins": 0.3313834071159363,
400
- "rewards/rejected": -1.1466342210769653,
401
  "step": 270
402
  },
403
  {
404
  "epoch": 0.79,
405
- "learning_rate": 1.5169332005662589e-06,
406
- "logits/chosen": 0.036943964660167694,
407
- "logits/rejected": 0.12250173091888428,
408
- "logps/chosen": -484.43389892578125,
409
- "logps/rejected": -375.3249816894531,
410
- "loss": 0.2936,
411
- "rewards/accuracies": 0.706250011920929,
412
- "rewards/chosen": -0.9159828424453735,
413
- "rewards/margins": 0.4136818051338196,
414
- "rewards/rejected": -1.3296645879745483,
415
  "step": 280
416
  },
417
  {
418
  "epoch": 0.81,
419
- "learning_rate": 1.474239709917218e-06,
420
- "logits/chosen": 0.05337507277727127,
421
- "logits/rejected": 0.19436481595039368,
422
- "logps/chosen": -443.375244140625,
423
- "logps/rejected": -374.6485900878906,
424
- "loss": 0.2803,
425
- "rewards/accuracies": 0.612500011920929,
426
- "rewards/chosen": -0.916093647480011,
427
- "rewards/margins": 0.25273779034614563,
428
- "rewards/rejected": -1.1688312292099,
429
  "step": 290
430
  },
431
  {
432
  "epoch": 0.84,
433
- "learning_rate": 1.430400156338457e-06,
434
- "logits/chosen": -0.04387836158275604,
435
- "logits/rejected": 0.08865033835172653,
436
- "logps/chosen": -432.93719482421875,
437
- "logps/rejected": -336.19378662109375,
438
- "loss": 0.2739,
439
- "rewards/accuracies": 0.6499999761581421,
440
- "rewards/chosen": -0.5575471520423889,
441
- "rewards/margins": 0.33055704832077026,
442
- "rewards/rejected": -0.888104259967804,
443
  "step": 300
444
  },
445
  {
446
  "epoch": 0.87,
447
- "learning_rate": 1.3855204839045892e-06,
448
- "logits/chosen": -0.09471622109413147,
449
- "logits/rejected": 0.04182841628789902,
450
- "logps/chosen": -394.19952392578125,
451
- "logps/rejected": -326.4076232910156,
452
- "loss": 0.2828,
453
- "rewards/accuracies": 0.65625,
454
- "rewards/chosen": -0.5341824293136597,
455
- "rewards/margins": 0.30772727727890015,
456
- "rewards/rejected": -0.8419097661972046,
457
  "step": 310
458
  },
459
  {
460
  "epoch": 0.9,
461
- "learning_rate": 1.3397091502748927e-06,
462
- "logits/chosen": 0.06928735971450806,
463
- "logits/rejected": 0.12053768336772919,
464
- "logps/chosen": -407.7933654785156,
465
- "logps/rejected": -357.62225341796875,
466
- "loss": 0.3016,
467
- "rewards/accuracies": 0.581250011920929,
468
- "rewards/chosen": -0.7507076263427734,
469
- "rewards/margins": 0.2525845170021057,
470
- "rewards/rejected": -1.0032920837402344,
471
  "step": 320
472
  },
473
  {
474
  "epoch": 0.93,
475
- "learning_rate": 1.2930768645910449e-06,
476
- "logits/chosen": -0.04908103495836258,
477
- "logits/rejected": 0.05275033786892891,
478
- "logps/chosen": -489.91741943359375,
479
- "logps/rejected": -391.38861083984375,
480
- "loss": 0.269,
481
- "rewards/accuracies": 0.625,
482
- "rewards/chosen": -0.7464247941970825,
483
- "rewards/margins": 0.3772323727607727,
484
- "rewards/rejected": -1.1236572265625,
485
  "step": 330
486
  },
487
  {
488
  "epoch": 0.96,
489
- "learning_rate": 1.2457363199338495e-06,
490
- "logits/chosen": -0.023084616288542747,
491
- "logits/rejected": 0.06722521036863327,
492
- "logps/chosen": -424.7259216308594,
493
- "logps/rejected": -344.1465148925781,
494
- "loss": 0.2641,
495
- "rewards/accuracies": 0.643750011920929,
496
- "rewards/chosen": -0.7394579648971558,
497
- "rewards/margins": 0.2995792627334595,
498
- "rewards/rejected": -1.0390373468399048,
499
  "step": 340
500
  },
501
  {
502
  "epoch": 0.98,
503
- "learning_rate": 1.1978019209855173e-06,
504
- "logits/chosen": -0.03397887200117111,
505
- "logits/rejected": 0.0008539498085156083,
506
- "logps/chosen": -443.153076171875,
507
- "logps/rejected": -367.44195556640625,
508
- "loss": 0.2896,
509
- "rewards/accuracies": 0.668749988079071,
510
- "rewards/chosen": -0.816065788269043,
511
- "rewards/margins": 0.3647410273551941,
512
- "rewards/rejected": -1.1808068752288818,
513
  "step": 350
514
  },
515
  {
516
  "epoch": 1.01,
517
- "learning_rate": 1.14938950755563e-06,
518
- "logits/chosen": -0.0028023570775985718,
519
- "logits/rejected": 0.09356020390987396,
520
- "logps/chosen": -409.0108947753906,
521
- "logps/rejected": -419.5281677246094,
522
- "loss": 0.2361,
523
- "rewards/accuracies": 0.6937500238418579,
524
- "rewards/chosen": -0.937981128692627,
525
- "rewards/margins": 0.5299195051193237,
526
- "rewards/rejected": -1.4679005146026611,
527
  "step": 360
528
  },
529
  {
530
  "epoch": 1.04,
531
- "learning_rate": 1.1006160746389332e-06,
532
- "logits/chosen": -0.15354487299919128,
533
- "logits/rejected": -0.0770144909620285,
534
- "logps/chosen": -461.51806640625,
535
- "logps/rejected": -445.2923278808594,
536
- "loss": 0.102,
537
- "rewards/accuracies": 0.8125,
538
- "rewards/chosen": -1.0436856746673584,
539
- "rewards/margins": 0.894515335559845,
540
- "rewards/rejected": -1.9382011890411377,
541
  "step": 370
542
  },
543
  {
544
  "epoch": 1.07,
545
- "learning_rate": 1.0515994896814731e-06,
546
- "logits/chosen": -0.12813356518745422,
547
- "logits/rejected": -0.008146116510033607,
548
- "logps/chosen": -474.5690002441406,
549
- "logps/rejected": -505.58367919921875,
550
- "loss": 0.076,
551
- "rewards/accuracies": 0.800000011920929,
552
- "rewards/chosen": -1.539747953414917,
553
- "rewards/margins": 0.9511996507644653,
554
- "rewards/rejected": -2.490947723388672,
555
  "step": 380
556
  },
557
  {
558
  "epoch": 1.1,
559
- "learning_rate": 1.002458207738333e-06,
560
- "logits/chosen": 0.014720706269145012,
561
- "logits/rejected": 0.19409914314746857,
562
- "logps/chosen": -462.5455627441406,
563
- "logps/rejected": -508.26947021484375,
564
- "loss": 0.0717,
565
- "rewards/accuracies": 0.793749988079071,
566
- "rewards/chosen": -1.7438398599624634,
567
- "rewards/margins": 1.1367576122283936,
568
- "rewards/rejected": -2.8805973529815674,
569
  "step": 390
570
  },
571
  {
572
  "epoch": 1.12,
573
- "learning_rate": 9.533109852113413e-07,
574
- "logits/chosen": 0.06460610777139664,
575
- "logits/rejected": 0.15287192165851593,
576
- "logps/chosen": -476.78753662109375,
577
- "logps/rejected": -506.93701171875,
578
- "loss": 0.0591,
579
- "rewards/accuracies": 0.8062499761581421,
580
- "rewards/chosen": -1.3379477262496948,
581
- "rewards/margins": 1.1289650201797485,
582
- "rewards/rejected": -2.4669127464294434,
583
  "step": 400
584
  },
585
  {
586
  "epoch": 1.15,
587
- "learning_rate": 9.042765928585326e-07,
588
- "logits/chosen": 0.21788999438285828,
589
- "logits/rejected": 0.2529798150062561,
590
- "logps/chosen": -483.2903747558594,
591
- "logps/rejected": -527.7869262695312,
592
- "loss": 0.0602,
593
- "rewards/accuracies": 0.856249988079071,
594
- "rewards/chosen": -1.615551233291626,
595
- "rewards/margins": 1.4097917079925537,
596
- "rewards/rejected": -3.0253429412841797,
597
  "step": 410
598
  },
599
  {
600
  "epoch": 1.18,
601
- "learning_rate": 8.554735287689148e-07,
602
- "logits/chosen": 0.1305566430091858,
603
- "logits/rejected": 0.27758485078811646,
604
- "logps/chosen": -527.9423217773438,
605
- "logps/rejected": -558.6550903320312,
606
- "loss": 0.062,
607
- "rewards/accuracies": 0.768750011920929,
608
- "rewards/chosen": -1.9209682941436768,
609
- "rewards/margins": 1.1167800426483154,
610
- "rewards/rejected": -3.037747859954834,
611
  "step": 420
612
  },
613
  {
614
  "epoch": 1.21,
615
- "learning_rate": 8.070197319961782e-07,
616
- "logits/chosen": 0.25931409001350403,
617
- "logits/rejected": 0.3246735632419586,
618
- "logps/chosen": -550.6995849609375,
619
- "logps/rejected": -537.9232177734375,
620
- "loss": 0.0486,
621
- "rewards/accuracies": 0.8500000238418579,
622
- "rewards/chosen": -1.6232093572616577,
623
- "rewards/margins": 1.3964422941207886,
624
- "rewards/rejected": -3.0196518898010254,
625
  "step": 430
626
  },
627
  {
628
  "epoch": 1.24,
629
- "learning_rate": 7.590322975433856e-07,
630
- "logits/chosen": 0.2402382791042328,
631
- "logits/rejected": 0.37099042534828186,
632
- "logps/chosen": -511.0347595214844,
633
- "logps/rejected": -555.0653686523438,
634
- "loss": 0.0533,
635
- "rewards/accuracies": 0.78125,
636
- "rewards/chosen": -1.9829241037368774,
637
- "rewards/margins": 1.3394062519073486,
638
- "rewards/rejected": -3.3223304748535156,
639
  "step": 440
640
  },
641
  {
642
  "epoch": 1.26,
643
- "learning_rate": 7.116271933874245e-07,
644
- "logits/chosen": 0.2019539177417755,
645
- "logits/rejected": 0.3705137372016907,
646
- "logps/chosen": -551.7188110351562,
647
- "logps/rejected": -568.9718017578125,
648
- "loss": 0.0512,
649
  "rewards/accuracies": 0.8062499761581421,
650
- "rewards/chosen": -1.8252284526824951,
651
- "rewards/margins": 1.3718860149383545,
652
- "rewards/rejected": -3.1971147060394287,
653
  "step": 450
654
  },
655
  {
656
  "epoch": 1.29,
657
- "learning_rate": 6.649189802270652e-07,
658
- "logits/chosen": 0.2931487560272217,
659
- "logits/rejected": 0.4406962990760803,
660
- "logps/chosen": -505.7900390625,
661
- "logps/rejected": -604.8612060546875,
662
- "loss": 0.0489,
663
- "rewards/accuracies": 0.793749988079071,
664
- "rewards/chosen": -1.829275131225586,
665
- "rewards/margins": 1.4683018922805786,
666
- "rewards/rejected": -3.297576904296875,
667
  "step": 460
668
  },
669
  {
670
  "epoch": 1.32,
671
- "learning_rate": 6.190205346318926e-07,
672
- "logits/chosen": 0.15103109180927277,
673
- "logits/rejected": 0.2090083360671997,
674
- "logps/chosen": -548.1585693359375,
675
- "logps/rejected": -613.2918701171875,
676
- "loss": 0.0454,
677
- "rewards/accuracies": 0.8187500238418579,
678
- "rewards/chosen": -2.007948398590088,
679
- "rewards/margins": 1.6015655994415283,
680
- "rewards/rejected": -3.609513759613037,
681
  "step": 470
682
  },
683
  {
684
  "epoch": 1.35,
685
- "learning_rate": 5.740427762611604e-07,
686
- "logits/chosen": 0.24043090641498566,
687
- "logits/rejected": 0.37472882866859436,
688
- "logps/chosen": -557.3897705078125,
689
- "logps/rejected": -610.3823852539062,
690
- "loss": 0.0382,
691
- "rewards/accuracies": 0.8125,
692
- "rewards/chosen": -2.103377103805542,
693
- "rewards/margins": 1.6295030117034912,
694
- "rewards/rejected": -3.732880115509033,
695
  "step": 480
696
  },
697
  {
698
  "epoch": 1.38,
699
- "learning_rate": 5.300943998117749e-07,
700
- "logits/chosen": 0.23212356865406036,
701
- "logits/rejected": 0.3468802273273468,
702
- "logps/chosen": -589.6870727539062,
703
- "logps/rejected": -632.7547607421875,
704
- "loss": 0.0425,
705
- "rewards/accuracies": 0.84375,
706
- "rewards/chosen": -2.1241652965545654,
707
- "rewards/margins": 1.638109564781189,
708
- "rewards/rejected": -3.762274980545044,
709
  "step": 490
710
  },
711
  {
712
  "epoch": 1.4,
713
- "learning_rate": 4.872816123431976e-07,
714
- "logits/chosen": 0.17420414090156555,
715
- "logits/rejected": 0.3029765188694,
716
- "logps/chosen": -583.6693115234375,
717
- "logps/rejected": -632.7525024414062,
718
- "loss": 0.0332,
719
- "rewards/accuracies": 0.8374999761581421,
720
- "rewards/chosen": -2.0016562938690186,
721
- "rewards/margins": 1.703657865524292,
722
- "rewards/rejected": -3.7053139209747314,
723
  "step": 500
724
  },
725
  {
726
  "epoch": 1.43,
727
- "learning_rate": 4.4570787661405e-07,
728
- "logits/chosen": 0.24483375251293182,
729
- "logits/rejected": 0.43443599343299866,
730
- "logps/chosen": -568.5266723632812,
731
- "logps/rejected": -630.6636962890625,
732
- "loss": 0.0398,
733
- "rewards/accuracies": 0.800000011920929,
734
- "rewards/chosen": -2.0844273567199707,
735
- "rewards/margins": 1.567047357559204,
736
- "rewards/rejected": -3.651474714279175,
737
  "step": 510
738
  },
739
  {
740
  "epoch": 1.46,
741
- "learning_rate": 4.0547366105068347e-07,
742
- "logits/chosen": 0.2337774932384491,
743
- "logits/rejected": 0.3223033547401428,
744
- "logps/chosen": -556.1533813476562,
745
- "logps/rejected": -626.2739868164062,
746
- "loss": 0.041,
747
- "rewards/accuracies": 0.831250011920929,
748
- "rewards/chosen": -2.1958367824554443,
749
- "rewards/margins": 1.7631685733795166,
750
- "rewards/rejected": -3.959005355834961,
751
  "step": 520
752
  },
753
  {
754
  "epoch": 1.49,
755
- "learning_rate": 3.666761969519528e-07,
756
- "logits/chosen": 0.2308119833469391,
757
- "logits/rejected": 0.40482252836227417,
758
- "logps/chosen": -531.3250732421875,
759
- "logps/rejected": -586.0523071289062,
760
- "loss": 0.0321,
761
- "rewards/accuracies": 0.78125,
762
- "rewards/chosen": -1.9775184392929077,
763
- "rewards/margins": 1.4050841331481934,
764
- "rewards/rejected": -3.3826022148132324,
765
  "step": 530
766
  },
767
  {
768
  "epoch": 1.52,
769
- "learning_rate": 3.2940924351693213e-07,
770
- "logits/chosen": 0.18636593222618103,
771
- "logits/rejected": 0.3547353744506836,
772
- "logps/chosen": -605.5850830078125,
773
- "logps/rejected": -643.3690185546875,
774
- "loss": 0.0335,
775
- "rewards/accuracies": 0.8500000238418579,
776
- "rewards/chosen": -2.183945894241333,
777
- "rewards/margins": 1.709857702255249,
778
- "rewards/rejected": -3.8938040733337402,
779
  "step": 540
780
  },
781
  {
782
  "epoch": 1.55,
783
- "learning_rate": 2.937628612634184e-07,
784
- "logits/chosen": 0.2420307844877243,
785
- "logits/rejected": 0.41368550062179565,
786
- "logps/chosen": -569.5245971679688,
787
- "logps/rejected": -632.2020263671875,
788
- "loss": 0.0344,
789
- "rewards/accuracies": 0.8187500238418579,
790
- "rewards/chosen": -2.410912036895752,
791
- "rewards/margins": 1.8822616338729858,
792
- "rewards/rejected": -4.293173789978027,
793
  "step": 550
794
  },
795
  {
796
  "epoch": 1.57,
797
- "learning_rate": 2.598231943847916e-07,
798
- "logits/chosen": 0.22710604965686798,
799
- "logits/rejected": 0.4254523813724518,
800
- "logps/chosen": -606.672119140625,
801
- "logps/rejected": -696.2296142578125,
802
- "loss": 0.0376,
803
- "rewards/accuracies": 0.78125,
804
- "rewards/chosen": -2.5400805473327637,
805
- "rewards/margins": 1.9197076559066772,
806
- "rewards/rejected": -4.4597883224487305,
807
  "step": 560
808
  },
809
  {
810
  "epoch": 1.6,
811
- "learning_rate": 2.276722625711861e-07,
812
- "logits/chosen": 0.2909803092479706,
813
- "logits/rejected": 0.4825025200843811,
814
- "logps/chosen": -583.4527587890625,
815
- "logps/rejected": -639.517333984375,
816
- "loss": 0.0302,
817
- "rewards/accuracies": 0.8500000238418579,
818
- "rewards/chosen": -2.0665905475616455,
819
- "rewards/margins": 1.7480655908584595,
820
- "rewards/rejected": -3.8146564960479736,
821
  "step": 570
822
  },
823
  {
824
  "epoch": 1.63,
825
- "learning_rate": 1.973877627980699e-07,
826
- "logits/chosen": 0.26504355669021606,
827
- "logits/rejected": 0.332762211561203,
828
- "logps/chosen": -514.5237426757812,
829
- "logps/rejected": -629.0364990234375,
830
- "loss": 0.0301,
831
- "rewards/accuracies": 0.8187500238418579,
832
- "rewards/chosen": -1.9947683811187744,
833
- "rewards/margins": 1.5820239782333374,
834
- "rewards/rejected": -3.5767929553985596,
835
  "step": 580
836
  },
837
  {
838
  "epoch": 1.66,
839
- "learning_rate": 1.6904288156123636e-07,
840
- "logits/chosen": 0.23839843273162842,
841
- "logits/rejected": 0.39869600534439087,
842
- "logps/chosen": -580.44921875,
843
- "logps/rejected": -675.9835815429688,
844
- "loss": 0.0358,
845
- "rewards/accuracies": 0.856249988079071,
846
- "rewards/chosen": -2.2089920043945312,
847
- "rewards/margins": 1.9142862558364868,
848
- "rewards/rejected": -4.12327766418457,
849
  "step": 590
850
  },
851
  {
852
  "epoch": 1.69,
853
- "learning_rate": 1.4270611801196642e-07,
854
- "logits/chosen": 0.32313600182533264,
855
- "logits/rejected": 0.4433811604976654,
856
- "logps/chosen": -588.3478393554688,
857
- "logps/rejected": -604.0819091796875,
858
- "loss": 0.031,
859
- "rewards/accuracies": 0.793749988079071,
860
- "rewards/chosen": -2.1629788875579834,
861
- "rewards/margins": 1.6147849559783936,
862
- "rewards/rejected": -3.777763843536377,
863
  "step": 600
864
  },
865
  {
866
  "epoch": 1.71,
867
- "learning_rate": 1.1844111841977633e-07,
868
- "logits/chosen": 0.2681792378425598,
869
- "logits/rejected": 0.34347304701805115,
870
- "logps/chosen": -597.367431640625,
871
- "logps/rejected": -694.4761962890625,
872
- "loss": 0.0319,
873
- "rewards/accuracies": 0.8187500238418579,
874
- "rewards/chosen": -2.333829879760742,
875
- "rewards/margins": 1.9641401767730713,
876
- "rewards/rejected": -4.297970294952393,
877
  "step": 610
878
  },
879
  {
880
  "epoch": 1.74,
881
- "learning_rate": 9.630652236279625e-08,
882
- "logits/chosen": 0.28177735209465027,
883
- "logits/rejected": 0.3899393081665039,
884
- "logps/chosen": -565.4825439453125,
885
- "logps/rejected": -616.7823486328125,
886
- "loss": 0.0329,
887
- "rewards/accuracies": 0.8374999761581421,
888
- "rewards/chosen": -2.153439521789551,
889
- "rewards/margins": 1.7681152820587158,
890
- "rewards/rejected": -3.9215550422668457,
891
  "step": 620
892
  },
893
  {
894
  "epoch": 1.77,
895
- "learning_rate": 7.63558210174814e-08,
896
- "logits/chosen": 0.21648459136486053,
897
- "logits/rejected": 0.4712795615196228,
898
- "logps/chosen": -561.133056640625,
899
- "logps/rejected": -631.644287109375,
900
- "loss": 0.0334,
901
- "rewards/accuracies": 0.800000011920929,
902
- "rewards/chosen": -2.387845754623413,
903
- "rewards/margins": 1.6112064123153687,
904
- "rewards/rejected": -3.9990525245666504,
905
  "step": 630
906
  },
907
  {
908
  "epoch": 1.8,
909
- "learning_rate": 5.8637227890115273e-08,
910
- "logits/chosen": 0.23778888583183289,
911
- "logits/rejected": 0.34435296058654785,
912
- "logps/chosen": -649.9002685546875,
913
- "logps/rejected": -715.91943359375,
914
- "loss": 0.0299,
915
- "rewards/accuracies": 0.856249988079071,
916
- "rewards/chosen": -2.4873223304748535,
917
- "rewards/margins": 2.0735325813293457,
918
- "rewards/rejected": -4.560854911804199,
919
  "step": 640
920
  },
921
  {
922
  "epoch": 1.83,
923
- "learning_rate": 4.3193562302499046e-08,
924
- "logits/chosen": 0.3359209895133972,
925
- "logits/rejected": 0.4597659707069397,
926
- "logps/chosen": -579.0816650390625,
927
- "logps/rejected": -621.2535400390625,
928
- "loss": 0.0336,
929
- "rewards/accuracies": 0.8374999761581421,
930
- "rewards/chosen": -2.270595073699951,
931
- "rewards/margins": 1.7250922918319702,
932
- "rewards/rejected": -3.995687484741211,
933
  "step": 650
934
  },
935
  {
936
  "epoch": 1.85,
937
- "learning_rate": 3.006214591340339e-08,
938
- "logits/chosen": 0.2430475950241089,
939
- "logits/rejected": 0.3344910740852356,
940
- "logps/chosen": -618.7166748046875,
941
- "logps/rejected": -714.7845458984375,
942
- "loss": 0.0302,
943
- "rewards/accuracies": 0.8687499761581421,
944
- "rewards/chosen": -2.330672264099121,
945
- "rewards/margins": 2.0827202796936035,
946
- "rewards/rejected": -4.413392543792725,
947
  "step": 660
948
  },
949
  {
950
  "epoch": 1.88,
951
- "learning_rate": 1.9274712525847447e-08,
952
- "logits/chosen": 0.3407454490661621,
953
- "logits/rejected": 0.4727950692176819,
954
- "logps/chosen": -568.6539306640625,
955
- "logps/rejected": -646.0457153320312,
956
- "loss": 0.0271,
957
- "rewards/accuracies": 0.8125,
958
- "rewards/chosen": -2.3302292823791504,
959
- "rewards/margins": 1.696593999862671,
960
- "rewards/rejected": -4.0268235206604,
961
  "step": 670
962
  },
963
  {
964
  "epoch": 1.91,
965
- "learning_rate": 1.0857331398169577e-08,
966
- "logits/chosen": 0.16540665924549103,
967
- "logits/rejected": 0.4224317669868469,
968
- "logps/chosen": -581.6246337890625,
969
- "logps/rejected": -642.5072021484375,
970
- "loss": 0.0299,
971
- "rewards/accuracies": 0.824999988079071,
972
- "rewards/chosen": -2.267688274383545,
973
- "rewards/margins": 1.918178915977478,
974
- "rewards/rejected": -4.185866832733154,
975
  "step": 680
976
  },
977
  {
978
  "epoch": 1.94,
979
- "learning_rate": 4.830344244220686e-09,
980
- "logits/chosen": 0.14440816640853882,
981
- "logits/rejected": 0.27977341413497925,
982
- "logps/chosen": -637.5778198242188,
983
- "logps/rejected": -684.380859375,
984
- "loss": 0.0277,
985
- "rewards/accuracies": 0.8687499761581421,
986
- "rewards/chosen": -2.4349887371063232,
987
- "rewards/margins": 1.7913310527801514,
988
- "rewards/rejected": -4.226320266723633,
989
  "step": 690
990
  },
991
  {
992
  "epoch": 1.97,
993
- "learning_rate": 1.2083160749236653e-09,
994
- "logits/chosen": 0.23438580334186554,
995
- "logits/rejected": 0.3880893290042877,
996
- "logps/chosen": -624.1004028320312,
997
- "logps/rejected": -729.334716796875,
998
- "loss": 0.0276,
999
- "rewards/accuracies": 0.831250011920929,
1000
- "rewards/chosen": -2.6064438819885254,
1001
- "rewards/margins": 1.8713786602020264,
1002
- "rewards/rejected": -4.477822780609131,
1003
  "step": 700
1004
  },
1005
  {
1006
  "epoch": 2.0,
1007
  "learning_rate": 0.0,
1008
- "logits/chosen": 0.3188737630844116,
1009
- "logits/rejected": 0.4447614252567291,
1010
- "logps/chosen": -564.446533203125,
1011
- "logps/rejected": -686.6253662109375,
1012
- "loss": 0.0308,
1013
- "rewards/accuracies": 0.887499988079071,
1014
- "rewards/chosen": -2.405637264251709,
1015
- "rewards/margins": 1.9922983646392822,
1016
- "rewards/rejected": -4.39793586730957,
1017
  "step": 710
1018
  },
1019
  {
1020
  "epoch": 2.0,
1021
  "step": 710,
1022
  "total_flos": 0.0,
1023
- "train_loss": 0.18113358499298632,
1024
- "train_runtime": 8124.3794,
1025
- "train_samples_per_second": 11.213,
1026
- "train_steps_per_second": 0.087
1027
  }
1028
  ],
1029
  "logging_steps": 10,
 
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.408450704225352e-08,
14
  "logits/chosen": -0.20257116854190826,
15
  "logits/rejected": -0.07568661123514175,
16
  "logps/chosen": -521.0269775390625,
 
24
  },
25
  {
26
  "epoch": 0.03,
27
+ "learning_rate": 1.4084507042253522e-07,
28
+ "logits/chosen": -0.13786724209785461,
29
+ "logits/rejected": -0.08816312253475189,
30
+ "logps/chosen": -345.8097229003906,
31
+ "logps/rejected": -280.6612854003906,
32
  "loss": 0.3323,
33
+ "rewards/accuracies": 0.4444444477558136,
34
+ "rewards/chosen": 0.0006422583828680217,
35
+ "rewards/margins": 0.0004346017958596349,
36
+ "rewards/rejected": 0.00020765644148923457,
37
  "step": 10
38
  },
39
  {
40
  "epoch": 0.06,
41
+ "learning_rate": 2.8169014084507043e-07,
42
+ "logits/chosen": -0.09473811089992523,
43
+ "logits/rejected": -0.056791383773088455,
44
+ "logps/chosen": -331.81280517578125,
45
+ "logps/rejected": -241.23910522460938,
46
+ "loss": 0.3272,
47
+ "rewards/accuracies": 0.5562499761581421,
48
+ "rewards/chosen": 0.0002513823565095663,
49
+ "rewards/margins": 0.0021114428527653217,
50
+ "rewards/rejected": -0.001860060729086399,
51
  "step": 20
52
  },
53
  {
54
  "epoch": 0.08,
55
+ "learning_rate": 4.225352112676056e-07,
56
+ "logits/chosen": -0.05887429043650627,
57
+ "logits/rejected": -0.0021190904080867767,
58
+ "logps/chosen": -387.2514953613281,
59
+ "logps/rejected": -252.19833374023438,
60
+ "loss": 0.3291,
61
+ "rewards/accuracies": 0.612500011920929,
62
+ "rewards/chosen": 0.004256925545632839,
63
+ "rewards/margins": 0.018127940595149994,
64
+ "rewards/rejected": -0.01387101411819458,
65
  "step": 30
66
  },
67
  {
68
  "epoch": 0.11,
69
+ "learning_rate": 5.633802816901409e-07,
70
+ "logits/chosen": -0.07777061313390732,
71
+ "logits/rejected": -0.023820143193006516,
72
+ "logps/chosen": -335.6570129394531,
73
+ "logps/rejected": -263.23907470703125,
74
+ "loss": 0.3421,
75
+ "rewards/accuracies": 0.5375000238418579,
76
+ "rewards/chosen": -0.004789065103977919,
77
+ "rewards/margins": 0.021946147084236145,
78
+ "rewards/rejected": -0.02673521265387535,
79
  "step": 40
80
  },
81
  {
82
  "epoch": 0.14,
83
+ "learning_rate": 7.04225352112676e-07,
84
+ "logits/chosen": -0.03277156129479408,
85
+ "logits/rejected": 0.012101896107196808,
86
+ "logps/chosen": -336.37176513671875,
87
+ "logps/rejected": -272.0014953613281,
88
+ "loss": 0.358,
89
+ "rewards/accuracies": 0.59375,
90
+ "rewards/chosen": -0.026565855368971825,
91
+ "rewards/margins": 0.06764300167560577,
92
+ "rewards/rejected": -0.09420885145664215,
93
  "step": 50
94
  },
95
  {
96
  "epoch": 0.17,
97
+ "learning_rate": 8.450704225352112e-07,
98
+ "logits/chosen": -0.11417659372091293,
99
+ "logits/rejected": -0.05113930255174637,
100
+ "logps/chosen": -427.59600830078125,
101
+ "logps/rejected": -268.9815673828125,
102
+ "loss": 0.3784,
103
  "rewards/accuracies": 0.6875,
104
+ "rewards/chosen": -0.052729975432157516,
105
+ "rewards/margins": 0.18380871415138245,
106
+ "rewards/rejected": -0.23653869330883026,
107
  "step": 60
108
  },
109
  {
110
  "epoch": 0.2,
111
+ "learning_rate": 9.859154929577465e-07,
112
+ "logits/chosen": -0.12808646261692047,
113
+ "logits/rejected": -0.009413952939212322,
114
+ "logps/chosen": -417.78680419921875,
115
+ "logps/rejected": -287.1832580566406,
116
+ "loss": 0.3848,
117
+ "rewards/accuracies": 0.5874999761581421,
118
+ "rewards/chosen": -0.10650447756052017,
119
+ "rewards/margins": 0.15690955519676208,
120
+ "rewards/rejected": -0.26341402530670166,
121
  "step": 70
122
  },
123
  {
124
  "epoch": 0.22,
125
+ "learning_rate": 9.995106132599868e-07,
126
+ "logits/chosen": -0.0769750103354454,
127
+ "logits/rejected": -0.06951722502708435,
128
+ "logps/chosen": -294.3128662109375,
129
+ "logps/rejected": -227.47463989257812,
130
+ "loss": 0.3948,
131
+ "rewards/accuracies": 0.606249988079071,
132
+ "rewards/chosen": -0.06992264091968536,
133
+ "rewards/margins": 0.16703224182128906,
134
+ "rewards/rejected": -0.23695488274097443,
135
  "step": 80
136
  },
137
  {
138
  "epoch": 0.25,
139
+ "learning_rate": 9.978201358980644e-07,
140
+ "logits/chosen": -0.2559957504272461,
141
+ "logits/rejected": -0.20053008198738098,
142
+ "logps/chosen": -389.33929443359375,
143
+ "logps/rejected": -247.1188201904297,
144
+ "loss": 0.4103,
145
  "rewards/accuracies": 0.7124999761581421,
146
+ "rewards/chosen": 0.11513955891132355,
147
+ "rewards/margins": 0.3396330177783966,
148
+ "rewards/rejected": -0.22449345886707306,
149
  "step": 90
150
  },
151
  {
152
  "epoch": 0.28,
153
+ "learning_rate": 9.949266103908894e-07,
154
+ "logits/chosen": -0.18818387389183044,
155
+ "logits/rejected": -0.16776612401008606,
156
+ "logps/chosen": -360.2618103027344,
157
+ "logps/rejected": -283.3966979980469,
158
+ "loss": 0.4011,
159
+ "rewards/accuracies": 0.6312500238418579,
160
+ "rewards/chosen": 0.07560204714536667,
161
+ "rewards/margins": 0.13260915875434875,
162
+ "rewards/rejected": -0.05700710415840149,
163
  "step": 100
164
  },
165
  {
166
  "epoch": 0.31,
167
+ "learning_rate": 9.908370293252287e-07,
168
+ "logits/chosen": -0.2396635115146637,
169
+ "logits/rejected": -0.16480541229248047,
170
+ "logps/chosen": -319.79925537109375,
171
+ "logps/rejected": -279.1387023925781,
172
+ "loss": 0.3716,
173
+ "rewards/accuracies": 0.5375000238418579,
174
+ "rewards/chosen": 0.08118756860494614,
175
+ "rewards/margins": 0.0795254111289978,
176
+ "rewards/rejected": 0.0016621515387669206,
177
  "step": 110
178
  },
179
  {
180
  "epoch": 0.34,
181
+ "learning_rate": 9.855612757141654e-07,
182
+ "logits/chosen": -0.2065432071685791,
183
+ "logits/rejected": -0.17942771315574646,
184
+ "logps/chosen": -320.2702941894531,
185
+ "logps/rejected": -222.2958221435547,
186
+ "loss": 0.3645,
187
  "rewards/accuracies": 0.6312500238418579,
188
+ "rewards/chosen": 0.12114707380533218,
189
+ "rewards/margins": 0.17006908357143402,
190
+ "rewards/rejected": -0.04892200976610184,
191
  "step": 120
192
  },
193
  {
194
  "epoch": 0.37,
195
+ "learning_rate": 9.791120991134902e-07,
196
+ "logits/chosen": -0.162322998046875,
197
+ "logits/rejected": -0.08893263339996338,
198
+ "logps/chosen": -376.00360107421875,
199
+ "logps/rejected": -244.6248016357422,
200
+ "loss": 0.3721,
201
+ "rewards/accuracies": 0.637499988079071,
202
+ "rewards/chosen": 0.10144136846065521,
203
+ "rewards/margins": 0.25357523560523987,
204
+ "rewards/rejected": -0.15213388204574585,
205
  "step": 130
206
  },
207
  {
208
  "epoch": 0.39,
209
+ "learning_rate": 9.715050848107168e-07,
210
+ "logits/chosen": -0.14301994442939758,
211
+ "logits/rejected": -0.13107290863990784,
212
+ "logps/chosen": -337.5934753417969,
213
+ "logps/rejected": -264.13153076171875,
214
+ "loss": 0.3693,
215
+ "rewards/accuracies": 0.550000011920929,
216
+ "rewards/chosen": -0.008726155385375023,
217
+ "rewards/margins": 0.13979385793209076,
218
+ "rewards/rejected": -0.14851999282836914,
219
  "step": 140
220
  },
221
  {
222
  "epoch": 0.42,
223
+ "learning_rate": 9.627586161611731e-07,
224
+ "logits/chosen": -0.25995808839797974,
225
+ "logits/rejected": -0.22309823334217072,
226
+ "logps/chosen": -305.5233459472656,
227
+ "logps/rejected": -264.06097412109375,
228
+ "loss": 0.387,
229
+ "rewards/accuracies": 0.612500011920929,
230
+ "rewards/chosen": -0.004295418970286846,
231
+ "rewards/margins": 0.1561904400587082,
232
+ "rewards/rejected": -0.16048584878444672,
233
  "step": 150
234
  },
235
  {
236
  "epoch": 0.45,
237
+ "learning_rate": 9.528938301621955e-07,
238
+ "logits/chosen": -0.17277738451957703,
239
+ "logits/rejected": -0.054037339985370636,
240
+ "logps/chosen": -388.6319885253906,
241
+ "logps/rejected": -281.8338317871094,
242
+ "loss": 0.3624,
243
+ "rewards/accuracies": 0.581250011920929,
244
+ "rewards/chosen": -0.027996420860290527,
245
+ "rewards/margins": 0.2142985761165619,
246
+ "rewards/rejected": -0.2422950267791748,
247
  "step": 160
248
  },
249
  {
250
  "epoch": 0.48,
251
+ "learning_rate": 9.419345663727804e-07,
252
+ "logits/chosen": 0.1668136566877365,
253
+ "logits/rejected": 0.24267610907554626,
254
+ "logps/chosen": -348.5519104003906,
255
+ "logps/rejected": -298.996826171875,
256
+ "loss": 0.3338,
257
+ "rewards/accuracies": 0.59375,
258
+ "rewards/chosen": -0.16783149540424347,
259
+ "rewards/margins": 0.18178348243236542,
260
+ "rewards/rejected": -0.3496149778366089,
261
  "step": 170
262
  },
263
  {
264
  "epoch": 0.51,
265
+ "learning_rate": 9.299073093021404e-07,
266
+ "logits/chosen": 0.047697070986032486,
267
+ "logits/rejected": 0.1468953937292099,
268
+ "logps/chosen": -363.13934326171875,
269
+ "logps/rejected": -292.49822998046875,
270
+ "loss": 0.3368,
271
+ "rewards/accuracies": 0.606249988079071,
272
+ "rewards/chosen": -0.21875329315662384,
273
+ "rewards/margins": 0.1854187548160553,
274
+ "rewards/rejected": -0.40417203307151794,
275
  "step": 180
276
  },
277
  {
278
  "epoch": 0.53,
279
+ "learning_rate": 9.168411244063861e-07,
280
+ "logits/chosen": 0.016937877982854843,
281
+ "logits/rejected": 0.10573382675647736,
282
+ "logps/chosen": -369.8836975097656,
283
+ "logps/rejected": -270.32574462890625,
284
+ "loss": 0.3274,
285
  "rewards/accuracies": 0.643750011920929,
286
+ "rewards/chosen": -0.134426087141037,
287
+ "rewards/margins": 0.2750268578529358,
288
+ "rewards/rejected": -0.40945297479629517,
289
  "step": 190
290
  },
291
  {
292
  "epoch": 0.56,
293
+ "learning_rate": 9.02767587848013e-07,
294
+ "logits/chosen": -0.031068842858076096,
295
+ "logits/rejected": 0.08885425329208374,
296
+ "logps/chosen": -358.65374755859375,
297
+ "logps/rejected": -290.309814453125,
298
+ "loss": 0.3447,
299
+ "rewards/accuracies": 0.5625,
300
+ "rewards/chosen": -0.28612247109413147,
301
+ "rewards/margins": 0.17394599318504333,
302
+ "rewards/rejected": -0.4600685238838196,
303
  "step": 200
304
  },
305
  {
306
  "epoch": 0.59,
307
+ "learning_rate": 8.877207101879301e-07,
308
+ "logits/chosen": 0.1502365916967392,
309
+ "logits/rejected": 0.24661250412464142,
310
+ "logps/chosen": -428.3617248535156,
311
+ "logps/rejected": -313.82257080078125,
312
+ "loss": 0.3299,
313
+ "rewards/accuracies": 0.5687500238418579,
314
+ "rewards/chosen": -0.25656384229660034,
315
+ "rewards/margins": 0.19923393428325653,
316
+ "rewards/rejected": -0.4557977616786957,
317
  "step": 210
318
  },
319
  {
320
  "epoch": 0.62,
321
+ "learning_rate": 8.717368541944452e-07,
322
+ "logits/chosen": 0.08803655207157135,
323
+ "logits/rejected": 0.1639718860387802,
324
+ "logps/chosen": -423.03228759765625,
325
+ "logps/rejected": -313.6676025390625,
326
+ "loss": 0.3218,
327
+ "rewards/accuracies": 0.637499988079071,
328
+ "rewards/chosen": -0.3232230246067047,
329
+ "rewards/margins": 0.1633337289094925,
330
+ "rewards/rejected": -0.4865567088127136,
331
  "step": 220
332
  },
333
  {
334
  "epoch": 0.65,
335
+ "learning_rate": 8.54854646967831e-07,
336
+ "logits/chosen": -0.0030194534920156,
337
+ "logits/rejected": 0.14094045758247375,
338
+ "logps/chosen": -369.0701904296875,
339
+ "logps/rejected": -286.33990478515625,
340
+ "loss": 0.3382,
341
  "rewards/accuracies": 0.612500011920929,
342
+ "rewards/chosen": -0.18699464201927185,
343
+ "rewards/margins": 0.23964881896972656,
344
+ "rewards/rejected": -0.4266434609889984,
345
  "step": 230
346
  },
347
  {
348
  "epoch": 0.67,
349
+ "learning_rate": 8.371148865928318e-07,
350
+ "logits/chosen": -0.2055165320634842,
351
+ "logits/rejected": -0.08643672615289688,
352
+ "logps/chosen": -377.8648986816406,
353
+ "logps/rejected": -279.52703857421875,
354
+ "loss": 0.3443,
355
+ "rewards/accuracies": 0.6812499761581421,
356
+ "rewards/chosen": -0.19047299027442932,
357
+ "rewards/margins": 0.21315133571624756,
358
+ "rewards/rejected": -0.40362435579299927,
359
  "step": 240
360
  },
361
  {
362
  "epoch": 0.7,
363
+ "learning_rate": 8.185604435447001e-07,
364
+ "logits/chosen": -0.21048621833324432,
365
+ "logits/rejected": -0.12320221960544586,
366
+ "logps/chosen": -396.4952392578125,
367
+ "logps/rejected": -284.85693359375,
368
+ "loss": 0.3476,
369
+ "rewards/accuracies": 0.625,
370
+ "rewards/chosen": -0.14602257311344147,
371
+ "rewards/margins": 0.29507723450660706,
372
+ "rewards/rejected": -0.4410998225212097,
373
  "step": 250
374
  },
375
  {
376
  "epoch": 0.73,
377
+ "learning_rate": 7.992361570870287e-07,
378
+ "logits/chosen": -0.09357413649559021,
379
+ "logits/rejected": -0.02147941291332245,
380
+ "logps/chosen": -391.3370666503906,
381
+ "logps/rejected": -304.73248291015625,
382
+ "loss": 0.3457,
383
+ "rewards/accuracies": 0.6625000238418579,
384
+ "rewards/chosen": -0.30016764998435974,
385
+ "rewards/margins": 0.2906726002693176,
386
+ "rewards/rejected": -0.590840220451355,
387
  "step": 260
388
  },
389
  {
390
  "epoch": 0.76,
391
+ "learning_rate": 7.791887269117441e-07,
392
+ "logits/chosen": -0.01912299357354641,
393
+ "logits/rejected": 0.015499010682106018,
394
+ "logps/chosen": -385.39788818359375,
395
+ "logps/rejected": -318.03045654296875,
396
+ "loss": 0.3332,
397
+ "rewards/accuracies": 0.6000000238418579,
398
+ "rewards/chosen": -0.46580857038497925,
399
+ "rewards/margins": 0.2723786234855652,
400
+ "rewards/rejected": -0.7381871938705444,
401
  "step": 270
402
  },
403
  {
404
  "epoch": 0.79,
405
+ "learning_rate": 7.584666002831294e-07,
406
+ "logits/chosen": -0.1838226318359375,
407
+ "logits/rejected": -0.11540188640356064,
408
+ "logps/chosen": -431.819580078125,
409
+ "logps/rejected": -314.17315673828125,
410
+ "loss": 0.3741,
411
+ "rewards/accuracies": 0.65625,
412
+ "rewards/chosen": -0.38983967900276184,
413
+ "rewards/margins": 0.3283058702945709,
414
+ "rewards/rejected": -0.718145489692688,
415
  "step": 280
416
  },
417
  {
418
  "epoch": 0.81,
419
+ "learning_rate": 7.37119854958609e-07,
420
+ "logits/chosen": -0.16166582703590393,
421
+ "logits/rejected": -0.06856708228588104,
422
+ "logps/chosen": -389.5946960449219,
423
+ "logps/rejected": -316.12677001953125,
424
+ "loss": 0.3638,
425
+ "rewards/accuracies": 0.6000000238418579,
426
+ "rewards/chosen": -0.3782878816127777,
427
+ "rewards/margins": 0.20532508194446564,
428
+ "rewards/rejected": -0.5836129188537598,
429
  "step": 290
430
  },
431
  {
432
  "epoch": 0.84,
433
+ "learning_rate": 7.152000781692285e-07,
434
+ "logits/chosen": -0.15225966274738312,
435
+ "logits/rejected": 0.023759862408041954,
436
+ "logps/chosen": -398.35040283203125,
437
+ "logps/rejected": -296.66046142578125,
438
+ "loss": 0.3393,
439
+ "rewards/accuracies": 0.606249988079071,
440
+ "rewards/chosen": -0.21167917549610138,
441
+ "rewards/margins": 0.28109148144721985,
442
+ "rewards/rejected": -0.4927706718444824,
443
  "step": 300
444
  },
445
  {
446
  "epoch": 0.87,
447
+ "learning_rate": 6.927602419522946e-07,
448
+ "logits/chosen": -0.1469275802373886,
449
+ "logits/rejected": 0.009778399951756,
450
+ "logps/chosen": -370.21051025390625,
451
+ "logps/rejected": -298.3064880371094,
452
+ "loss": 0.3295,
453
+ "rewards/accuracies": 0.6499999761581421,
454
+ "rewards/chosen": -0.29429182410240173,
455
+ "rewards/margins": 0.266606867313385,
456
+ "rewards/rejected": -0.5608987212181091,
457
  "step": 310
458
  },
459
  {
460
  "epoch": 0.9,
461
+ "learning_rate": 6.698545751374463e-07,
462
+ "logits/chosen": -0.008808034472167492,
463
+ "logits/rejected": 0.06853709369897842,
464
+ "logps/chosen": -371.8343200683594,
465
+ "logps/rejected": -314.5874328613281,
466
+ "loss": 0.3437,
467
+ "rewards/accuracies": 0.59375,
468
+ "rewards/chosen": -0.3911176323890686,
469
+ "rewards/margins": 0.1818263977766037,
470
+ "rewards/rejected": -0.5729440450668335,
471
  "step": 320
472
  },
473
  {
474
  "epoch": 0.93,
475
+ "learning_rate": 6.465384322955224e-07,
476
+ "logits/chosen": -0.17208532989025116,
477
+ "logits/rejected": -0.05994819477200508,
478
+ "logps/chosen": -450.02850341796875,
479
+ "logps/rejected": -345.80419921875,
480
+ "loss": 0.3342,
481
+ "rewards/accuracies": 0.59375,
482
+ "rewards/chosen": -0.3475356698036194,
483
+ "rewards/margins": 0.32027798891067505,
484
+ "rewards/rejected": -0.6678135991096497,
485
  "step": 330
486
  },
487
  {
488
  "epoch": 0.96,
489
+ "learning_rate": 6.228681599669248e-07,
490
+ "logits/chosen": 0.0015781521797180176,
491
+ "logits/rejected": 0.043049950152635574,
492
+ "logps/chosen": -387.4454345703125,
493
+ "logps/rejected": -302.407958984375,
494
+ "loss": 0.3188,
495
+ "rewards/accuracies": 0.6000000238418579,
496
+ "rewards/chosen": -0.3666529059410095,
497
+ "rewards/margins": 0.25499850511550903,
498
+ "rewards/rejected": -0.6216514110565186,
499
  "step": 340
500
  },
501
  {
502
  "epoch": 0.98,
503
+ "learning_rate": 5.989009604927586e-07,
504
+ "logits/chosen": -0.03641257435083389,
505
+ "logits/rejected": 0.037856753915548325,
506
+ "logps/chosen": -401.0362854003906,
507
+ "logps/rejected": -317.9659729003906,
508
+ "loss": 0.3231,
509
+ "rewards/accuracies": 0.6875,
510
+ "rewards/chosen": -0.3948978781700134,
511
+ "rewards/margins": 0.2911498546600342,
512
+ "rewards/rejected": -0.6860476732254028,
513
  "step": 350
514
  },
515
  {
516
  "epoch": 1.01,
517
+ "learning_rate": 5.74694753777815e-07,
518
+ "logits/chosen": -0.12373703718185425,
519
+ "logits/rejected": -0.014805403538048267,
520
+ "logps/chosen": -356.97100830078125,
521
+ "logps/rejected": -344.86102294921875,
522
+ "loss": 0.3052,
523
+ "rewards/accuracies": 0.643750011920929,
524
+ "rewards/chosen": -0.41758236289024353,
525
+ "rewards/margins": 0.3036467730998993,
526
+ "rewards/rejected": -0.721229076385498,
527
  "step": 360
528
  },
529
  {
530
  "epoch": 1.04,
531
+ "learning_rate": 5.503080373194666e-07,
532
+ "logits/chosen": -0.13851913809776306,
533
+ "logits/rejected": -0.07055823504924774,
534
+ "logps/chosen": -396.57769775390625,
535
+ "logps/rejected": -335.1228942871094,
536
+ "loss": 0.2391,
537
+ "rewards/accuracies": 0.7749999761581421,
538
+ "rewards/chosen": -0.39428186416625977,
539
+ "rewards/margins": 0.44222506880760193,
540
+ "rewards/rejected": -0.8365070223808289,
541
  "step": 370
542
  },
543
  {
544
  "epoch": 1.07,
545
+ "learning_rate": 5.257997448407366e-07,
546
+ "logits/chosen": -0.13537368178367615,
547
+ "logits/rejected": 0.009506871923804283,
548
+ "logps/chosen": -375.4892578125,
549
+ "logps/rejected": -355.96136474609375,
550
+ "loss": 0.232,
551
+ "rewards/accuracies": 0.7562500238418579,
552
+ "rewards/chosen": -0.5489501357078552,
553
+ "rewards/margins": 0.44577375054359436,
554
+ "rewards/rejected": -0.9947239756584167,
555
  "step": 380
556
  },
557
  {
558
  "epoch": 1.1,
559
+ "learning_rate": 5.012291038691665e-07,
560
+ "logits/chosen": -0.292557954788208,
561
+ "logits/rejected": -0.12162590026855469,
562
+ "logps/chosen": -359.124755859375,
563
+ "logps/rejected": -338.5267333984375,
564
+ "loss": 0.2175,
565
+ "rewards/accuracies": 0.71875,
566
+ "rewards/chosen": -0.7096319794654846,
567
+ "rewards/margins": 0.47353801131248474,
568
+ "rewards/rejected": -1.1831700801849365,
569
  "step": 390
570
  },
571
  {
572
  "epoch": 1.12,
573
+ "learning_rate": 4.7665549260567063e-07,
574
+ "logits/chosen": -0.24239158630371094,
575
+ "logits/rejected": -0.13934944570064545,
576
+ "logps/chosen": -426.60369873046875,
577
+ "logps/rejected": -403.0608215332031,
578
+ "loss": 0.1717,
579
+ "rewards/accuracies": 0.78125,
580
+ "rewards/chosen": -0.8361096382141113,
581
+ "rewards/margins": 0.5920412540435791,
582
+ "rewards/rejected": -1.4281507730484009,
583
  "step": 400
584
  },
585
  {
586
  "epoch": 1.15,
587
+ "learning_rate": 4.521382964292663e-07,
588
+ "logits/chosen": -0.040043365210294724,
589
+ "logits/rejected": -0.01921779289841652,
590
+ "logps/chosen": -417.27215576171875,
591
+ "logps/rejected": -388.7860412597656,
592
+ "loss": 0.1755,
593
+ "rewards/accuracies": 0.7749999761581421,
594
+ "rewards/chosen": -0.9553691744804382,
595
+ "rewards/margins": 0.6799647212028503,
596
+ "rewards/rejected": -1.6353340148925781,
597
  "step": 410
598
  },
599
  {
600
  "epoch": 1.18,
601
+ "learning_rate": 4.277367643844574e-07,
602
+ "logits/chosen": -0.097591333091259,
603
+ "logits/rejected": 0.017194192856550217,
604
+ "logps/chosen": -437.2616271972656,
605
+ "logps/rejected": -411.5907287597656,
606
+ "loss": 0.1823,
607
+ "rewards/accuracies": 0.7250000238418579,
608
+ "rewards/chosen": -1.0141613483428955,
609
+ "rewards/margins": 0.5529431700706482,
610
+ "rewards/rejected": -1.567104458808899,
611
  "step": 420
612
  },
613
  {
614
  "epoch": 1.21,
615
+ "learning_rate": 4.035098659980891e-07,
616
+ "logits/chosen": 0.12445954233407974,
617
+ "logits/rejected": 0.16896803677082062,
618
+ "logps/chosen": -484.2261657714844,
619
+ "logps/rejected": -403.1665344238281,
620
+ "loss": 0.1577,
621
+ "rewards/accuracies": 0.8125,
622
+ "rewards/chosen": -0.9584755897521973,
623
+ "rewards/margins": 0.7136095762252808,
624
+ "rewards/rejected": -1.672085165977478,
625
  "step": 430
626
  },
627
  {
628
  "epoch": 1.24,
629
+ "learning_rate": 3.795161487716928e-07,
630
+ "logits/chosen": 0.10622860491275787,
631
+ "logits/rejected": 0.20460453629493713,
632
+ "logps/chosen": -416.31878662109375,
633
+ "logps/rejected": -390.43353271484375,
634
+ "loss": 0.1601,
635
+ "rewards/accuracies": 0.731249988079071,
636
+ "rewards/chosen": -1.0357643365859985,
637
+ "rewards/margins": 0.6402472257614136,
638
+ "rewards/rejected": -1.6760118007659912,
639
  "step": 440
640
  },
641
  {
642
  "epoch": 1.26,
643
+ "learning_rate": 3.5581359669371223e-07,
644
+ "logits/chosen": 0.04869859293103218,
645
+ "logits/rejected": 0.1988603174686432,
646
+ "logps/chosen": -459.97161865234375,
647
+ "logps/rejected": -406.51129150390625,
648
+ "loss": 0.1485,
649
  "rewards/accuracies": 0.8062499761581421,
650
+ "rewards/chosen": -0.9077569246292114,
651
+ "rewards/margins": 0.6647528409957886,
652
+ "rewards/rejected": -1.572509527206421,
653
  "step": 450
654
  },
655
  {
656
  "epoch": 1.29,
657
+ "learning_rate": 3.324594901135326e-07,
658
+ "logits/chosen": 0.09600436687469482,
659
+ "logits/rejected": 0.2769695520401001,
660
+ "logps/chosen": -412.62481689453125,
661
+ "logps/rejected": -427.94573974609375,
662
+ "loss": 0.1594,
663
+ "rewards/accuracies": 0.762499988079071,
664
+ "rewards/chosen": -0.8976230621337891,
665
+ "rewards/margins": 0.6307999491691589,
666
+ "rewards/rejected": -1.5284230709075928,
667
  "step": 460
668
  },
669
  {
670
  "epoch": 1.32,
671
+ "learning_rate": 3.095102673159463e-07,
672
+ "logits/chosen": -0.04745341092348099,
673
+ "logits/rejected": 0.04035920277237892,
674
+ "logps/chosen": -448.31304931640625,
675
+ "logps/rejected": -434.83026123046875,
676
+ "loss": 0.162,
677
+ "rewards/accuracies": 0.800000011920929,
678
+ "rewards/chosen": -1.009493112564087,
679
+ "rewards/margins": 0.8154042363166809,
680
+ "rewards/rejected": -1.8248971700668335,
681
  "step": 470
682
  },
683
  {
684
  "epoch": 1.35,
685
+ "learning_rate": 2.870213881305802e-07,
686
+ "logits/chosen": 0.003581273602321744,
687
+ "logits/rejected": 0.15165765583515167,
688
+ "logps/chosen": -442.5594177246094,
689
+ "logps/rejected": -411.17144775390625,
690
+ "loss": 0.1555,
691
+ "rewards/accuracies": 0.800000011920929,
692
+ "rewards/chosen": -0.9550750851631165,
693
+ "rewards/margins": 0.7856963872909546,
694
+ "rewards/rejected": -1.7407715320587158,
695
  "step": 480
696
  },
697
  {
698
  "epoch": 1.38,
699
+ "learning_rate": 2.6504719990588745e-07,
700
+ "logits/chosen": -0.00950574316084385,
701
+ "logits/rejected": 0.15859754383563995,
702
+ "logps/chosen": -484.21954345703125,
703
+ "logps/rejected": -441.0923767089844,
704
+ "loss": 0.1608,
705
+ "rewards/accuracies": 0.78125,
706
+ "rewards/chosen": -1.0694897174835205,
707
+ "rewards/margins": 0.7761608362197876,
708
+ "rewards/rejected": -1.8456504344940186,
709
  "step": 490
710
  },
711
  {
712
  "epoch": 1.4,
713
+ "learning_rate": 2.436408061715988e-07,
714
+ "logits/chosen": -0.03855917602777481,
715
+ "logits/rejected": 0.09556344896554947,
716
+ "logps/chosen": -491.30523681640625,
717
+ "logps/rejected": -452.349609375,
718
+ "loss": 0.1511,
719
+ "rewards/accuracies": 0.8125,
720
+ "rewards/chosen": -1.0780155658721924,
721
+ "rewards/margins": 0.823270320892334,
722
+ "rewards/rejected": -1.9012857675552368,
723
  "step": 500
724
  },
725
  {
726
  "epoch": 1.43,
727
+ "learning_rate": 2.22853938307025e-07,
728
+ "logits/chosen": 0.043701499700546265,
729
+ "logits/rejected": 0.22214682400226593,
730
+ "logps/chosen": -456.72662353515625,
731
+ "logps/rejected": -426.0232849121094,
732
+ "loss": 0.1501,
733
+ "rewards/accuracies": 0.8374999761581421,
734
+ "rewards/chosen": -0.9664266705513,
735
+ "rewards/margins": 0.6386443376541138,
736
+ "rewards/rejected": -1.6050708293914795,
737
  "step": 510
738
  },
739
  {
740
  "epoch": 1.46,
741
+ "learning_rate": 2.0273683052534173e-07,
742
+ "logits/chosen": 0.0689583569765091,
743
+ "logits/rejected": 0.1506797820329666,
744
+ "logps/chosen": -445.80133056640625,
745
+ "logps/rejected": -414.3836975097656,
746
+ "loss": 0.1396,
747
+ "rewards/accuracies": 0.824999988079071,
748
+ "rewards/chosen": -1.0923160314559937,
749
+ "rewards/margins": 0.7477865219116211,
750
+ "rewards/rejected": -1.8401025533676147,
751
  "step": 520
752
  },
753
  {
754
  "epoch": 1.49,
755
+ "learning_rate": 1.833380984759764e-07,
756
+ "logits/chosen": 0.10293974727392197,
757
+ "logits/rejected": 0.26486995816230774,
758
+ "logps/chosen": -436.48907470703125,
759
+ "logps/rejected": -422.7872009277344,
760
+ "loss": 0.1299,
761
+ "rewards/accuracies": 0.762499988079071,
762
+ "rewards/chosen": -1.0291588306427002,
763
+ "rewards/margins": 0.720792829990387,
764
+ "rewards/rejected": -1.749951720237732,
765
  "step": 530
766
  },
767
  {
768
  "epoch": 1.52,
769
+ "learning_rate": 1.6470462175846606e-07,
770
+ "logits/chosen": 0.07386808097362518,
771
+ "logits/rejected": 0.2429957389831543,
772
+ "logps/chosen": -517.3760986328125,
773
+ "logps/rejected": -460.2138671875,
774
+ "loss": 0.1413,
775
+ "rewards/accuracies": 0.8187500238418579,
776
+ "rewards/chosen": -1.3018550872802734,
777
+ "rewards/margins": 0.7603974342346191,
778
+ "rewards/rejected": -2.0622525215148926,
779
  "step": 540
780
  },
781
  {
782
  "epoch": 1.55,
783
+ "learning_rate": 1.468814306317092e-07,
784
+ "logits/chosen": 0.11437557637691498,
785
+ "logits/rejected": 0.2697022557258606,
786
+ "logps/chosen": -451.03326416015625,
787
+ "logps/rejected": -420.84100341796875,
788
+ "loss": 0.1376,
789
+ "rewards/accuracies": 0.793749988079071,
790
+ "rewards/chosen": -1.2259981632232666,
791
+ "rewards/margins": 0.9535650014877319,
792
+ "rewards/rejected": -2.179563045501709,
793
  "step": 550
794
  },
795
  {
796
  "epoch": 1.57,
797
+ "learning_rate": 1.299115971923958e-07,
798
+ "logits/chosen": 0.07330118119716644,
799
+ "logits/rejected": 0.24497468769550323,
800
+ "logps/chosen": -482.2550354003906,
801
+ "logps/rejected": -460.70367431640625,
802
+ "loss": 0.1354,
803
+ "rewards/accuracies": 0.762499988079071,
804
+ "rewards/chosen": -1.2959102392196655,
805
+ "rewards/margins": 0.8086190223693848,
806
+ "rewards/rejected": -2.10452938079834,
807
  "step": 560
808
  },
809
  {
810
  "epoch": 1.6,
811
+ "learning_rate": 1.1383613128559305e-07,
812
+ "logits/chosen": 0.15279309451580048,
813
+ "logits/rejected": 0.3158939480781555,
814
+ "logps/chosen": -485.0721740722656,
815
+ "logps/rejected": -450.25799560546875,
816
+ "loss": 0.1225,
817
+ "rewards/accuracies": 0.8062499761581421,
818
+ "rewards/chosen": -1.0827850103378296,
819
+ "rewards/margins": 0.839277446269989,
820
+ "rewards/rejected": -1.9220625162124634,
821
  "step": 570
822
  },
823
  {
824
  "epoch": 1.63,
825
+ "learning_rate": 9.869388139903495e-08,
826
+ "logits/chosen": 0.09564773738384247,
827
+ "logits/rejected": 0.150350421667099,
828
+ "logps/chosen": -427.63616943359375,
829
+ "logps/rejected": -459.97064208984375,
830
+ "loss": 0.1297,
831
+ "rewards/accuracies": 0.793749988079071,
832
+ "rewards/chosen": -1.125893473625183,
833
+ "rewards/margins": 0.7602401971817017,
834
+ "rewards/rejected": -1.8861335515975952,
835
  "step": 580
836
  },
837
  {
838
  "epoch": 1.66,
839
+ "learning_rate": 8.452144078061818e-08,
840
+ "logits/chosen": 0.08417952060699463,
841
+ "logits/rejected": 0.249791219830513,
842
+ "logps/chosen": -476.9566345214844,
843
+ "logps/rejected": -474.86944580078125,
844
+ "loss": 0.1333,
845
+ "rewards/accuracies": 0.8125,
846
+ "rewards/chosen": -1.174066185951233,
847
+ "rewards/margins": 0.9380697011947632,
848
+ "rewards/rejected": -2.112135648727417,
849
  "step": 590
850
  },
851
  {
852
  "epoch": 1.69,
853
+ "learning_rate": 7.135305900598321e-08,
854
+ "logits/chosen": 0.11366138607263565,
855
+ "logits/rejected": 0.24674567580223083,
856
+ "logps/chosen": -484.5184631347656,
857
+ "logps/rejected": -416.90863037109375,
858
+ "loss": 0.1273,
859
+ "rewards/accuracies": 0.75,
860
+ "rewards/chosen": -1.1246851682662964,
861
+ "rewards/margins": 0.781345546245575,
862
+ "rewards/rejected": -1.9060306549072266,
863
  "step": 600
864
  },
865
  {
866
  "epoch": 1.71,
867
+ "learning_rate": 5.9220559209888166e-08,
868
+ "logits/chosen": 0.10064254701137543,
869
+ "logits/rejected": 0.16818474233150482,
870
+ "logps/chosen": -491.97662353515625,
871
+ "logps/rejected": -483.4784240722656,
872
+ "loss": 0.1357,
873
+ "rewards/accuracies": 0.7562500238418579,
874
+ "rewards/chosen": -1.2799218893051147,
875
+ "rewards/margins": 0.9080696105957031,
876
+ "rewards/rejected": -2.1879916191101074,
877
  "step": 610
878
  },
879
  {
880
  "epoch": 1.74,
881
+ "learning_rate": 4.815326118139812e-08,
882
+ "logits/chosen": 0.08698529005050659,
883
+ "logits/rejected": 0.18599075078964233,
884
+ "logps/chosen": -458.9814453125,
885
+ "logps/rejected": -416.03424072265625,
886
+ "loss": 0.1329,
887
+ "rewards/accuracies": 0.8125,
888
+ "rewards/chosen": -1.0884288549423218,
889
+ "rewards/margins": 0.8256447911262512,
890
+ "rewards/rejected": -1.9140735864639282,
891
  "step": 620
892
  },
893
  {
894
  "epoch": 1.77,
895
+ "learning_rate": 3.81779105087407e-08,
896
+ "logits/chosen": 0.028252029791474342,
897
+ "logits/rejected": 0.24228385090827942,
898
+ "logps/chosen": -447.22283935546875,
899
+ "logps/rejected": -428.1376953125,
900
+ "loss": 0.1349,
901
+ "rewards/accuracies": 0.7749999761581421,
902
+ "rewards/chosen": -1.2487438917160034,
903
+ "rewards/margins": 0.7152423858642578,
904
+ "rewards/rejected": -1.9639861583709717,
905
  "step": 630
906
  },
907
  {
908
  "epoch": 1.8,
909
+ "learning_rate": 2.9318613945057637e-08,
910
+ "logits/chosen": 0.03168308362364769,
911
+ "logits/rejected": 0.14438685774803162,
912
+ "logps/chosen": -530.6989135742188,
913
+ "logps/rejected": -481.19317626953125,
914
+ "loss": 0.1283,
915
+ "rewards/accuracies": 0.8374999761581421,
916
+ "rewards/chosen": -1.2953087091445923,
917
+ "rewards/margins": 0.9182836413383484,
918
+ "rewards/rejected": -2.213592529296875,
919
  "step": 640
920
  },
921
  {
922
  "epoch": 1.83,
923
+ "learning_rate": 2.1596781151249523e-08,
924
+ "logits/chosen": 0.15784808993339539,
925
+ "logits/rejected": 0.28222331404685974,
926
+ "logps/chosen": -461.800048828125,
927
+ "logps/rejected": -412.47320556640625,
928
+ "loss": 0.136,
929
+ "rewards/accuracies": 0.824999988079071,
930
+ "rewards/chosen": -1.0977789163589478,
931
+ "rewards/margins": 0.8101053237915039,
932
+ "rewards/rejected": -1.9078842401504517,
933
  "step": 650
934
  },
935
  {
936
  "epoch": 1.85,
937
+ "learning_rate": 1.5031072956701695e-08,
938
+ "logits/chosen": 0.04679325968027115,
939
+ "logits/rejected": 0.1709231436252594,
940
+ "logps/chosen": -501.66656494140625,
941
+ "logps/rejected": -491.91925048828125,
942
+ "loss": 0.1335,
943
+ "rewards/accuracies": 0.824999988079071,
944
+ "rewards/chosen": -1.1601709127426147,
945
+ "rewards/margins": 1.024567723274231,
946
+ "rewards/rejected": -2.1847386360168457,
947
  "step": 660
948
  },
949
  {
950
  "epoch": 1.88,
951
+ "learning_rate": 9.637356262923723e-09,
952
+ "logits/chosen": 0.12124764919281006,
953
+ "logits/rejected": 0.2632105350494385,
954
+ "logps/chosen": -449.41064453125,
955
+ "logps/rejected": -440.3700256347656,
956
+ "loss": 0.125,
957
+ "rewards/accuracies": 0.7875000238418579,
958
+ "rewards/chosen": -1.137796401977539,
959
+ "rewards/margins": 0.832270622253418,
960
+ "rewards/rejected": -1.970067024230957,
961
  "step": 670
962
  },
963
  {
964
  "epoch": 1.91,
965
+ "learning_rate": 5.428665699084789e-09,
966
+ "logits/chosen": -0.006133219692856073,
967
+ "logits/rejected": 0.22374041378498077,
968
+ "logps/chosen": -471.9480895996094,
969
+ "logps/rejected": -433.7080078125,
970
+ "loss": 0.1286,
971
+ "rewards/accuracies": 0.8187500238418579,
972
+ "rewards/chosen": -1.1709232330322266,
973
+ "rewards/margins": 0.926952064037323,
974
+ "rewards/rejected": -2.0978751182556152,
975
  "step": 680
976
  },
977
  {
978
  "epoch": 1.94,
979
+ "learning_rate": 2.415172122110343e-09,
980
+ "logits/chosen": -0.02816038951277733,
981
+ "logits/rejected": 0.07123323529958725,
982
+ "logps/chosen": -522.4472045898438,
983
+ "logps/rejected": -474.63934326171875,
984
+ "loss": 0.1306,
985
+ "rewards/accuracies": 0.793749988079071,
986
+ "rewards/chosen": -1.2836827039718628,
987
+ "rewards/margins": 0.8452218174934387,
988
+ "rewards/rejected": -2.1289045810699463,
989
  "step": 690
990
  },
991
  {
992
  "epoch": 1.97,
993
+ "learning_rate": 6.041580374618327e-10,
994
+ "logits/chosen": 0.06725062429904938,
995
+ "logits/rejected": 0.20876435935497284,
996
+ "logps/chosen": -496.45098876953125,
997
+ "logps/rejected": -499.33856201171875,
998
+ "loss": 0.1292,
999
+ "rewards/accuracies": 0.75,
1000
+ "rewards/chosen": -1.3299496173858643,
1001
+ "rewards/margins": 0.8479114770889282,
1002
+ "rewards/rejected": -2.177860975265503,
1003
  "step": 700
1004
  },
1005
  {
1006
  "epoch": 2.0,
1007
  "learning_rate": 0.0,
1008
+ "logits/chosen": 0.11434557288885117,
1009
+ "logits/rejected": 0.21849770843982697,
1010
+ "logps/chosen": -448.5828552246094,
1011
+ "logps/rejected": -464.2832946777344,
1012
+ "loss": 0.1326,
1013
+ "rewards/accuracies": 0.831250011920929,
1014
+ "rewards/chosen": -1.2470009326934814,
1015
+ "rewards/margins": 0.9275137782096863,
1016
+ "rewards/rejected": -2.1745147705078125,
1017
  "step": 710
1018
  },
1019
  {
1020
  "epoch": 2.0,
1021
  "step": 710,
1022
  "total_flos": 0.0,
1023
+ "train_loss": 0.25192113166124047,
1024
+ "train_runtime": 8113.4503,
1025
+ "train_samples_per_second": 11.228,
1026
+ "train_steps_per_second": 0.088
1027
  }
1028
  ],
1029
  "logging_steps": 10,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fda369e142d3189323c2d59bf379d7fe0026912ef983c553220d80f8569efb58
3
  size 6008
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c481ba85df8845bff76e10943520a1179a74d1197923c5d013b878850f576711
3
  size 6008