Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
Inference Endpoints
text-generation-inference
Model card Files Community
wzhouad commited on
Commit
445f6fa
•
1 Parent(s): 9b89140

Model save

Browse files
Files changed (8) hide show
  1. README.md +17 -1
  2. all_results.json +5 -5
  3. model-00001-of-00003.safetensors +1 -1
  4. model-00002-of-00003.safetensors +1 -1
  5. model-00003-of-00003.safetensors +1 -1
  6. train_results.json +5 -5
  7. trainer_state.json +615 -453
  8. training_args.bin +1 -1
README.md CHANGED
@@ -16,6 +16,16 @@ should probably proofread and complete it, then remove this comment. -->
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 
 
 
 
 
 
 
 
 
19
 
20
  ## Model description
21
 
@@ -37,7 +47,7 @@ The following hyperparameters were used during training:
37
  - learning_rate: 5e-07
38
  - train_batch_size: 8
39
  - eval_batch_size: 8
40
- - seed: 5
41
  - distributed_type: multi-GPU
42
  - num_devices: 8
43
  - gradient_accumulation_steps: 2
@@ -50,6 +60,12 @@ The following hyperparameters were used during training:
50
 
51
  ### Training results
52
 
 
 
 
 
 
 
53
 
54
 
55
  ### Framework versions
 
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
+ It achieves the following results on the evaluation set:
20
+ - Loss: 0.0680
21
+ - Rewards/chosen: -1.6802
22
+ - Rewards/rejected: -2.4505
23
+ - Rewards/accuracies: 0.7109
24
+ - Rewards/margins: 0.7703
25
+ - Logps/rejected: -502.4064
26
+ - Logps/chosen: -425.0607
27
+ - Logits/rejected: -2.2693
28
+ - Logits/chosen: -2.2870
29
 
30
  ## Model description
31
 
 
47
  - learning_rate: 5e-07
48
  - train_batch_size: 8
49
  - eval_batch_size: 8
50
+ - seed: 1
51
  - distributed_type: multi-GPU
52
  - num_devices: 8
53
  - gradient_accumulation_steps: 2
 
60
 
61
  ### Training results
62
 
63
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 0.1201 | 0.21 | 100 | 0.1358 | -0.5060 | -0.9026 | 0.6992 | 0.3966 | -347.6168 | -307.6421 | -2.7272 | -2.7404 |
66
+ | 0.0885 | 0.42 | 200 | 0.0939 | -0.9340 | -1.6072 | 0.7383 | 0.6732 | -418.0752 | -350.4443 | -2.5184 | -2.5309 |
67
+ | 0.0652 | 0.63 | 300 | 0.0711 | -1.5440 | -2.2912 | 0.7266 | 0.7471 | -486.4697 | -411.4413 | -2.3324 | -2.3504 |
68
+ | 0.0725 | 0.84 | 400 | 0.0680 | -1.6802 | -2.4505 | 0.7109 | 0.7703 | -502.4064 | -425.0607 | -2.2693 | -2.2870 |
69
 
70
 
71
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.08637741568647785,
4
- "train_runtime": 3241.0704,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 16.011,
7
- "train_steps_per_second": 0.125
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.10764013274444197,
4
+ "train_runtime": 3957.0601,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.449,
7
+ "train_steps_per_second": 0.121
8
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:95705cc51e5605b33982ea0df54777bcf1c925b1f1ddedae22001a4d729bf3cd
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:354828d7f1a6e78b676ee4108b8791bd101a9d6b19505bc80998de098cc88409
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:acbfb972feae803aa016bf84a68601e82ede3de4db685302a0702f4eed2610c8
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44ddf8bc3cfd335ceead18b1b7a4ce211c3b59e34accae3e5fdf3cf6260d125c
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b859840e065ab8f09e5fb0a306f50c6cfc01bfee07ca846d4556b15e6766b7b6
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21f0c108104404168a5c38fe7a54cf1c38aefe437e8fd1f288694a3d5423b20b
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.08637741568647785,
4
- "train_runtime": 3241.0704,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 16.011,
7
- "train_steps_per_second": 0.125
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.10764013274444197,
4
+ "train_runtime": 3957.0601,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.449,
7
+ "train_steps_per_second": 0.121
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.998766954377312,
5
- "eval_steps": 1000,
6
- "global_step": 405,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 1.2195121951219512e-08,
14
- "logits/chosen": -2.8695335388183594,
15
- "logits/rejected": -2.8522377014160156,
16
- "logps/chosen": -537.80126953125,
17
- "logps/rejected": -108.91968536376953,
18
- "loss": 0.3731,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -24,576 +24,738 @@
24
  },
25
  {
26
  "epoch": 0.02,
27
- "learning_rate": 1.219512195121951e-07,
28
- "logits/chosen": -2.8006839752197266,
29
- "logits/rejected": -2.7513468265533447,
30
- "logps/chosen": -339.1521911621094,
31
- "logps/rejected": -113.4110336303711,
32
- "loss": 0.3869,
33
  "rewards/accuracies": 0.5,
34
- "rewards/chosen": 0.0008597287815064192,
35
- "rewards/margins": 0.0015750755555927753,
36
- "rewards/rejected": -0.0007153467740863562,
37
  "step": 10
38
  },
39
  {
40
- "epoch": 0.05,
41
- "learning_rate": 2.439024390243902e-07,
42
- "logits/chosen": -2.8162007331848145,
43
- "logits/rejected": -2.8079230785369873,
44
- "logps/chosen": -435.24627685546875,
45
- "logps/rejected": -116.09075927734375,
46
- "loss": 0.3762,
47
- "rewards/accuracies": 0.7875000238418579,
48
- "rewards/chosen": 0.021398674696683884,
49
- "rewards/margins": 0.03878290206193924,
50
- "rewards/rejected": -0.017384223639965057,
51
  "step": 20
52
  },
53
  {
54
- "epoch": 0.07,
55
- "learning_rate": 3.6585365853658536e-07,
56
- "logits/chosen": -2.7241621017456055,
57
- "logits/rejected": -2.686893939971924,
58
- "logps/chosen": -437.8358459472656,
59
- "logps/rejected": -141.41867065429688,
60
- "loss": 0.3485,
61
- "rewards/accuracies": 0.84375,
62
- "rewards/chosen": 0.08630482107400894,
63
- "rewards/margins": 0.20522558689117432,
64
- "rewards/rejected": -0.11892076581716537,
65
  "step": 30
66
  },
67
  {
68
- "epoch": 0.1,
69
- "learning_rate": 4.878048780487804e-07,
70
- "logits/chosen": -2.5932071208953857,
71
- "logits/rejected": -2.5736117362976074,
72
- "logps/chosen": -414.60986328125,
73
- "logps/rejected": -169.4441375732422,
74
- "loss": 0.2897,
75
- "rewards/accuracies": 0.7875000238418579,
76
- "rewards/chosen": 0.01923329569399357,
77
- "rewards/margins": 0.4725423753261566,
78
- "rewards/rejected": -0.4533091187477112,
79
  "step": 40
80
  },
81
  {
82
- "epoch": 0.12,
83
- "learning_rate": 4.992461696250783e-07,
84
- "logits/chosen": -2.471208333969116,
85
- "logits/rejected": -2.458268642425537,
86
- "logps/chosen": -421.943359375,
87
- "logps/rejected": -212.53466796875,
88
- "loss": 0.2122,
89
- "rewards/accuracies": 0.7250000238418579,
90
- "rewards/chosen": -0.12829390168190002,
91
- "rewards/margins": 0.7995172739028931,
92
- "rewards/rejected": -0.9278111457824707,
93
  "step": 50
94
  },
95
  {
96
- "epoch": 0.15,
97
- "learning_rate": 4.966461721767899e-07,
98
- "logits/chosen": -2.404550075531006,
99
- "logits/rejected": -2.372710704803467,
100
- "logps/chosen": -378.82940673828125,
101
- "logps/rejected": -210.00704956054688,
102
- "loss": 0.1372,
103
- "rewards/accuracies": 0.7562500238418579,
104
- "rewards/chosen": -0.3348023593425751,
105
- "rewards/margins": 0.8655646443367004,
106
- "rewards/rejected": -1.2003670930862427,
107
  "step": 60
108
  },
109
  {
110
- "epoch": 0.17,
111
- "learning_rate": 4.922100518015975e-07,
112
- "logits/chosen": -2.402493953704834,
113
- "logits/rejected": -2.36720871925354,
114
- "logps/chosen": -450.5000915527344,
115
- "logps/rejected": -274.5812683105469,
116
- "loss": 0.1023,
117
- "rewards/accuracies": 0.762499988079071,
118
- "rewards/chosen": -0.5065652132034302,
119
- "rewards/margins": 1.22487473487854,
120
- "rewards/rejected": -1.7314398288726807,
121
  "step": 70
122
  },
123
  {
124
- "epoch": 0.2,
125
- "learning_rate": 4.859708325770919e-07,
126
- "logits/chosen": -2.3470957279205322,
127
- "logits/rejected": -2.2893269062042236,
128
- "logps/chosen": -458.8729553222656,
129
- "logps/rejected": -316.4507141113281,
130
- "loss": 0.0833,
131
- "rewards/accuracies": 0.8374999761581421,
132
- "rewards/chosen": -0.47927385568618774,
133
- "rewards/margins": 1.5929757356643677,
134
- "rewards/rejected": -2.0722496509552,
135
  "step": 80
136
  },
137
  {
138
- "epoch": 0.22,
139
- "learning_rate": 4.779749614980225e-07,
140
- "logits/chosen": -2.353701114654541,
141
- "logits/rejected": -2.29787015914917,
142
- "logps/chosen": -460.30462646484375,
143
- "logps/rejected": -341.1560974121094,
144
- "loss": 0.0731,
145
- "rewards/accuracies": 0.7875000238418579,
146
- "rewards/chosen": -0.7498773336410522,
147
- "rewards/margins": 1.5740221738815308,
148
- "rewards/rejected": -2.323899745941162,
149
  "step": 90
150
  },
151
  {
152
- "epoch": 0.25,
153
- "learning_rate": 4.682819627081427e-07,
154
- "logits/chosen": -2.3233580589294434,
155
- "logits/rejected": -2.2358224391937256,
156
- "logps/chosen": -494.71783447265625,
157
- "logps/rejected": -365.74462890625,
158
- "loss": 0.0787,
159
- "rewards/accuracies": 0.737500011920929,
160
- "rewards/chosen": -0.7132088541984558,
161
- "rewards/margins": 1.8270628452301025,
162
- "rewards/rejected": -2.540271520614624,
163
  "step": 100
164
  },
165
  {
166
- "epoch": 0.27,
167
- "learning_rate": 4.569639943810477e-07,
168
- "logits/chosen": -2.3129024505615234,
169
- "logits/rejected": -2.2176918983459473,
170
- "logps/chosen": -534.7408447265625,
171
- "logps/rejected": -389.4866638183594,
172
- "loss": 0.0587,
173
- "rewards/accuracies": 0.793749988079071,
174
- "rewards/chosen": -0.9212908744812012,
175
- "rewards/margins": 1.8763319253921509,
176
- "rewards/rejected": -2.7976229190826416,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
  "step": 110
178
  },
179
  {
180
- "epoch": 0.3,
181
- "learning_rate": 4.4410531154874543e-07,
182
- "logits/chosen": -2.316215991973877,
183
- "logits/rejected": -2.223778486251831,
184
- "logps/chosen": -538.7808837890625,
185
- "logps/rejected": -417.7806701660156,
186
- "loss": 0.0448,
187
- "rewards/accuracies": 0.8062499761581421,
188
- "rewards/chosen": -1.0477784872055054,
189
- "rewards/margins": 1.9544658660888672,
190
- "rewards/rejected": -3.002244234085083,
191
  "step": 120
192
  },
193
  {
194
- "epoch": 0.32,
195
- "learning_rate": 4.298016388768561e-07,
196
- "logits/chosen": -2.342869281768799,
197
- "logits/rejected": -2.2497687339782715,
198
- "logps/chosen": -501.41131591796875,
199
- "logps/rejected": -390.2490234375,
200
- "loss": 0.0676,
201
- "rewards/accuracies": 0.8500000238418579,
202
- "rewards/chosen": -0.7455350160598755,
203
- "rewards/margins": 1.8945093154907227,
204
- "rewards/rejected": -2.6400444507598877,
205
  "step": 130
206
  },
207
  {
208
- "epoch": 0.35,
209
- "learning_rate": 4.1415945805573005e-07,
210
- "logits/chosen": -2.343029499053955,
211
- "logits/rejected": -2.2743804454803467,
212
- "logps/chosen": -512.1886596679688,
213
- "logps/rejected": -406.03045654296875,
214
- "loss": 0.0574,
215
- "rewards/accuracies": 0.856249988079071,
216
- "rewards/chosen": -0.6720421314239502,
217
- "rewards/margins": 2.2327818870544434,
218
- "rewards/rejected": -2.9048237800598145,
219
  "step": 140
220
  },
221
  {
222
- "epoch": 0.37,
223
- "learning_rate": 3.972952151123984e-07,
224
- "logits/chosen": -2.246107816696167,
225
- "logits/rejected": -2.1575684547424316,
226
- "logps/chosen": -559.8818969726562,
227
- "logps/rejected": -472.61920166015625,
228
- "loss": 0.0424,
229
- "rewards/accuracies": 0.8374999761581421,
230
- "rewards/chosen": -1.3547112941741943,
231
- "rewards/margins": 2.0968470573425293,
232
- "rewards/rejected": -3.4515585899353027,
233
  "step": 150
234
  },
235
  {
236
- "epoch": 0.39,
237
- "learning_rate": 3.793344535444142e-07,
238
- "logits/chosen": -2.282726526260376,
239
- "logits/rejected": -2.1828060150146484,
240
- "logps/chosen": -590.2351684570312,
241
- "logps/rejected": -452.65087890625,
242
- "loss": 0.0402,
243
- "rewards/accuracies": 0.856249988079071,
244
- "rewards/chosen": -1.2386449575424194,
245
- "rewards/margins": 2.1290228366851807,
246
- "rewards/rejected": -3.3676681518554688,
247
  "step": 160
248
  },
249
  {
250
- "epoch": 0.42,
251
- "learning_rate": 3.604108797288461e-07,
252
- "logits/chosen": -2.2551448345184326,
253
- "logits/rejected": -2.1805520057678223,
254
- "logps/chosen": -529.7485961914062,
255
- "logps/rejected": -446.873779296875,
256
- "loss": 0.042,
257
- "rewards/accuracies": 0.793749988079071,
258
- "rewards/chosen": -1.2568132877349854,
259
- "rewards/margins": 2.019052505493164,
260
- "rewards/rejected": -3.2758660316467285,
261
  "step": 170
262
  },
263
  {
264
- "epoch": 0.44,
265
- "learning_rate": 3.40665367563858e-07,
266
- "logits/chosen": -2.2764365673065186,
267
- "logits/rejected": -2.175426483154297,
268
- "logps/chosen": -488.26385498046875,
269
- "logps/rejected": -383.0807189941406,
270
- "loss": 0.049,
271
- "rewards/accuracies": 0.7749999761581421,
272
- "rewards/chosen": -1.1468641757965088,
273
- "rewards/margins": 1.7448303699493408,
274
- "rewards/rejected": -2.8916945457458496,
275
  "step": 180
276
  },
277
  {
278
- "epoch": 0.47,
279
- "learning_rate": 3.202449097526798e-07,
280
- "logits/chosen": -2.3102352619171143,
281
- "logits/rejected": -2.215864419937134,
282
- "logps/chosen": -495.09588623046875,
283
- "logps/rejected": -415.6620178222656,
284
- "loss": 0.0605,
285
- "rewards/accuracies": 0.831250011920929,
286
- "rewards/chosen": -1.0821647644042969,
287
- "rewards/margins": 1.9470455646514893,
288
- "rewards/rejected": -3.0292105674743652,
289
  "step": 190
290
  },
291
  {
292
- "epoch": 0.49,
293
- "learning_rate": 2.993015235369905e-07,
294
- "logits/chosen": -2.3042426109313965,
295
- "logits/rejected": -2.210401773452759,
296
- "logps/chosen": -522.3523559570312,
297
- "logps/rejected": -429.0350036621094,
298
- "loss": 0.0546,
299
- "rewards/accuracies": 0.887499988079071,
300
- "rewards/chosen": -0.8798307180404663,
301
- "rewards/margins": 2.224181890487671,
302
- "rewards/rejected": -3.1040127277374268,
303
  "step": 200
304
  },
305
  {
306
- "epoch": 0.52,
307
- "learning_rate": 2.7799111902582693e-07,
308
- "logits/chosen": -2.227726936340332,
309
- "logits/rejected": -2.1312689781188965,
310
- "logps/chosen": -507.91546630859375,
311
- "logps/rejected": -440.345703125,
312
- "loss": 0.0461,
313
- "rewards/accuracies": 0.8374999761581421,
314
- "rewards/chosen": -1.2374424934387207,
315
- "rewards/margins": 2.0522685050964355,
316
- "rewards/rejected": -3.2897109985351562,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  "step": 210
318
  },
319
  {
320
- "epoch": 0.54,
321
- "learning_rate": 2.564723385445869e-07,
322
- "logits/chosen": -2.1844215393066406,
323
- "logits/rejected": -2.0822980403900146,
324
- "logps/chosen": -558.802978515625,
325
- "logps/rejected": -482.9683532714844,
326
- "loss": 0.0439,
327
- "rewards/accuracies": 0.8187500238418579,
328
- "rewards/chosen": -1.1958107948303223,
329
- "rewards/margins": 2.303767204284668,
330
- "rewards/rejected": -3.4995779991149902,
331
  "step": 220
332
  },
333
  {
334
- "epoch": 0.57,
335
- "learning_rate": 2.3490537564442845e-07,
336
- "logits/chosen": -2.213812828063965,
337
- "logits/rejected": -2.1118857860565186,
338
- "logps/chosen": -527.6865234375,
339
- "logps/rejected": -451.37347412109375,
340
- "loss": 0.0375,
341
- "rewards/accuracies": 0.793749988079071,
342
- "rewards/chosen": -1.212794303894043,
343
- "rewards/margins": 2.118138074874878,
344
- "rewards/rejected": -3.3309326171875,
345
  "step": 230
346
  },
347
  {
348
- "epoch": 0.59,
349
- "learning_rate": 2.1345078256378801e-07,
350
- "logits/chosen": -2.228127956390381,
351
- "logits/rejected": -2.1170496940612793,
352
- "logps/chosen": -518.7861938476562,
353
- "logps/rejected": -471.34930419921875,
354
- "loss": 0.0325,
355
- "rewards/accuracies": 0.856249988079071,
356
- "rewards/chosen": -1.140577793121338,
357
- "rewards/margins": 2.372342348098755,
358
- "rewards/rejected": -3.5129196643829346,
359
  "step": 240
360
  },
361
  {
362
- "epoch": 0.62,
363
- "learning_rate": 1.9226827501969865e-07,
364
- "logits/chosen": -2.22481107711792,
365
- "logits/rejected": -2.097669839859009,
366
- "logps/chosen": -545.9178466796875,
367
- "logps/rejected": -495.44329833984375,
368
- "loss": 0.0482,
369
- "rewards/accuracies": 0.862500011920929,
370
- "rewards/chosen": -1.1768572330474854,
371
- "rewards/margins": 2.6003925800323486,
372
- "rewards/rejected": -3.777249574661255,
373
  "step": 250
374
  },
375
  {
376
- "epoch": 0.64,
377
- "learning_rate": 1.715155432264775e-07,
378
- "logits/chosen": -2.218748092651367,
379
- "logits/rejected": -2.1094837188720703,
380
- "logps/chosen": -496.226318359375,
381
- "logps/rejected": -427.3712463378906,
382
- "loss": 0.0615,
383
- "rewards/accuracies": 0.84375,
384
- "rewards/chosen": -1.1070716381072998,
385
- "rewards/margins": 2.072988986968994,
386
- "rewards/rejected": -3.180060386657715,
387
  "step": 260
388
  },
389
  {
390
- "epoch": 0.67,
391
- "learning_rate": 1.51347077992983e-07,
392
- "logits/chosen": -2.2541496753692627,
393
- "logits/rejected": -2.1432924270629883,
394
- "logps/chosen": -493.8655700683594,
395
- "logps/rejected": -428.19403076171875,
396
- "loss": 0.0488,
397
- "rewards/accuracies": 0.84375,
398
- "rewards/chosen": -0.9823986291885376,
399
- "rewards/margins": 2.0746474266052246,
400
- "rewards/rejected": -3.057046413421631,
401
  "step": 270
402
  },
403
  {
404
- "epoch": 0.69,
405
- "learning_rate": 1.3191302063739906e-07,
406
- "logits/chosen": -2.2505111694335938,
407
- "logits/rejected": -2.137561798095703,
408
- "logps/chosen": -520.2976684570312,
409
- "logps/rejected": -448.7735900878906,
410
- "loss": 0.045,
411
- "rewards/accuracies": 0.824999988079071,
412
- "rewards/chosen": -1.226133108139038,
413
- "rewards/margins": 2.1102890968322754,
414
- "rewards/rejected": -3.3364219665527344,
415
  "step": 280
416
  },
417
  {
418
- "epoch": 0.72,
419
- "learning_rate": 1.1335804528119475e-07,
420
- "logits/chosen": -2.232018232345581,
421
- "logits/rejected": -2.0879502296447754,
422
- "logps/chosen": -566.6788330078125,
423
- "logps/rejected": -472.78228759765625,
424
- "loss": 0.0366,
425
- "rewards/accuracies": 0.862500011920929,
426
- "rewards/chosen": -1.2060538530349731,
427
- "rewards/margins": 2.433617353439331,
428
- "rewards/rejected": -3.6396713256835938,
429
  "step": 290
430
  },
431
  {
432
- "epoch": 0.74,
433
- "learning_rate": 9.582028184286423e-08,
434
- "logits/chosen": -2.226560354232788,
435
- "logits/rejected": -2.1372122764587402,
436
- "logps/chosen": -492.98162841796875,
437
- "logps/rejected": -440.3003845214844,
438
- "loss": 0.0416,
439
- "rewards/accuracies": 0.762499988079071,
440
- "rewards/chosen": -1.3394794464111328,
441
- "rewards/margins": 1.9935524463653564,
442
- "rewards/rejected": -3.3330321311950684,
443
  "step": 300
444
  },
445
  {
446
- "epoch": 0.76,
447
- "learning_rate": 7.943028774907065e-08,
448
- "logits/chosen": -2.216187000274658,
449
- "logits/rejected": -2.1210825443267822,
450
- "logps/chosen": -514.5095825195312,
451
- "logps/rejected": -437.02569580078125,
452
- "loss": 0.0429,
453
- "rewards/accuracies": 0.800000011920929,
454
- "rewards/chosen": -1.0884000062942505,
455
- "rewards/margins": 2.209381580352783,
456
- "rewards/rejected": -3.2977817058563232,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
457
  "step": 310
458
  },
459
  {
460
- "epoch": 0.79,
461
- "learning_rate": 6.431007601814637e-08,
462
- "logits/chosen": -2.319559335708618,
463
- "logits/rejected": -2.206366777420044,
464
- "logps/chosen": -559.8201904296875,
465
- "logps/rejected": -476.9498596191406,
466
- "loss": 0.0418,
467
- "rewards/accuracies": 0.800000011920929,
468
- "rewards/chosen": -1.3187508583068848,
469
- "rewards/margins": 2.246459722518921,
470
- "rewards/rejected": -3.5652108192443848,
471
  "step": 320
472
  },
473
  {
474
- "epoch": 0.81,
475
- "learning_rate": 5.0572206951246e-08,
476
- "logits/chosen": -2.2584736347198486,
477
- "logits/rejected": -2.124389410018921,
478
- "logps/chosen": -562.3218994140625,
479
- "logps/rejected": -474.72393798828125,
480
- "loss": 0.0445,
481
- "rewards/accuracies": 0.8812500238418579,
482
- "rewards/chosen": -1.0979220867156982,
483
- "rewards/margins": 2.4956517219543457,
484
- "rewards/rejected": -3.593573808670044,
485
  "step": 330
486
  },
487
  {
488
- "epoch": 0.84,
489
- "learning_rate": 3.831895019292897e-08,
490
- "logits/chosen": -2.296125888824463,
491
- "logits/rejected": -2.1779675483703613,
492
- "logps/chosen": -574.4061889648438,
493
- "logps/rejected": -500.864501953125,
494
- "loss": 0.0416,
495
- "rewards/accuracies": 0.856249988079071,
496
- "rewards/chosen": -1.0942927598953247,
497
- "rewards/margins": 2.5931239128112793,
498
- "rewards/rejected": -3.6874160766601562,
499
  "step": 340
500
  },
501
  {
502
- "epoch": 0.86,
503
- "learning_rate": 2.764152339909756e-08,
504
- "logits/chosen": -2.2426624298095703,
505
- "logits/rejected": -2.1238327026367188,
506
- "logps/chosen": -519.2926025390625,
507
- "logps/rejected": -443.22265625,
508
- "loss": 0.0479,
509
- "rewards/accuracies": 0.8062499761581421,
510
- "rewards/chosen": -1.2578840255737305,
511
- "rewards/margins": 2.1321566104888916,
512
- "rewards/rejected": -3.390040636062622,
513
  "step": 350
514
  },
515
  {
516
- "epoch": 0.89,
517
- "learning_rate": 1.861941317991664e-08,
518
- "logits/chosen": -2.227678060531616,
519
- "logits/rejected": -2.120879888534546,
520
- "logps/chosen": -526.3479614257812,
521
- "logps/rejected": -452.90447998046875,
522
- "loss": 0.0426,
523
- "rewards/accuracies": 0.768750011920929,
524
- "rewards/chosen": -1.417136788368225,
525
- "rewards/margins": 1.8834213018417358,
526
- "rewards/rejected": -3.3005576133728027,
527
  "step": 360
528
  },
529
  {
530
- "epoch": 0.91,
531
- "learning_rate": 1.13197833728636e-08,
532
- "logits/chosen": -2.2354464530944824,
533
- "logits/rejected": -2.125568151473999,
534
- "logps/chosen": -536.7144775390625,
535
- "logps/rejected": -455.36993408203125,
536
- "loss": 0.0411,
537
- "rewards/accuracies": 0.800000011920929,
538
- "rewards/chosen": -1.1943690776824951,
539
- "rewards/margins": 2.2170112133026123,
540
- "rewards/rejected": -3.4113802909851074,
541
  "step": 370
542
  },
543
  {
544
- "epoch": 0.94,
545
- "learning_rate": 5.79697505093521e-09,
546
- "logits/chosen": -2.2122418880462646,
547
- "logits/rejected": -2.113687515258789,
548
- "logps/chosen": -501.590087890625,
549
- "logps/rejected": -421.33984375,
550
- "loss": 0.0386,
551
- "rewards/accuracies": 0.78125,
552
- "rewards/chosen": -1.1698087453842163,
553
- "rewards/margins": 1.9851791858673096,
554
- "rewards/rejected": -3.1549878120422363,
555
  "step": 380
556
  },
557
  {
558
- "epoch": 0.96,
559
- "learning_rate": 2.092101988131256e-09,
560
- "logits/chosen": -2.291910409927368,
561
- "logits/rejected": -2.174203395843506,
562
- "logps/chosen": -549.9415893554688,
563
- "logps/rejected": -461.7567443847656,
564
- "loss": 0.0376,
565
- "rewards/accuracies": 0.800000011920929,
566
- "rewards/chosen": -1.1343244314193726,
567
- "rewards/margins": 2.2652435302734375,
568
- "rewards/rejected": -3.3995678424835205,
569
  "step": 390
570
  },
571
  {
572
- "epoch": 0.99,
573
- "learning_rate": 2.327445937151673e-10,
574
- "logits/chosen": -2.214627742767334,
575
- "logits/rejected": -2.1191391944885254,
576
- "logps/chosen": -537.9795532226562,
577
- "logps/rejected": -451.3349609375,
578
- "loss": 0.0457,
579
- "rewards/accuracies": 0.84375,
580
- "rewards/chosen": -1.1944307088851929,
581
- "rewards/margins": 2.0842816829681396,
582
- "rewards/rejected": -3.278712511062622,
583
  "step": 400
584
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
585
  {
586
  "epoch": 1.0,
587
- "step": 405,
588
  "total_flos": 0.0,
589
- "train_loss": 0.08637741568647785,
590
- "train_runtime": 3241.0704,
591
- "train_samples_per_second": 16.011,
592
- "train_steps_per_second": 0.125
593
  }
594
  ],
595
  "logging_steps": 10,
596
- "max_steps": 405,
597
  "num_train_epochs": 1,
598
  "save_steps": 1000,
599
  "total_flos": 0.0,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 100,
6
+ "global_step": 478,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.0416666666666666e-08,
14
+ "logits/chosen": -2.847970962524414,
15
+ "logits/rejected": -2.79160213470459,
16
+ "logps/chosen": -284.9612731933594,
17
+ "logps/rejected": -276.45928955078125,
18
+ "loss": 0.2823,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
24
  },
25
  {
26
  "epoch": 0.02,
27
+ "learning_rate": 1.0416666666666667e-07,
28
+ "logits/chosen": -2.754774570465088,
29
+ "logits/rejected": -2.7528445720672607,
30
+ "logps/chosen": -249.93875122070312,
31
+ "logps/rejected": -223.09078979492188,
32
+ "loss": 0.2705,
33
  "rewards/accuracies": 0.5,
34
+ "rewards/chosen": 8.998061093734577e-05,
35
+ "rewards/margins": 0.0005183664616197348,
36
+ "rewards/rejected": -0.0004283858579583466,
37
  "step": 10
38
  },
39
  {
40
+ "epoch": 0.04,
41
+ "learning_rate": 2.0833333333333333e-07,
42
+ "logits/chosen": -2.7447123527526855,
43
+ "logits/rejected": -2.7451329231262207,
44
+ "logps/chosen": -257.4326477050781,
45
+ "logps/rejected": -247.49862670898438,
46
+ "loss": 0.2724,
47
+ "rewards/accuracies": 0.5249999761581421,
48
+ "rewards/chosen": 0.00022877982701174915,
49
+ "rewards/margins": 0.0009082018514163792,
50
+ "rewards/rejected": -0.0006794218206778169,
51
  "step": 20
52
  },
53
  {
54
+ "epoch": 0.06,
55
+ "learning_rate": 3.1249999999999997e-07,
56
+ "logits/chosen": -2.8000025749206543,
57
+ "logits/rejected": -2.7526888847351074,
58
+ "logps/chosen": -300.5729675292969,
59
+ "logps/rejected": -261.90777587890625,
60
+ "loss": 0.2731,
61
+ "rewards/accuracies": 0.7124999761581421,
62
+ "rewards/chosen": 4.068213820573874e-05,
63
+ "rewards/margins": 0.008200095035135746,
64
+ "rewards/rejected": -0.008159412071108818,
65
  "step": 30
66
  },
67
  {
68
+ "epoch": 0.08,
69
+ "learning_rate": 4.1666666666666667e-07,
70
+ "logits/chosen": -2.7602667808532715,
71
+ "logits/rejected": -2.748173236846924,
72
+ "logps/chosen": -256.903564453125,
73
+ "logps/rejected": -274.865234375,
74
+ "loss": 0.2706,
75
+ "rewards/accuracies": 0.699999988079071,
76
+ "rewards/chosen": -0.004696812480688095,
77
+ "rewards/margins": 0.023122502490878105,
78
+ "rewards/rejected": -0.027819311246275902,
79
  "step": 40
80
  },
81
  {
82
+ "epoch": 0.1,
83
+ "learning_rate": 4.999733114418725e-07,
84
+ "logits/chosen": -2.7563138008117676,
85
+ "logits/rejected": -2.728476047515869,
86
+ "logps/chosen": -286.0164489746094,
87
+ "logps/rejected": -257.14837646484375,
88
+ "loss": 0.2561,
89
+ "rewards/accuracies": 0.637499988079071,
90
+ "rewards/chosen": -0.03937093913555145,
91
+ "rewards/margins": 0.05508055165410042,
92
+ "rewards/rejected": -0.09445149451494217,
93
  "step": 50
94
  },
95
  {
96
+ "epoch": 0.13,
97
+ "learning_rate": 4.990398100856366e-07,
98
+ "logits/chosen": -2.713901996612549,
99
+ "logits/rejected": -2.702225685119629,
100
+ "logps/chosen": -286.9951171875,
101
+ "logps/rejected": -260.49859619140625,
102
+ "loss": 0.235,
103
+ "rewards/accuracies": 0.6625000238418579,
104
+ "rewards/chosen": -0.11110644042491913,
105
+ "rewards/margins": 0.09340100735425949,
106
+ "rewards/rejected": -0.2045074701309204,
107
  "step": 60
108
  },
109
  {
110
+ "epoch": 0.15,
111
+ "learning_rate": 4.967775735898179e-07,
112
+ "logits/chosen": -2.75459623336792,
113
+ "logits/rejected": -2.713303565979004,
114
+ "logps/chosen": -284.32904052734375,
115
+ "logps/rejected": -262.72161865234375,
116
+ "loss": 0.2081,
117
+ "rewards/accuracies": 0.675000011920929,
118
+ "rewards/chosen": -0.17193709313869476,
119
+ "rewards/margins": 0.1358724683523178,
120
+ "rewards/rejected": -0.3078095614910126,
121
  "step": 70
122
  },
123
  {
124
+ "epoch": 0.17,
125
+ "learning_rate": 4.931986719649298e-07,
126
+ "logits/chosen": -2.7569193840026855,
127
+ "logits/rejected": -2.7307088375091553,
128
+ "logps/chosen": -285.26812744140625,
129
+ "logps/rejected": -318.84442138671875,
130
+ "loss": 0.1761,
131
+ "rewards/accuracies": 0.643750011920929,
132
+ "rewards/chosen": -0.24915137887001038,
133
+ "rewards/margins": 0.2163180410861969,
134
+ "rewards/rejected": -0.4654693603515625,
135
  "step": 80
136
  },
137
  {
138
+ "epoch": 0.19,
139
+ "learning_rate": 4.883222001996351e-07,
140
+ "logits/chosen": -2.7807908058166504,
141
+ "logits/rejected": -2.7560718059539795,
142
+ "logps/chosen": -321.1363525390625,
143
+ "logps/rejected": -328.7376403808594,
144
+ "loss": 0.1319,
145
+ "rewards/accuracies": 0.7124999761581421,
146
+ "rewards/chosen": -0.457695335149765,
147
+ "rewards/margins": 0.317694753408432,
148
+ "rewards/rejected": -0.775390088558197,
149
  "step": 90
150
  },
151
  {
152
+ "epoch": 0.21,
153
+ "learning_rate": 4.821741763807186e-07,
154
+ "logits/chosen": -2.773000478744507,
155
+ "logits/rejected": -2.7421765327453613,
156
+ "logps/chosen": -345.4520568847656,
157
+ "logps/rejected": -353.56634521484375,
158
+ "loss": 0.1201,
159
+ "rewards/accuracies": 0.71875,
160
+ "rewards/chosen": -0.5648708343505859,
161
+ "rewards/margins": 0.3486442565917969,
162
+ "rewards/rejected": -0.9135150909423828,
163
  "step": 100
164
  },
165
  {
166
+ "epoch": 0.21,
167
+ "eval_logits/chosen": -2.740356683731079,
168
+ "eval_logits/rejected": -2.7272045612335205,
169
+ "eval_logps/chosen": -307.64208984375,
170
+ "eval_logps/rejected": -347.6167907714844,
171
+ "eval_loss": 0.1357610523700714,
172
+ "eval_rewards/accuracies": 0.69921875,
173
+ "eval_rewards/chosen": -0.5060242414474487,
174
+ "eval_rewards/margins": 0.39661118388175964,
175
+ "eval_rewards/rejected": -0.902635395526886,
176
+ "eval_runtime": 53.6573,
177
+ "eval_samples_per_second": 37.274,
178
+ "eval_steps_per_second": 0.596,
179
+ "step": 100
180
+ },
181
+ {
182
+ "epoch": 0.23,
183
+ "learning_rate": 4.747874028753375e-07,
184
+ "logits/chosen": -2.6956498622894287,
185
+ "logits/rejected": -2.691917896270752,
186
+ "logps/chosen": -321.36102294921875,
187
+ "logps/rejected": -378.28814697265625,
188
+ "loss": 0.1217,
189
+ "rewards/accuracies": 0.7250000238418579,
190
+ "rewards/chosen": -0.5709770917892456,
191
+ "rewards/margins": 0.4847913384437561,
192
+ "rewards/rejected": -1.055768370628357,
193
  "step": 110
194
  },
195
  {
196
+ "epoch": 0.25,
197
+ "learning_rate": 4.662012913161997e-07,
198
+ "logits/chosen": -2.673557758331299,
199
+ "logits/rejected": -2.6405575275421143,
200
+ "logps/chosen": -356.52197265625,
201
+ "logps/rejected": -376.398193359375,
202
+ "loss": 0.0957,
203
+ "rewards/accuracies": 0.625,
204
+ "rewards/chosen": -0.8898218274116516,
205
+ "rewards/margins": 0.4003666043281555,
206
+ "rewards/rejected": -1.2901885509490967,
207
  "step": 120
208
  },
209
  {
210
+ "epoch": 0.27,
211
+ "learning_rate": 4.5646165232345103e-07,
212
+ "logits/chosen": -2.635021209716797,
213
+ "logits/rejected": -2.60518741607666,
214
+ "logps/chosen": -384.78753662109375,
215
+ "logps/rejected": -372.69256591796875,
216
+ "loss": 0.0994,
217
+ "rewards/accuracies": 0.6625000238418579,
218
+ "rewards/chosen": -0.926561176776886,
219
+ "rewards/margins": 0.3979315757751465,
220
+ "rewards/rejected": -1.3244928121566772,
221
  "step": 130
222
  },
223
  {
224
+ "epoch": 0.29,
225
+ "learning_rate": 4.456204510851956e-07,
226
+ "logits/chosen": -2.539923667907715,
227
+ "logits/rejected": -2.522677183151245,
228
+ "logps/chosen": -342.8623962402344,
229
+ "logps/rejected": -359.2884521484375,
230
+ "loss": 0.0884,
231
+ "rewards/accuracies": 0.606249988079071,
232
+ "rewards/chosen": -0.9450332522392273,
233
+ "rewards/margins": 0.4502335488796234,
234
+ "rewards/rejected": -1.3952668905258179,
235
  "step": 140
236
  },
237
  {
238
+ "epoch": 0.31,
239
+ "learning_rate": 4.337355301007335e-07,
240
+ "logits/chosen": -2.579383373260498,
241
+ "logits/rejected": -2.567451000213623,
242
+ "logps/chosen": -346.7669982910156,
243
+ "logps/rejected": -357.2131652832031,
244
+ "loss": 0.1028,
245
+ "rewards/accuracies": 0.706250011920929,
246
+ "rewards/chosen": -0.8153373599052429,
247
+ "rewards/margins": 0.3571455180644989,
248
+ "rewards/rejected": -1.1724828481674194,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.33,
253
+ "learning_rate": 4.2087030056579986e-07,
254
+ "logits/chosen": -2.5627307891845703,
255
+ "logits/rejected": -2.5484416484832764,
256
+ "logps/chosen": -333.15423583984375,
257
+ "logps/rejected": -348.72247314453125,
258
+ "loss": 0.1015,
259
+ "rewards/accuracies": 0.6812499761581421,
260
+ "rewards/chosen": -0.7217438220977783,
261
+ "rewards/margins": 0.45014509558677673,
262
+ "rewards/rejected": -1.171889066696167,
263
  "step": 160
264
  },
265
  {
266
+ "epoch": 0.36,
267
+ "learning_rate": 4.070934040463998e-07,
268
+ "logits/chosen": -2.5039992332458496,
269
+ "logits/rejected": -2.481449842453003,
270
+ "logps/chosen": -368.68890380859375,
271
+ "logps/rejected": -394.72833251953125,
272
+ "loss": 0.0894,
273
+ "rewards/accuracies": 0.71875,
274
+ "rewards/chosen": -1.0003241300582886,
275
+ "rewards/margins": 0.4406798481941223,
276
+ "rewards/rejected": -1.4410040378570557,
277
  "step": 170
278
  },
279
  {
280
+ "epoch": 0.38,
281
+ "learning_rate": 3.9247834624635404e-07,
282
+ "logits/chosen": -2.554518222808838,
283
+ "logits/rejected": -2.513300657272339,
284
+ "logps/chosen": -394.7706604003906,
285
+ "logps/rejected": -433.4938049316406,
286
+ "loss": 0.0698,
287
+ "rewards/accuracies": 0.7437499761581421,
288
+ "rewards/chosen": -1.1832129955291748,
289
+ "rewards/margins": 0.6722770929336548,
290
+ "rewards/rejected": -1.8554903268814087,
291
  "step": 180
292
  },
293
  {
294
+ "epoch": 0.4,
295
+ "learning_rate": 3.7710310482256523e-07,
296
+ "logits/chosen": -2.4630064964294434,
297
+ "logits/rejected": -2.4450414180755615,
298
+ "logps/chosen": -377.68878173828125,
299
+ "logps/rejected": -411.2811584472656,
300
+ "loss": 0.0853,
301
+ "rewards/accuracies": 0.7124999761581421,
302
+ "rewards/chosen": -1.2047637701034546,
303
+ "rewards/margins": 0.5398370027542114,
304
+ "rewards/rejected": -1.7446008920669556,
305
  "step": 190
306
  },
307
  {
308
+ "epoch": 0.42,
309
+ "learning_rate": 3.610497133404795e-07,
310
+ "logits/chosen": -2.568976879119873,
311
+ "logits/rejected": -2.5176620483398438,
312
+ "logps/chosen": -427.2583923339844,
313
+ "logps/rejected": -454.1412658691406,
314
+ "loss": 0.0885,
315
+ "rewards/accuracies": 0.699999988079071,
316
+ "rewards/chosen": -0.9268187284469604,
317
+ "rewards/margins": 0.8155732154846191,
318
+ "rewards/rejected": -1.7423919439315796,
319
  "step": 200
320
  },
321
  {
322
+ "epoch": 0.42,
323
+ "eval_logits/chosen": -2.530900001525879,
324
+ "eval_logits/rejected": -2.518378257751465,
325
+ "eval_logps/chosen": -350.44427490234375,
326
+ "eval_logps/rejected": -418.0752258300781,
327
+ "eval_loss": 0.09392453730106354,
328
+ "eval_rewards/accuracies": 0.73828125,
329
+ "eval_rewards/chosen": -0.93404620885849,
330
+ "eval_rewards/margins": 0.6731735467910767,
331
+ "eval_rewards/rejected": -1.6072198152542114,
332
+ "eval_runtime": 53.6143,
333
+ "eval_samples_per_second": 37.304,
334
+ "eval_steps_per_second": 0.597,
335
+ "step": 200
336
+ },
337
+ {
338
+ "epoch": 0.44,
339
+ "learning_rate": 3.4440382358952115e-07,
340
+ "logits/chosen": -2.501657009124756,
341
+ "logits/rejected": -2.459075927734375,
342
+ "logps/chosen": -377.94622802734375,
343
+ "logps/rejected": -407.7030029296875,
344
+ "loss": 0.0865,
345
+ "rewards/accuracies": 0.6937500238418579,
346
+ "rewards/chosen": -1.107395887374878,
347
+ "rewards/margins": 0.5653480291366577,
348
+ "rewards/rejected": -1.672743797302246,
349
  "step": 210
350
  },
351
  {
352
+ "epoch": 0.46,
353
+ "learning_rate": 3.272542485937368e-07,
354
+ "logits/chosen": -2.4609360694885254,
355
+ "logits/rejected": -2.4514641761779785,
356
+ "logps/chosen": -406.9936218261719,
357
+ "logps/rejected": -421.62713623046875,
358
+ "loss": 0.0847,
359
+ "rewards/accuracies": 0.6937500238418579,
360
+ "rewards/chosen": -1.1243317127227783,
361
+ "rewards/margins": 0.6815956830978394,
362
+ "rewards/rejected": -1.8059272766113281,
363
  "step": 220
364
  },
365
  {
366
+ "epoch": 0.48,
367
+ "learning_rate": 3.096924887558854e-07,
368
+ "logits/chosen": -2.4575066566467285,
369
+ "logits/rejected": -2.4561474323272705,
370
+ "logps/chosen": -397.59552001953125,
371
+ "logps/rejected": -457.85406494140625,
372
+ "loss": 0.0886,
373
+ "rewards/accuracies": 0.6812499761581421,
374
+ "rewards/chosen": -1.157854437828064,
375
+ "rewards/margins": 0.5120266675949097,
376
+ "rewards/rejected": -1.6698811054229736,
377
  "step": 230
378
  },
379
  {
380
+ "epoch": 0.5,
381
+ "learning_rate": 2.9181224366319943e-07,
382
+ "logits/chosen": -2.431061029434204,
383
+ "logits/rejected": -2.4033331871032715,
384
+ "logps/chosen": -411.0369567871094,
385
+ "logps/rejected": -480.3746643066406,
386
+ "loss": 0.0745,
387
+ "rewards/accuracies": 0.675000011920929,
388
+ "rewards/chosen": -1.5733411312103271,
389
+ "rewards/margins": 0.6451319456100464,
390
+ "rewards/rejected": -2.218472957611084,
391
  "step": 240
392
  },
393
  {
394
+ "epoch": 0.52,
395
+ "learning_rate": 2.7370891215954565e-07,
396
+ "logits/chosen": -2.4078166484832764,
397
+ "logits/rejected": -2.3578503131866455,
398
+ "logps/chosen": -425.4388122558594,
399
+ "logps/rejected": -467.2682189941406,
400
+ "loss": 0.069,
401
+ "rewards/accuracies": 0.7250000238418579,
402
+ "rewards/chosen": -1.6651356220245361,
403
+ "rewards/margins": 0.6532995104789734,
404
+ "rewards/rejected": -2.3184351921081543,
405
  "step": 250
406
  },
407
  {
408
+ "epoch": 0.54,
409
+ "learning_rate": 2.55479083351317e-07,
410
+ "logits/chosen": -2.393721580505371,
411
+ "logits/rejected": -2.3937153816223145,
412
+ "logps/chosen": -403.1827697753906,
413
+ "logps/rejected": -439.49090576171875,
414
+ "loss": 0.0841,
415
+ "rewards/accuracies": 0.6875,
416
+ "rewards/chosen": -1.345739722251892,
417
+ "rewards/margins": 0.5644765496253967,
418
+ "rewards/rejected": -1.9102160930633545,
419
  "step": 260
420
  },
421
  {
422
+ "epoch": 0.56,
423
+ "learning_rate": 2.3722002126275822e-07,
424
+ "logits/chosen": -2.353586435317993,
425
+ "logits/rejected": -2.3474771976470947,
426
+ "logps/chosen": -418.86053466796875,
427
+ "logps/rejected": -471.0635681152344,
428
+ "loss": 0.0734,
429
+ "rewards/accuracies": 0.7875000238418579,
430
+ "rewards/chosen": -1.4400711059570312,
431
+ "rewards/margins": 0.71495521068573,
432
+ "rewards/rejected": -2.1550261974334717,
433
  "step": 270
434
  },
435
  {
436
+ "epoch": 0.59,
437
+ "learning_rate": 2.19029145890313e-07,
438
+ "logits/chosen": -2.3498165607452393,
439
+ "logits/rejected": -2.316803216934204,
440
+ "logps/chosen": -401.8115539550781,
441
+ "logps/rejected": -451.2264709472656,
442
+ "loss": 0.074,
443
+ "rewards/accuracies": 0.71875,
444
+ "rewards/chosen": -1.6040595769882202,
445
+ "rewards/margins": 0.6061344742774963,
446
+ "rewards/rejected": -2.2101941108703613,
447
  "step": 280
448
  },
449
  {
450
+ "epoch": 0.61,
451
+ "learning_rate": 2.0100351342479216e-07,
452
+ "logits/chosen": -2.3287718296051025,
453
+ "logits/rejected": -2.287240505218506,
454
+ "logps/chosen": -392.10748291015625,
455
+ "logps/rejected": -440.1629333496094,
456
+ "loss": 0.0706,
457
+ "rewards/accuracies": 0.6937500238418579,
458
+ "rewards/chosen": -1.4549624919891357,
459
+ "rewards/margins": 0.6042402982711792,
460
+ "rewards/rejected": -2.0592026710510254,
461
  "step": 290
462
  },
463
  {
464
+ "epoch": 0.63,
465
+ "learning_rate": 1.8323929841460178e-07,
466
+ "logits/chosen": -2.3261923789978027,
467
+ "logits/rejected": -2.2948966026306152,
468
+ "logps/chosen": -419.4139099121094,
469
+ "logps/rejected": -481.1145935058594,
470
+ "loss": 0.0652,
471
+ "rewards/accuracies": 0.675000011920929,
472
+ "rewards/chosen": -1.6314531564712524,
473
+ "rewards/margins": 0.6061808466911316,
474
+ "rewards/rejected": -2.2376341819763184,
475
  "step": 300
476
  },
477
  {
478
+ "epoch": 0.63,
479
+ "eval_logits/chosen": -2.3504366874694824,
480
+ "eval_logits/rejected": -2.3324406147003174,
481
+ "eval_logps/chosen": -411.44134521484375,
482
+ "eval_logps/rejected": -486.46966552734375,
483
+ "eval_loss": 0.07114572077989578,
484
+ "eval_rewards/accuracies": 0.7265625,
485
+ "eval_rewards/chosen": -1.5440171957015991,
486
+ "eval_rewards/margins": 0.7471470236778259,
487
+ "eval_rewards/rejected": -2.2911641597747803,
488
+ "eval_runtime": 53.5979,
489
+ "eval_samples_per_second": 37.315,
490
+ "eval_steps_per_second": 0.597,
491
+ "step": 300
492
+ },
493
+ {
494
+ "epoch": 0.65,
495
+ "learning_rate": 1.6583128063291573e-07,
496
+ "logits/chosen": -2.2733964920043945,
497
+ "logits/rejected": -2.211473226547241,
498
+ "logps/chosen": -423.8321838378906,
499
+ "logps/rejected": -472.1368713378906,
500
+ "loss": 0.0737,
501
+ "rewards/accuracies": 0.7875000238418579,
502
+ "rewards/chosen": -1.506532907485962,
503
+ "rewards/margins": 0.743132472038269,
504
+ "rewards/rejected": -2.2496652603149414,
505
  "step": 310
506
  },
507
  {
508
+ "epoch": 0.67,
509
+ "learning_rate": 1.488723393865766e-07,
510
+ "logits/chosen": -2.288228750228882,
511
+ "logits/rejected": -2.2716031074523926,
512
+ "logps/chosen": -450.2115173339844,
513
+ "logps/rejected": -444.7579650878906,
514
+ "loss": 0.0709,
515
+ "rewards/accuracies": 0.7250000238418579,
516
+ "rewards/chosen": -1.5382916927337646,
517
+ "rewards/margins": 0.5552228689193726,
518
+ "rewards/rejected": -2.0935144424438477,
519
  "step": 320
520
  },
521
  {
522
+ "epoch": 0.69,
523
+ "learning_rate": 1.3245295796480788e-07,
524
+ "logits/chosen": -2.314911365509033,
525
+ "logits/rejected": -2.2801930904388428,
526
+ "logps/chosen": -391.4331970214844,
527
+ "logps/rejected": -434.216064453125,
528
+ "loss": 0.0708,
529
+ "rewards/accuracies": 0.737500011920929,
530
+ "rewards/chosen": -1.3922003507614136,
531
+ "rewards/margins": 0.6136517524719238,
532
+ "rewards/rejected": -2.005852222442627,
533
  "step": 330
534
  },
535
  {
536
+ "epoch": 0.71,
537
+ "learning_rate": 1.1666074087171627e-07,
538
+ "logits/chosen": -2.251643180847168,
539
+ "logits/rejected": -2.2339367866516113,
540
+ "logps/chosen": -396.1293640136719,
541
+ "logps/rejected": -456.2854919433594,
542
+ "loss": 0.0784,
543
+ "rewards/accuracies": 0.699999988079071,
544
+ "rewards/chosen": -1.448860764503479,
545
+ "rewards/margins": 0.6947659254074097,
546
+ "rewards/rejected": -2.1436266899108887,
547
  "step": 340
548
  },
549
  {
550
+ "epoch": 0.73,
551
+ "learning_rate": 1.0157994641835734e-07,
552
+ "logits/chosen": -2.2622525691986084,
553
+ "logits/rejected": -2.230787992477417,
554
+ "logps/chosen": -384.36944580078125,
555
+ "logps/rejected": -466.31817626953125,
556
+ "loss": 0.0663,
557
+ "rewards/accuracies": 0.699999988079071,
558
+ "rewards/chosen": -1.5841923952102661,
559
+ "rewards/margins": 0.6720179319381714,
560
+ "rewards/rejected": -2.2562103271484375,
561
  "step": 350
562
  },
563
  {
564
+ "epoch": 0.75,
565
+ "learning_rate": 8.729103716819111e-08,
566
+ "logits/chosen": -2.2487359046936035,
567
+ "logits/rejected": -2.199259042739868,
568
+ "logps/chosen": -426.635009765625,
569
+ "logps/rejected": -474.2801818847656,
570
+ "loss": 0.0645,
571
+ "rewards/accuracies": 0.699999988079071,
572
+ "rewards/chosen": -1.5936682224273682,
573
+ "rewards/margins": 0.7557476162910461,
574
+ "rewards/rejected": -2.3494160175323486,
575
  "step": 360
576
  },
577
  {
578
+ "epoch": 0.77,
579
+ "learning_rate": 7.387025063449081e-08,
580
+ "logits/chosen": -2.2165608406066895,
581
+ "logits/rejected": -2.2152469158172607,
582
+ "logps/chosen": -418.34503173828125,
583
+ "logps/rejected": -497.946533203125,
584
+ "loss": 0.0625,
585
+ "rewards/accuracies": 0.71875,
586
+ "rewards/chosen": -1.6309754848480225,
587
+ "rewards/margins": 0.805810272693634,
588
+ "rewards/rejected": -2.436785936355591,
589
  "step": 370
590
  },
591
  {
592
+ "epoch": 0.79,
593
+ "learning_rate": 6.138919252022435e-08,
594
+ "logits/chosen": -2.256133556365967,
595
+ "logits/rejected": -2.2393546104431152,
596
+ "logps/chosen": -441.43310546875,
597
+ "logps/rejected": -475.23712158203125,
598
+ "loss": 0.0628,
599
+ "rewards/accuracies": 0.737500011920929,
600
+ "rewards/chosen": -1.756037950515747,
601
+ "rewards/margins": 0.628067135810852,
602
+ "rewards/rejected": -2.3841052055358887,
603
  "step": 380
604
  },
605
  {
606
+ "epoch": 0.82,
607
+ "learning_rate": 4.991445467064689e-08,
608
+ "logits/chosen": -2.239122152328491,
609
+ "logits/rejected": -2.224815845489502,
610
+ "logps/chosen": -416.82708740234375,
611
+ "logps/rejected": -484.25341796875,
612
+ "loss": 0.0601,
613
+ "rewards/accuracies": 0.7749999761581421,
614
+ "rewards/chosen": -1.5962765216827393,
615
+ "rewards/margins": 0.82659912109375,
616
+ "rewards/rejected": -2.4228758811950684,
617
  "step": 390
618
  },
619
  {
620
+ "epoch": 0.84,
621
+ "learning_rate": 3.9507259776993954e-08,
622
+ "logits/chosen": -2.274501085281372,
623
+ "logits/rejected": -2.2635765075683594,
624
+ "logps/chosen": -472.01751708984375,
625
+ "logps/rejected": -512.2493286132812,
626
+ "loss": 0.0725,
627
+ "rewards/accuracies": 0.71875,
628
+ "rewards/chosen": -1.7185752391815186,
629
+ "rewards/margins": 0.6748024225234985,
630
+ "rewards/rejected": -2.3933777809143066,
631
  "step": 400
632
  },
633
+ {
634
+ "epoch": 0.84,
635
+ "eval_logits/chosen": -2.287046432495117,
636
+ "eval_logits/rejected": -2.269306182861328,
637
+ "eval_logps/chosen": -425.06072998046875,
638
+ "eval_logps/rejected": -502.4063720703125,
639
+ "eval_loss": 0.06801841408014297,
640
+ "eval_rewards/accuracies": 0.7109375,
641
+ "eval_rewards/chosen": -1.6802107095718384,
642
+ "eval_rewards/margins": 0.7703201174736023,
643
+ "eval_rewards/rejected": -2.450531005859375,
644
+ "eval_runtime": 53.6035,
645
+ "eval_samples_per_second": 37.311,
646
+ "eval_steps_per_second": 0.597,
647
+ "step": 400
648
+ },
649
+ {
650
+ "epoch": 0.86,
651
+ "learning_rate": 3.022313472693447e-08,
652
+ "logits/chosen": -2.258284091949463,
653
+ "logits/rejected": -2.2528746128082275,
654
+ "logps/chosen": -426.47216796875,
655
+ "logps/rejected": -454.63909912109375,
656
+ "loss": 0.0632,
657
+ "rewards/accuracies": 0.643750011920929,
658
+ "rewards/chosen": -1.7178280353546143,
659
+ "rewards/margins": 0.503908634185791,
660
+ "rewards/rejected": -2.221736431121826,
661
+ "step": 410
662
+ },
663
+ {
664
+ "epoch": 0.88,
665
+ "learning_rate": 2.2111614344599684e-08,
666
+ "logits/chosen": -2.310163974761963,
667
+ "logits/rejected": -2.267204999923706,
668
+ "logps/chosen": -435.6988220214844,
669
+ "logps/rejected": -493.78741455078125,
670
+ "loss": 0.0657,
671
+ "rewards/accuracies": 0.78125,
672
+ "rewards/chosen": -1.5800026655197144,
673
+ "rewards/margins": 0.861242949962616,
674
+ "rewards/rejected": -2.4412455558776855,
675
+ "step": 420
676
+ },
677
+ {
678
+ "epoch": 0.9,
679
+ "learning_rate": 1.521597710086439e-08,
680
+ "logits/chosen": -2.3116965293884277,
681
+ "logits/rejected": -2.279507637023926,
682
+ "logps/chosen": -431.66619873046875,
683
+ "logps/rejected": -467.61712646484375,
684
+ "loss": 0.0723,
685
+ "rewards/accuracies": 0.6937500238418579,
686
+ "rewards/chosen": -1.5671308040618896,
687
+ "rewards/margins": 0.6525163054466248,
688
+ "rewards/rejected": -2.21964693069458,
689
+ "step": 430
690
+ },
691
+ {
692
+ "epoch": 0.92,
693
+ "learning_rate": 9.57301420397924e-09,
694
+ "logits/chosen": -2.238548755645752,
695
+ "logits/rejected": -2.2381691932678223,
696
+ "logps/chosen": -443.258056640625,
697
+ "logps/rejected": -469.04437255859375,
698
+ "loss": 0.0706,
699
+ "rewards/accuracies": 0.7124999761581421,
700
+ "rewards/chosen": -1.6499240398406982,
701
+ "rewards/margins": 0.6377564668655396,
702
+ "rewards/rejected": -2.2876806259155273,
703
+ "step": 440
704
+ },
705
+ {
706
+ "epoch": 0.94,
707
+ "learning_rate": 5.212833302556258e-09,
708
+ "logits/chosen": -2.256770610809326,
709
+ "logits/rejected": -2.234055757522583,
710
+ "logps/chosen": -437.27239990234375,
711
+ "logps/rejected": -505.88067626953125,
712
+ "loss": 0.0669,
713
+ "rewards/accuracies": 0.71875,
714
+ "rewards/chosen": -1.6065483093261719,
715
+ "rewards/margins": 0.8035796284675598,
716
+ "rewards/rejected": -2.410127878189087,
717
+ "step": 450
718
+ },
719
+ {
720
+ "epoch": 0.96,
721
+ "learning_rate": 2.158697848236607e-09,
722
+ "logits/chosen": -2.2325797080993652,
723
+ "logits/rejected": -2.224133014678955,
724
+ "logps/chosen": -422.4014587402344,
725
+ "logps/rejected": -469.94580078125,
726
+ "loss": 0.0636,
727
+ "rewards/accuracies": 0.731249988079071,
728
+ "rewards/chosen": -1.5709946155548096,
729
+ "rewards/margins": 0.6590017080307007,
730
+ "rewards/rejected": -2.229996681213379,
731
+ "step": 460
732
+ },
733
+ {
734
+ "epoch": 0.98,
735
+ "learning_rate": 4.269029751107489e-10,
736
+ "logits/chosen": -2.278139591217041,
737
+ "logits/rejected": -2.249516487121582,
738
+ "logps/chosen": -439.8173828125,
739
+ "logps/rejected": -465.82080078125,
740
+ "loss": 0.0685,
741
+ "rewards/accuracies": 0.7437499761581421,
742
+ "rewards/chosen": -1.5395467281341553,
743
+ "rewards/margins": 0.7614324688911438,
744
+ "rewards/rejected": -2.3009796142578125,
745
+ "step": 470
746
+ },
747
  {
748
  "epoch": 1.0,
749
+ "step": 478,
750
  "total_flos": 0.0,
751
+ "train_loss": 0.10764013274444197,
752
+ "train_runtime": 3957.0601,
753
+ "train_samples_per_second": 15.449,
754
+ "train_steps_per_second": 0.121
755
  }
756
  ],
757
  "logging_steps": 10,
758
+ "max_steps": 478,
759
  "num_train_epochs": 1,
760
  "save_steps": 1000,
761
  "total_flos": 0.0,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d71b48e7a062b5b93db1d9a47f6551977b7c6cb8be396f9ad2015197910fd029
3
  size 5944
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5abb84cd4af1786cd515c01762184077e49b9b57d0f788907a5f476659af00d2
3
  size 5944