wzhouad commited on
Commit
2ad798c
1 Parent(s): 677f8ac

Model save

Browse files
README.md CHANGED
@@ -16,6 +16,16 @@ should probably proofread and complete it, then remove this comment. -->
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 
 
 
 
 
 
 
 
 
19
 
20
  ## Model description
21
 
@@ -50,6 +60,12 @@ The following hyperparameters were used during training:
50
 
51
  ### Training results
52
 
 
 
 
 
 
 
53
 
54
 
55
  ### Framework versions
 
16
  # zephyr-7b-dpo-full
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
+ It achieves the following results on the evaluation set:
20
+ - Loss: 0.0194
21
+ - Rewards/chosen: -1.8199
22
+ - Rewards/rejected: -2.5238
23
+ - Rewards/accuracies: 0.7031
24
+ - Rewards/margins: 0.7039
25
+ - Logps/rejected: -509.7351
26
+ - Logps/chosen: -439.0294
27
+ - Logits/rejected: -2.2378
28
+ - Logits/chosen: -2.2625
29
 
30
  ## Model description
31
 
 
60
 
61
  ### Training results
62
 
63
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 0.0502 | 0.21 | 100 | 0.0391 | -0.7692 | -1.1939 | 0.6914 | 0.4246 | -376.7401 | -333.9633 | -2.7065 | -2.7235 |
66
+ | 0.0237 | 0.42 | 200 | 0.0268 | -1.2426 | -1.8386 | 0.6680 | 0.5960 | -441.2099 | -381.2964 | -2.4965 | -2.5105 |
67
+ | 0.0248 | 0.63 | 300 | 0.0254 | -1.3833 | -2.0353 | 0.6914 | 0.6521 | -460.8870 | -395.3680 | -2.3196 | -2.3397 |
68
+ | 0.0194 | 0.84 | 400 | 0.0194 | -1.8199 | -2.5238 | 0.7031 | 0.7039 | -509.7351 | -439.0294 | -2.2378 | -2.2625 |
69
 
70
 
71
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.02901148688002133,
4
- "train_runtime": 3226.6015,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 16.083,
7
- "train_steps_per_second": 0.126
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.03868693582495386,
4
+ "train_runtime": 3950.8818,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.474,
7
+ "train_steps_per_second": 0.121
8
  }
config.json CHANGED
@@ -3,7 +3,6 @@
3
  "architectures": [
4
  "MistralForCausalLM"
5
  ],
6
- "attention_dropout": 0.0,
7
  "bos_token_id": 1,
8
  "eos_token_id": 2,
9
  "hidden_act": "silu",
@@ -20,7 +19,7 @@
20
  "sliding_window": 4096,
21
  "tie_word_embeddings": false,
22
  "torch_dtype": "bfloat16",
23
- "transformers_version": "4.41.1",
24
  "use_cache": false,
25
  "vocab_size": 32000
26
  }
 
3
  "architectures": [
4
  "MistralForCausalLM"
5
  ],
 
6
  "bos_token_id": 1,
7
  "eos_token_id": 2,
8
  "hidden_act": "silu",
 
19
  "sliding_window": 4096,
20
  "tie_word_embeddings": false,
21
  "torch_dtype": "bfloat16",
22
+ "transformers_version": "4.35.2",
23
  "use_cache": false,
24
  "vocab_size": 32000
25
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8e23d2fcf85d3633c98976a3baaaab90dbd638a579a4ad13c62c9e5462f3a058
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:472b04e0cc50bda621fe5cbd642762d1767be557c1f585354c28a09172156977
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4cc6f3805f76e68ac4b92a78c11a93c5e6990c6f8d1070bed4d48bf55fb342a2
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aea925956bfb36d8ce6bf0529378c0a70c0af339b58425c70ea48f9146d63695
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5fb6b66cdd7b6dba191b98993395f84d50be9733154e028f6184263aac5c7c25
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70949f8d5a33dbbc2818f352d2585dd202e52822e017939aa5d1bc67dd71cac9
3
  size 4540516344
tokenizer.json CHANGED
@@ -134,7 +134,6 @@
134
  "end_of_word_suffix": null,
135
  "fuse_unk": true,
136
  "byte_fallback": true,
137
- "ignore_merges": false,
138
  "vocab": {
139
  "<unk>": 0,
140
  "<s>": 1,
 
134
  "end_of_word_suffix": null,
135
  "fuse_unk": true,
136
  "byte_fallback": true,
 
137
  "vocab": {
138
  "<unk>": 0,
139
  "<s>": 1,
tokenizer_config.json CHANGED
@@ -1,6 +1,4 @@
1
  {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
 
1
  {
 
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "<unk>",
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.02901148688002133,
4
- "train_runtime": 3226.6015,
5
- "train_samples": 51894,
6
- "train_samples_per_second": 16.083,
7
- "train_steps_per_second": 0.126
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.03868693582495386,
4
+ "train_runtime": 3950.8818,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 15.474,
7
+ "train_steps_per_second": 0.121
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.998766954377312,
5
- "eval_steps": 1000,
6
- "global_step": 405,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 1.2195121951219512e-08,
14
- "logits/chosen": -2.8412017822265625,
15
- "logits/rejected": -2.798035144805908,
16
- "logps/chosen": -481.1488037109375,
17
- "logps/rejected": -89.30835723876953,
18
- "loss": 0.1683,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -24,576 +24,738 @@
24
  },
25
  {
26
  "epoch": 0.02,
27
- "learning_rate": 1.219512195121951e-07,
28
- "logits/chosen": -2.760549306869507,
29
- "logits/rejected": -2.7219324111938477,
30
- "logps/chosen": -418.5386962890625,
31
- "logps/rejected": -116.76993560791016,
32
- "loss": 0.1668,
33
- "rewards/accuracies": 0.4791666567325592,
34
- "rewards/chosen": 0.0004514391766861081,
35
- "rewards/margins": 0.0010180326644331217,
36
- "rewards/rejected": -0.0005665934295393527,
37
  "step": 10
38
  },
39
  {
40
- "epoch": 0.05,
41
- "learning_rate": 2.439024390243902e-07,
42
- "logits/chosen": -2.786688804626465,
43
- "logits/rejected": -2.740872859954834,
44
- "logps/chosen": -396.0628967285156,
45
- "logps/rejected": -109.14559173583984,
46
- "loss": 0.1685,
47
- "rewards/accuracies": 0.7250000238418579,
48
- "rewards/chosen": 0.013597361743450165,
49
- "rewards/margins": 0.02673523500561714,
50
- "rewards/rejected": -0.013137872330844402,
51
  "step": 20
52
  },
53
  {
54
- "epoch": 0.07,
55
- "learning_rate": 3.6585365853658536e-07,
56
- "logits/chosen": -2.7145609855651855,
57
- "logits/rejected": -2.680011749267578,
58
- "logps/chosen": -375.7653503417969,
59
- "logps/rejected": -119.87037658691406,
60
- "loss": 0.1445,
61
- "rewards/accuracies": 0.78125,
62
- "rewards/chosen": 0.05958707258105278,
63
- "rewards/margins": 0.15927240252494812,
64
- "rewards/rejected": -0.09968532621860504,
65
  "step": 30
66
  },
67
  {
68
- "epoch": 0.1,
69
- "learning_rate": 4.878048780487804e-07,
70
- "logits/chosen": -2.554194211959839,
71
- "logits/rejected": -2.5306203365325928,
72
- "logps/chosen": -426.01214599609375,
73
- "logps/rejected": -156.78164672851562,
74
- "loss": 0.1068,
75
- "rewards/accuracies": 0.8374999761581421,
76
- "rewards/chosen": 0.07754631340503693,
77
- "rewards/margins": 0.4439311921596527,
78
- "rewards/rejected": -0.3663848638534546,
79
  "step": 40
80
  },
81
  {
82
- "epoch": 0.12,
83
- "learning_rate": 4.992461696250783e-07,
84
- "logits/chosen": -2.457973003387451,
85
- "logits/rejected": -2.458061456680298,
86
- "logps/chosen": -359.9261779785156,
87
- "logps/rejected": -179.5415802001953,
88
- "loss": 0.0678,
89
- "rewards/accuracies": 0.7749999761581421,
90
- "rewards/chosen": -0.0965609923005104,
91
- "rewards/margins": 0.6131138801574707,
92
- "rewards/rejected": -0.7096748352050781,
93
  "step": 50
94
  },
95
  {
96
- "epoch": 0.15,
97
- "learning_rate": 4.966461721767899e-07,
98
- "logits/chosen": -2.4080331325531006,
99
- "logits/rejected": -2.3387179374694824,
100
- "logps/chosen": -477.7005920410156,
101
- "logps/rejected": -213.6736297607422,
102
- "loss": 0.048,
103
- "rewards/accuracies": 0.8062499761581421,
104
- "rewards/chosen": -0.09366194158792496,
105
- "rewards/margins": 1.000090479850769,
106
- "rewards/rejected": -1.0937525033950806,
107
  "step": 60
108
  },
109
  {
110
- "epoch": 0.17,
111
- "learning_rate": 4.922100518015975e-07,
112
- "logits/chosen": -2.369874954223633,
113
- "logits/rejected": -2.32528018951416,
114
- "logps/chosen": -452.83404541015625,
115
- "logps/rejected": -244.33273315429688,
116
- "loss": 0.0332,
117
- "rewards/accuracies": 0.793749988079071,
118
- "rewards/chosen": -0.27397626638412476,
119
- "rewards/margins": 1.1336454153060913,
120
- "rewards/rejected": -1.4076217412948608,
121
  "step": 70
122
  },
123
  {
124
- "epoch": 0.2,
125
- "learning_rate": 4.859708325770919e-07,
126
- "logits/chosen": -2.4045441150665283,
127
- "logits/rejected": -2.361023426055908,
128
- "logps/chosen": -457.1015625,
129
- "logps/rejected": -260.1266784667969,
130
- "loss": 0.0321,
131
- "rewards/accuracies": 0.768750011920929,
132
- "rewards/chosen": -0.24543538689613342,
133
- "rewards/margins": 1.3190863132476807,
134
- "rewards/rejected": -1.5645216703414917,
135
  "step": 80
136
  },
137
  {
138
- "epoch": 0.22,
139
- "learning_rate": 4.779749614980225e-07,
140
- "logits/chosen": -2.383591651916504,
141
- "logits/rejected": -2.3290789127349854,
142
- "logps/chosen": -447.29949951171875,
143
- "logps/rejected": -292.9815368652344,
144
- "loss": 0.0229,
145
- "rewards/accuracies": 0.800000011920929,
146
- "rewards/chosen": -0.4048345983028412,
147
- "rewards/margins": 1.4302546977996826,
148
- "rewards/rejected": -1.8350893259048462,
149
  "step": 90
150
  },
151
  {
152
- "epoch": 0.25,
153
- "learning_rate": 4.682819627081427e-07,
154
- "logits/chosen": -2.392648458480835,
155
- "logits/rejected": -2.339670181274414,
156
- "logps/chosen": -497.0926818847656,
157
- "logps/rejected": -332.1764221191406,
158
- "loss": 0.0186,
159
- "rewards/accuracies": 0.824999988079071,
160
- "rewards/chosen": -0.6317045092582703,
161
- "rewards/margins": 1.6195662021636963,
162
- "rewards/rejected": -2.2512707710266113,
163
  "step": 100
164
  },
165
  {
166
- "epoch": 0.27,
167
- "learning_rate": 4.569639943810477e-07,
168
- "logits/chosen": -2.383877754211426,
169
- "logits/rejected": -2.328397274017334,
170
- "logps/chosen": -493.90594482421875,
171
- "logps/rejected": -354.2288513183594,
172
- "loss": 0.015,
173
- "rewards/accuracies": 0.7749999761581421,
174
- "rewards/chosen": -0.5901327729225159,
175
- "rewards/margins": 1.7880985736846924,
176
- "rewards/rejected": -2.3782310485839844,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
  "step": 110
178
  },
179
  {
180
- "epoch": 0.3,
181
- "learning_rate": 4.4410531154874543e-07,
182
- "logits/chosen": -2.322819232940674,
183
- "logits/rejected": -2.2361202239990234,
184
- "logps/chosen": -511.83636474609375,
185
- "logps/rejected": -377.99505615234375,
186
- "loss": 0.014,
187
- "rewards/accuracies": 0.8062499761581421,
188
- "rewards/chosen": -0.8583305478096008,
189
- "rewards/margins": 1.807847261428833,
190
- "rewards/rejected": -2.666177749633789,
191
  "step": 120
192
  },
193
  {
194
- "epoch": 0.32,
195
- "learning_rate": 4.298016388768561e-07,
196
- "logits/chosen": -2.2856903076171875,
197
- "logits/rejected": -2.205942153930664,
198
- "logps/chosen": -532.1744384765625,
199
- "logps/rejected": -420.78759765625,
200
- "loss": 0.0101,
201
- "rewards/accuracies": 0.8374999761581421,
202
- "rewards/chosen": -1.339316725730896,
203
- "rewards/margins": 1.7573401927947998,
204
- "rewards/rejected": -3.0966567993164062,
205
  "step": 130
206
  },
207
  {
208
- "epoch": 0.35,
209
- "learning_rate": 4.1415945805573005e-07,
210
- "logits/chosen": -2.2981626987457275,
211
- "logits/rejected": -2.216963529586792,
212
- "logps/chosen": -511.155029296875,
213
- "logps/rejected": -384.3242492675781,
214
- "loss": 0.0119,
215
- "rewards/accuracies": 0.78125,
216
- "rewards/chosen": -0.8295551538467407,
217
- "rewards/margins": 1.907530426979065,
218
- "rewards/rejected": -2.7370851039886475,
219
  "step": 140
220
  },
221
  {
222
- "epoch": 0.37,
223
- "learning_rate": 3.972952151123984e-07,
224
- "logits/chosen": -2.299988269805908,
225
- "logits/rejected": -2.2516579627990723,
226
- "logps/chosen": -495.01983642578125,
227
- "logps/rejected": -383.9313659667969,
228
- "loss": 0.013,
229
- "rewards/accuracies": 0.78125,
230
- "rewards/chosen": -0.8993278741836548,
231
- "rewards/margins": 1.656372308731079,
232
- "rewards/rejected": -2.5557000637054443,
233
  "step": 150
234
  },
235
  {
236
- "epoch": 0.39,
237
- "learning_rate": 3.793344535444142e-07,
238
- "logits/chosen": -2.2775027751922607,
239
- "logits/rejected": -2.1973774433135986,
240
- "logps/chosen": -531.9027709960938,
241
- "logps/rejected": -419.78973388671875,
242
- "loss": 0.0137,
243
- "rewards/accuracies": 0.762499988079071,
244
- "rewards/chosen": -1.0927542448043823,
245
- "rewards/margins": 1.8101270198822021,
246
- "rewards/rejected": -2.902881383895874,
247
  "step": 160
248
  },
249
  {
250
- "epoch": 0.42,
251
- "learning_rate": 3.604108797288461e-07,
252
- "logits/chosen": -2.250196933746338,
253
- "logits/rejected": -2.184962749481201,
254
- "logps/chosen": -502.00848388671875,
255
- "logps/rejected": -357.9796447753906,
256
- "loss": 0.0159,
257
- "rewards/accuracies": 0.8500000238418579,
258
- "rewards/chosen": -0.826766848564148,
259
- "rewards/margins": 1.7090247869491577,
260
- "rewards/rejected": -2.5357918739318848,
261
  "step": 170
262
  },
263
  {
264
- "epoch": 0.44,
265
- "learning_rate": 3.40665367563858e-07,
266
- "logits/chosen": -2.2178635597229004,
267
- "logits/rejected": -2.142676830291748,
268
- "logps/chosen": -526.4637451171875,
269
- "logps/rejected": -398.69586181640625,
270
- "loss": 0.0125,
271
- "rewards/accuracies": 0.8062499761581421,
272
- "rewards/chosen": -1.0359699726104736,
273
- "rewards/margins": 1.687019944190979,
274
- "rewards/rejected": -2.722989559173584,
275
  "step": 180
276
  },
277
  {
278
- "epoch": 0.47,
279
- "learning_rate": 3.202449097526798e-07,
280
- "logits/chosen": -2.2205810546875,
281
- "logits/rejected": -2.1265811920166016,
282
- "logps/chosen": -558.7676391601562,
283
- "logps/rejected": -478.9952087402344,
284
- "loss": 0.0106,
285
- "rewards/accuracies": 0.8062499761581421,
286
- "rewards/chosen": -1.2641270160675049,
287
- "rewards/margins": 2.266322612762451,
288
- "rewards/rejected": -3.530449628829956,
289
  "step": 190
290
  },
291
  {
292
- "epoch": 0.49,
293
- "learning_rate": 2.993015235369905e-07,
294
- "logits/chosen": -2.287707805633545,
295
- "logits/rejected": -2.2064316272735596,
296
- "logps/chosen": -542.5303955078125,
297
- "logps/rejected": -418.10614013671875,
298
- "loss": 0.0121,
299
- "rewards/accuracies": 0.831250011920929,
300
- "rewards/chosen": -0.9594089388847351,
301
- "rewards/margins": 2.0673024654388428,
302
- "rewards/rejected": -3.0267112255096436,
303
  "step": 200
304
  },
305
  {
306
- "epoch": 0.52,
307
- "learning_rate": 2.7799111902582693e-07,
308
- "logits/chosen": -2.2779388427734375,
309
- "logits/rejected": -2.217900037765503,
310
- "logps/chosen": -507.04736328125,
311
- "logps/rejected": -406.27593994140625,
312
- "loss": 0.0167,
313
- "rewards/accuracies": 0.800000011920929,
314
- "rewards/chosen": -0.7974947690963745,
315
- "rewards/margins": 1.9174222946166992,
316
- "rewards/rejected": -2.7149171829223633,
317
- "step": 210
 
 
318
  },
319
  {
320
- "epoch": 0.54,
321
- "learning_rate": 2.564723385445869e-07,
322
- "logits/chosen": -2.236158847808838,
323
- "logits/rejected": -2.1744391918182373,
324
- "logps/chosen": -486.0782165527344,
325
- "logps/rejected": -399.27587890625,
326
- "loss": 0.014,
327
  "rewards/accuracies": 0.768750011920929,
328
- "rewards/chosen": -0.9962589144706726,
329
- "rewards/margins": 1.6734790802001953,
330
- "rewards/rejected": -2.6697378158569336,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
331
  "step": 220
332
  },
333
  {
334
- "epoch": 0.57,
335
- "learning_rate": 2.3490537564442845e-07,
336
- "logits/chosen": -2.245281934738159,
337
- "logits/rejected": -2.1824960708618164,
338
- "logps/chosen": -487.1275329589844,
339
- "logps/rejected": -412.4351501464844,
340
- "loss": 0.0112,
341
- "rewards/accuracies": 0.7749999761581421,
342
- "rewards/chosen": -1.0274240970611572,
343
- "rewards/margins": 1.9516007900238037,
344
- "rewards/rejected": -2.97902512550354,
345
  "step": 230
346
  },
347
  {
348
- "epoch": 0.59,
349
- "learning_rate": 2.1345078256378801e-07,
350
- "logits/chosen": -2.2380142211914062,
351
- "logits/rejected": -2.1482958793640137,
352
- "logps/chosen": -488.2174377441406,
353
- "logps/rejected": -404.8780822753906,
354
- "loss": 0.0111,
355
- "rewards/accuracies": 0.800000011920929,
356
- "rewards/chosen": -1.1305696964263916,
357
- "rewards/margins": 1.7587543725967407,
358
- "rewards/rejected": -2.8893239498138428,
359
  "step": 240
360
  },
361
  {
362
- "epoch": 0.62,
363
- "learning_rate": 1.9226827501969865e-07,
364
- "logits/chosen": -2.2937111854553223,
365
- "logits/rejected": -2.209343433380127,
366
- "logps/chosen": -510.5157165527344,
367
- "logps/rejected": -376.5196838378906,
368
- "loss": 0.0143,
369
- "rewards/accuracies": 0.875,
370
- "rewards/chosen": -0.6733769178390503,
371
- "rewards/margins": 1.9475233554840088,
372
- "rewards/rejected": -2.6209003925323486,
373
  "step": 250
374
  },
375
  {
376
- "epoch": 0.64,
377
- "learning_rate": 1.715155432264775e-07,
378
- "logits/chosen": -2.2513487339019775,
379
- "logits/rejected": -2.203629732131958,
380
- "logps/chosen": -487.1371154785156,
381
- "logps/rejected": -376.69622802734375,
382
- "loss": 0.0154,
383
- "rewards/accuracies": 0.8125,
384
- "rewards/chosen": -0.7552510499954224,
385
- "rewards/margins": 1.8159189224243164,
386
- "rewards/rejected": -2.5711700916290283,
387
  "step": 260
388
  },
389
  {
390
- "epoch": 0.67,
391
- "learning_rate": 1.51347077992983e-07,
392
- "logits/chosen": -2.2633750438690186,
393
- "logits/rejected": -2.1752829551696777,
394
- "logps/chosen": -490.79290771484375,
395
- "logps/rejected": -407.4229431152344,
396
- "loss": 0.0126,
397
- "rewards/accuracies": 0.824999988079071,
398
- "rewards/chosen": -0.8112813234329224,
399
- "rewards/margins": 2.021395444869995,
400
- "rewards/rejected": -2.832677125930786,
401
  "step": 270
402
  },
403
  {
404
- "epoch": 0.69,
405
- "learning_rate": 1.3191302063739906e-07,
406
- "logits/chosen": -2.2145628929138184,
407
- "logits/rejected": -2.1269583702087402,
408
- "logps/chosen": -496.5582580566406,
409
- "logps/rejected": -396.8327941894531,
410
- "loss": 0.0118,
411
- "rewards/accuracies": 0.831250011920929,
412
- "rewards/chosen": -0.9003747701644897,
413
- "rewards/margins": 1.9780778884887695,
414
- "rewards/rejected": -2.878452777862549,
415
  "step": 280
416
  },
417
  {
418
- "epoch": 0.72,
419
- "learning_rate": 1.1335804528119475e-07,
420
- "logits/chosen": -2.2652816772460938,
421
- "logits/rejected": -2.1710205078125,
422
- "logps/chosen": -510.77435302734375,
423
- "logps/rejected": -426.341796875,
424
- "loss": 0.0122,
425
- "rewards/accuracies": 0.8125,
426
- "rewards/chosen": -1.0885112285614014,
427
- "rewards/margins": 2.0129168033599854,
428
- "rewards/rejected": -3.101428508758545,
429
  "step": 290
430
  },
431
  {
432
- "epoch": 0.74,
433
- "learning_rate": 9.582028184286423e-08,
434
- "logits/chosen": -2.234984874725342,
435
- "logits/rejected": -2.140819549560547,
436
- "logps/chosen": -563.6864013671875,
437
- "logps/rejected": -454.06292724609375,
438
- "loss": 0.0094,
439
- "rewards/accuracies": 0.8125,
440
- "rewards/chosen": -1.0196846723556519,
441
- "rewards/margins": 2.260946273803711,
442
- "rewards/rejected": -3.2806308269500732,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
443
  "step": 300
444
  },
445
  {
446
- "epoch": 0.76,
447
- "learning_rate": 7.943028774907065e-08,
448
- "logits/chosen": -2.199270248413086,
449
- "logits/rejected": -2.1144025325775146,
450
- "logps/chosen": -539.964111328125,
451
- "logps/rejected": -449.0367736816406,
452
- "loss": 0.0102,
453
- "rewards/accuracies": 0.793749988079071,
454
- "rewards/chosen": -1.2019588947296143,
455
- "rewards/margins": 2.055799722671509,
456
- "rewards/rejected": -3.2577590942382812,
457
  "step": 310
458
  },
459
  {
460
- "epoch": 0.79,
461
- "learning_rate": 6.431007601814637e-08,
462
- "logits/chosen": -2.2333831787109375,
463
- "logits/rejected": -2.1530561447143555,
464
- "logps/chosen": -524.09033203125,
465
- "logps/rejected": -443.92449951171875,
466
- "loss": 0.0109,
467
- "rewards/accuracies": 0.793749988079071,
468
- "rewards/chosen": -1.2250614166259766,
469
- "rewards/margins": 2.0103707313537598,
470
- "rewards/rejected": -3.2354321479797363,
471
  "step": 320
472
  },
473
  {
474
- "epoch": 0.81,
475
- "learning_rate": 5.0572206951246e-08,
476
- "logits/chosen": -2.2508292198181152,
477
- "logits/rejected": -2.168816328048706,
478
- "logps/chosen": -533.7239990234375,
479
- "logps/rejected": -462.59722900390625,
480
- "loss": 0.0133,
481
- "rewards/accuracies": 0.768750011920929,
482
- "rewards/chosen": -1.1595772504806519,
483
- "rewards/margins": 2.1093368530273438,
484
- "rewards/rejected": -3.268913984298706,
485
  "step": 330
486
  },
487
  {
488
- "epoch": 0.84,
489
- "learning_rate": 3.831895019292897e-08,
490
- "logits/chosen": -2.1972341537475586,
491
- "logits/rejected": -2.0935590267181396,
492
- "logps/chosen": -499.79541015625,
493
- "logps/rejected": -410.91278076171875,
494
- "loss": 0.01,
495
- "rewards/accuracies": 0.762499988079071,
496
- "rewards/chosen": -1.1320897340774536,
497
- "rewards/margins": 1.9490875005722046,
498
- "rewards/rejected": -3.0811774730682373,
499
  "step": 340
500
  },
501
  {
502
- "epoch": 0.86,
503
- "learning_rate": 2.764152339909756e-08,
504
- "logits/chosen": -2.2604408264160156,
505
- "logits/rejected": -2.1542410850524902,
506
- "logps/chosen": -529.6370239257812,
507
- "logps/rejected": -436.3409118652344,
508
- "loss": 0.0094,
509
- "rewards/accuracies": 0.800000011920929,
510
- "rewards/chosen": -1.1161201000213623,
511
- "rewards/margins": 2.068046808242798,
512
- "rewards/rejected": -3.184166431427002,
513
  "step": 350
514
  },
515
  {
516
- "epoch": 0.89,
517
- "learning_rate": 1.861941317991664e-08,
518
- "logits/chosen": -2.2331619262695312,
519
- "logits/rejected": -2.1493821144104004,
520
- "logps/chosen": -487.66253662109375,
521
- "logps/rejected": -430.4197692871094,
522
- "loss": 0.0113,
523
- "rewards/accuracies": 0.8125,
524
- "rewards/chosen": -1.1774896383285522,
525
- "rewards/margins": 2.0229172706604004,
526
- "rewards/rejected": -3.2004075050354004,
527
  "step": 360
528
  },
529
  {
530
- "epoch": 0.91,
531
- "learning_rate": 1.13197833728636e-08,
532
- "logits/chosen": -2.2261927127838135,
533
- "logits/rejected": -2.1371588706970215,
534
- "logps/chosen": -520.0165405273438,
535
- "logps/rejected": -424.5750427246094,
536
- "loss": 0.01,
537
- "rewards/accuracies": 0.800000011920929,
538
- "rewards/chosen": -0.9708250164985657,
539
- "rewards/margins": 2.168119430541992,
540
- "rewards/rejected": -3.138944149017334,
541
  "step": 370
542
  },
543
  {
544
- "epoch": 0.94,
545
- "learning_rate": 5.79697505093521e-09,
546
- "logits/chosen": -2.154387950897217,
547
- "logits/rejected": -2.0763607025146484,
548
- "logps/chosen": -512.8585205078125,
549
- "logps/rejected": -427.20465087890625,
550
- "loss": 0.0112,
551
- "rewards/accuracies": 0.8125,
552
- "rewards/chosen": -1.175691843032837,
553
- "rewards/margins": 2.0045082569122314,
554
- "rewards/rejected": -3.1802000999450684,
555
  "step": 380
556
  },
557
  {
558
- "epoch": 0.96,
559
- "learning_rate": 2.092101988131256e-09,
560
- "logits/chosen": -2.2704200744628906,
561
- "logits/rejected": -2.173677444458008,
562
- "logps/chosen": -558.5034790039062,
563
- "logps/rejected": -441.8016052246094,
564
- "loss": 0.0101,
565
- "rewards/accuracies": 0.800000011920929,
566
- "rewards/chosen": -1.062382698059082,
567
- "rewards/margins": 2.1931846141815186,
568
- "rewards/rejected": -3.2555670738220215,
569
  "step": 390
570
  },
571
  {
572
- "epoch": 0.99,
573
- "learning_rate": 2.327445937151673e-10,
574
- "logits/chosen": -2.2109153270721436,
575
- "logits/rejected": -2.132047176361084,
576
- "logps/chosen": -552.7464599609375,
577
- "logps/rejected": -436.6908264160156,
578
- "loss": 0.0093,
579
- "rewards/accuracies": 0.768750011920929,
580
- "rewards/chosen": -1.1525399684906006,
581
- "rewards/margins": 2.0077481269836426,
582
- "rewards/rejected": -3.160288095474243,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
583
  "step": 400
584
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
585
  {
586
  "epoch": 1.0,
587
- "step": 405,
588
  "total_flos": 0.0,
589
- "train_loss": 0.02901148688002133,
590
- "train_runtime": 3226.6015,
591
- "train_samples_per_second": 16.083,
592
- "train_steps_per_second": 0.126
593
  }
594
  ],
595
  "logging_steps": 10,
596
- "max_steps": 405,
597
  "num_train_epochs": 1,
598
  "save_steps": 1000,
599
  "total_flos": 0.0,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 100,
6
+ "global_step": 478,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.0416666666666666e-08,
14
+ "logits/chosen": -2.7050857543945312,
15
+ "logits/rejected": -2.7461352348327637,
16
+ "logps/chosen": -137.25845336914062,
17
+ "logps/rejected": -163.38693237304688,
18
+ "loss": 0.0968,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
24
  },
25
  {
26
  "epoch": 0.02,
27
+ "learning_rate": 1.0416666666666667e-07,
28
+ "logits/chosen": -2.731086015701294,
29
+ "logits/rejected": -2.69549298286438,
30
+ "logps/chosen": -255.12014770507812,
31
+ "logps/rejected": -238.85885620117188,
32
+ "loss": 0.1059,
33
+ "rewards/accuracies": 0.4722222089767456,
34
+ "rewards/chosen": 0.00021500665752682835,
35
+ "rewards/margins": 0.00010767025378299877,
36
+ "rewards/rejected": 0.00010733638919191435,
37
  "step": 10
38
  },
39
  {
40
+ "epoch": 0.04,
41
+ "learning_rate": 2.0833333333333333e-07,
42
+ "logits/chosen": -2.7516608238220215,
43
+ "logits/rejected": -2.7480316162109375,
44
+ "logps/chosen": -265.412109375,
45
+ "logps/rejected": -255.61865234375,
46
+ "loss": 0.1048,
47
+ "rewards/accuracies": 0.5687500238418579,
48
+ "rewards/chosen": -4.060972059960477e-05,
49
+ "rewards/margins": 0.0010573023464530706,
50
+ "rewards/rejected": -0.0010979120852425694,
51
  "step": 20
52
  },
53
  {
54
+ "epoch": 0.06,
55
+ "learning_rate": 3.1249999999999997e-07,
56
+ "logits/chosen": -2.741386651992798,
57
+ "logits/rejected": -2.698482036590576,
58
+ "logps/chosen": -253.40029907226562,
59
+ "logps/rejected": -252.72500610351562,
60
+ "loss": 0.1012,
61
+ "rewards/accuracies": 0.637499988079071,
62
+ "rewards/chosen": -0.0009587205713614821,
63
+ "rewards/margins": 0.009668193757534027,
64
+ "rewards/rejected": -0.010626914910972118,
65
  "step": 30
66
  },
67
  {
68
+ "epoch": 0.08,
69
+ "learning_rate": 4.1666666666666667e-07,
70
+ "logits/chosen": -2.792792797088623,
71
+ "logits/rejected": -2.778289556503296,
72
+ "logps/chosen": -274.712158203125,
73
+ "logps/rejected": -279.0121154785156,
74
+ "loss": 0.1022,
75
+ "rewards/accuracies": 0.612500011920929,
76
+ "rewards/chosen": -0.0003971463011112064,
77
+ "rewards/margins": 0.02575758472084999,
78
+ "rewards/rejected": -0.02615473046898842,
79
  "step": 40
80
  },
81
  {
82
+ "epoch": 0.1,
83
+ "learning_rate": 4.999733114418725e-07,
84
+ "logits/chosen": -2.760355234146118,
85
+ "logits/rejected": -2.7391133308410645,
86
+ "logps/chosen": -286.1962890625,
87
+ "logps/rejected": -295.90069580078125,
88
+ "loss": 0.0957,
89
+ "rewards/accuracies": 0.6499999761581421,
90
+ "rewards/chosen": -0.041570521891117096,
91
+ "rewards/margins": 0.03570377081632614,
92
+ "rewards/rejected": -0.07727430015802383,
93
  "step": 50
94
  },
95
  {
96
+ "epoch": 0.13,
97
+ "learning_rate": 4.990398100856366e-07,
98
+ "logits/chosen": -2.776111125946045,
99
+ "logits/rejected": -2.7512311935424805,
100
+ "logps/chosen": -276.09942626953125,
101
+ "logps/rejected": -258.3904724121094,
102
+ "loss": 0.0889,
103
+ "rewards/accuracies": 0.6187499761581421,
104
+ "rewards/chosen": -0.06735710799694061,
105
+ "rewards/margins": 0.1039455384016037,
106
+ "rewards/rejected": -0.1713026612997055,
107
  "step": 60
108
  },
109
  {
110
+ "epoch": 0.15,
111
+ "learning_rate": 4.967775735898179e-07,
112
+ "logits/chosen": -2.768890857696533,
113
+ "logits/rejected": -2.7490429878234863,
114
+ "logps/chosen": -270.66156005859375,
115
+ "logps/rejected": -301.940185546875,
116
+ "loss": 0.0783,
117
+ "rewards/accuracies": 0.612500011920929,
118
+ "rewards/chosen": -0.15578190982341766,
119
+ "rewards/margins": 0.11446709930896759,
120
+ "rewards/rejected": -0.27024897933006287,
121
  "step": 70
122
  },
123
  {
124
+ "epoch": 0.17,
125
+ "learning_rate": 4.931986719649298e-07,
126
+ "logits/chosen": -2.7411890029907227,
127
+ "logits/rejected": -2.71295428276062,
128
+ "logps/chosen": -270.99969482421875,
129
+ "logps/rejected": -290.6012878417969,
130
+ "loss": 0.0734,
131
+ "rewards/accuracies": 0.6625000238418579,
132
+ "rewards/chosen": -0.17659564316272736,
133
+ "rewards/margins": 0.2667813003063202,
134
+ "rewards/rejected": -0.44337692856788635,
135
  "step": 80
136
  },
137
  {
138
+ "epoch": 0.19,
139
+ "learning_rate": 4.883222001996351e-07,
140
+ "logits/chosen": -2.7742576599121094,
141
+ "logits/rejected": -2.7353005409240723,
142
+ "logps/chosen": -315.1212463378906,
143
+ "logps/rejected": -318.26153564453125,
144
+ "loss": 0.0628,
145
+ "rewards/accuracies": 0.7250000238418579,
146
+ "rewards/chosen": -0.37408870458602905,
147
+ "rewards/margins": 0.32524389028549194,
148
+ "rewards/rejected": -0.699332594871521,
149
  "step": 90
150
  },
151
  {
152
+ "epoch": 0.21,
153
+ "learning_rate": 4.821741763807186e-07,
154
+ "logits/chosen": -2.7804133892059326,
155
+ "logits/rejected": -2.755944013595581,
156
+ "logps/chosen": -322.177978515625,
157
+ "logps/rejected": -350.9091796875,
158
+ "loss": 0.0502,
159
+ "rewards/accuracies": 0.6937500238418579,
160
+ "rewards/chosen": -0.5584506988525391,
161
+ "rewards/margins": 0.32031145691871643,
162
+ "rewards/rejected": -0.8787621259689331,
163
  "step": 100
164
  },
165
  {
166
+ "epoch": 0.21,
167
+ "eval_logits/chosen": -2.7234888076782227,
168
+ "eval_logits/rejected": -2.7065277099609375,
169
+ "eval_logps/chosen": -333.9632873535156,
170
+ "eval_logps/rejected": -376.7401428222656,
171
+ "eval_loss": 0.03910641372203827,
172
+ "eval_rewards/accuracies": 0.69140625,
173
+ "eval_rewards/chosen": -0.76923668384552,
174
+ "eval_rewards/margins": 0.4246319532394409,
175
+ "eval_rewards/rejected": -1.193868637084961,
176
+ "eval_runtime": 53.787,
177
+ "eval_samples_per_second": 37.184,
178
+ "eval_steps_per_second": 0.595,
179
+ "step": 100
180
+ },
181
+ {
182
+ "epoch": 0.23,
183
+ "learning_rate": 4.747874028753375e-07,
184
+ "logits/chosen": -2.6888976097106934,
185
+ "logits/rejected": -2.650428056716919,
186
+ "logps/chosen": -366.8619079589844,
187
+ "logps/rejected": -376.2767028808594,
188
+ "loss": 0.0371,
189
+ "rewards/accuracies": 0.7124999761581421,
190
+ "rewards/chosen": -0.7290430665016174,
191
+ "rewards/margins": 0.3944670259952545,
192
+ "rewards/rejected": -1.1235100030899048,
193
  "step": 110
194
  },
195
  {
196
+ "epoch": 0.25,
197
+ "learning_rate": 4.662012913161997e-07,
198
+ "logits/chosen": -2.733076810836792,
199
+ "logits/rejected": -2.6883604526519775,
200
+ "logps/chosen": -342.3517150878906,
201
+ "logps/rejected": -385.7752990722656,
202
+ "loss": 0.0398,
203
+ "rewards/accuracies": 0.71875,
204
+ "rewards/chosen": -0.7808613181114197,
205
+ "rewards/margins": 0.501408040523529,
206
+ "rewards/rejected": -1.2822694778442383,
207
  "step": 120
208
  },
209
  {
210
+ "epoch": 0.27,
211
+ "learning_rate": 4.5646165232345103e-07,
212
+ "logits/chosen": -2.709172248840332,
213
+ "logits/rejected": -2.6797726154327393,
214
+ "logps/chosen": -348.2884216308594,
215
+ "logps/rejected": -364.2542724609375,
216
+ "loss": 0.0333,
217
+ "rewards/accuracies": 0.6499999761581421,
218
+ "rewards/chosen": -0.8973625898361206,
219
+ "rewards/margins": 0.41800689697265625,
220
+ "rewards/rejected": -1.3153694868087769,
221
  "step": 130
222
  },
223
  {
224
+ "epoch": 0.29,
225
+ "learning_rate": 4.456204510851956e-07,
226
+ "logits/chosen": -2.6971547603607178,
227
+ "logits/rejected": -2.6667492389678955,
228
+ "logps/chosen": -369.86016845703125,
229
+ "logps/rejected": -398.45721435546875,
230
+ "loss": 0.0363,
231
+ "rewards/accuracies": 0.6875,
232
+ "rewards/chosen": -0.8229220509529114,
233
+ "rewards/margins": 0.38307899236679077,
234
+ "rewards/rejected": -1.2060010433197021,
235
  "step": 140
236
  },
237
  {
238
+ "epoch": 0.31,
239
+ "learning_rate": 4.337355301007335e-07,
240
+ "logits/chosen": -2.5674502849578857,
241
+ "logits/rejected": -2.5130200386047363,
242
+ "logps/chosen": -363.6653747558594,
243
+ "logps/rejected": -388.075439453125,
244
+ "loss": 0.0322,
245
+ "rewards/accuracies": 0.706250011920929,
246
+ "rewards/chosen": -0.874055027961731,
247
+ "rewards/margins": 0.542679488658905,
248
+ "rewards/rejected": -1.4167345762252808,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.33,
253
+ "learning_rate": 4.2087030056579986e-07,
254
+ "logits/chosen": -2.5751426219940186,
255
+ "logits/rejected": -2.5513062477111816,
256
+ "logps/chosen": -367.0979919433594,
257
+ "logps/rejected": -382.1893615722656,
258
+ "loss": 0.0365,
259
+ "rewards/accuracies": 0.6937500238418579,
260
+ "rewards/chosen": -0.9746230840682983,
261
+ "rewards/margins": 0.31770768761634827,
262
+ "rewards/rejected": -1.2923307418823242,
263
  "step": 160
264
  },
265
  {
266
+ "epoch": 0.36,
267
+ "learning_rate": 4.070934040463998e-07,
268
+ "logits/chosen": -2.5913219451904297,
269
+ "logits/rejected": -2.5757110118865967,
270
+ "logps/chosen": -371.7869567871094,
271
+ "logps/rejected": -361.24609375,
272
+ "loss": 0.0393,
273
+ "rewards/accuracies": 0.637499988079071,
274
+ "rewards/chosen": -0.6607332825660706,
275
+ "rewards/margins": 0.31646376848220825,
276
+ "rewards/rejected": -0.977196991443634,
277
  "step": 170
278
  },
279
  {
280
+ "epoch": 0.38,
281
+ "learning_rate": 3.9247834624635404e-07,
282
+ "logits/chosen": -2.561448812484741,
283
+ "logits/rejected": -2.5715978145599365,
284
+ "logps/chosen": -352.5904846191406,
285
+ "logps/rejected": -386.1707763671875,
286
+ "loss": 0.0355,
287
+ "rewards/accuracies": 0.675000011920929,
288
+ "rewards/chosen": -0.8926581144332886,
289
+ "rewards/margins": 0.39244139194488525,
290
+ "rewards/rejected": -1.2850996255874634,
291
  "step": 180
292
  },
293
  {
294
+ "epoch": 0.4,
295
+ "learning_rate": 3.7710310482256523e-07,
296
+ "logits/chosen": -2.510988473892212,
297
+ "logits/rejected": -2.493633985519409,
298
+ "logps/chosen": -343.94232177734375,
299
+ "logps/rejected": -387.73297119140625,
300
+ "loss": 0.0284,
301
+ "rewards/accuracies": 0.71875,
302
+ "rewards/chosen": -1.0100765228271484,
303
+ "rewards/margins": 0.5000672340393066,
304
+ "rewards/rejected": -1.5101438760757446,
305
  "step": 190
306
  },
307
  {
308
+ "epoch": 0.42,
309
+ "learning_rate": 3.610497133404795e-07,
310
+ "logits/chosen": -2.456496238708496,
311
+ "logits/rejected": -2.442700147628784,
312
+ "logps/chosen": -379.01141357421875,
313
+ "logps/rejected": -432.1864318847656,
314
+ "loss": 0.0237,
315
+ "rewards/accuracies": 0.643750011920929,
316
+ "rewards/chosen": -1.3147755861282349,
317
+ "rewards/margins": 0.4870442748069763,
318
+ "rewards/rejected": -1.8018200397491455,
319
  "step": 200
320
  },
321
  {
322
+ "epoch": 0.42,
323
+ "eval_logits/chosen": -2.5105109214782715,
324
+ "eval_logits/rejected": -2.496497392654419,
325
+ "eval_logps/chosen": -381.2964172363281,
326
+ "eval_logps/rejected": -441.2099304199219,
327
+ "eval_loss": 0.026758365333080292,
328
+ "eval_rewards/accuracies": 0.66796875,
329
+ "eval_rewards/chosen": -1.242567539215088,
330
+ "eval_rewards/margins": 0.5959988832473755,
331
+ "eval_rewards/rejected": -1.8385663032531738,
332
+ "eval_runtime": 53.7251,
333
+ "eval_samples_per_second": 37.227,
334
+ "eval_steps_per_second": 0.596,
335
+ "step": 200
336
  },
337
  {
338
+ "epoch": 0.44,
339
+ "learning_rate": 3.4440382358952115e-07,
340
+ "logits/chosen": -2.464259624481201,
341
+ "logits/rejected": -2.4774632453918457,
342
+ "logps/chosen": -408.1732482910156,
343
+ "logps/rejected": -461.95947265625,
344
+ "loss": 0.0259,
345
  "rewards/accuracies": 0.768750011920929,
346
+ "rewards/chosen": -1.0447111129760742,
347
+ "rewards/margins": 0.6814562678337097,
348
+ "rewards/rejected": -1.7261674404144287,
349
+ "step": 210
350
+ },
351
+ {
352
+ "epoch": 0.46,
353
+ "learning_rate": 3.272542485937368e-07,
354
+ "logits/chosen": -2.454061985015869,
355
+ "logits/rejected": -2.4306459426879883,
356
+ "logps/chosen": -375.48541259765625,
357
+ "logps/rejected": -397.7698059082031,
358
+ "loss": 0.0304,
359
+ "rewards/accuracies": 0.6499999761581421,
360
+ "rewards/chosen": -1.1511690616607666,
361
+ "rewards/margins": 0.46213898062705994,
362
+ "rewards/rejected": -1.6133081912994385,
363
  "step": 220
364
  },
365
  {
366
+ "epoch": 0.48,
367
+ "learning_rate": 3.096924887558854e-07,
368
+ "logits/chosen": -2.4601683616638184,
369
+ "logits/rejected": -2.445401430130005,
370
+ "logps/chosen": -377.16510009765625,
371
+ "logps/rejected": -408.50640869140625,
372
+ "loss": 0.0273,
373
+ "rewards/accuracies": 0.675000011920929,
374
+ "rewards/chosen": -1.183936595916748,
375
+ "rewards/margins": 0.452188640832901,
376
+ "rewards/rejected": -1.6361253261566162,
377
  "step": 230
378
  },
379
  {
380
+ "epoch": 0.5,
381
+ "learning_rate": 2.9181224366319943e-07,
382
+ "logits/chosen": -2.386798620223999,
383
+ "logits/rejected": -2.3921284675598145,
384
+ "logps/chosen": -433.074951171875,
385
+ "logps/rejected": -453.2076110839844,
386
+ "loss": 0.0283,
387
+ "rewards/accuracies": 0.6812499761581421,
388
+ "rewards/chosen": -1.21742844581604,
389
+ "rewards/margins": 0.5803453326225281,
390
+ "rewards/rejected": -1.7977739572525024,
391
  "step": 240
392
  },
393
  {
394
+ "epoch": 0.52,
395
+ "learning_rate": 2.7370891215954565e-07,
396
+ "logits/chosen": -2.4028992652893066,
397
+ "logits/rejected": -2.392509937286377,
398
+ "logps/chosen": -390.3121032714844,
399
+ "logps/rejected": -434.3692321777344,
400
+ "loss": 0.0275,
401
+ "rewards/accuracies": 0.7124999761581421,
402
+ "rewards/chosen": -1.207838535308838,
403
+ "rewards/margins": 0.5656585097312927,
404
+ "rewards/rejected": -1.773497223854065,
405
  "step": 250
406
  },
407
  {
408
+ "epoch": 0.54,
409
+ "learning_rate": 2.55479083351317e-07,
410
+ "logits/chosen": -2.3397555351257324,
411
+ "logits/rejected": -2.3070874214172363,
412
+ "logps/chosen": -451.30047607421875,
413
+ "logps/rejected": -458.3089294433594,
414
+ "loss": 0.0239,
415
+ "rewards/accuracies": 0.731249988079071,
416
+ "rewards/chosen": -1.6053054332733154,
417
+ "rewards/margins": 0.562503457069397,
418
+ "rewards/rejected": -2.167808771133423,
419
  "step": 260
420
  },
421
  {
422
+ "epoch": 0.56,
423
+ "learning_rate": 2.3722002126275822e-07,
424
+ "logits/chosen": -2.328000068664551,
425
+ "logits/rejected": -2.298135280609131,
426
+ "logps/chosen": -414.98504638671875,
427
+ "logps/rejected": -454.4877014160156,
428
+ "loss": 0.0259,
429
+ "rewards/accuracies": 0.706250011920929,
430
+ "rewards/chosen": -1.4466311931610107,
431
+ "rewards/margins": 0.555857241153717,
432
+ "rewards/rejected": -2.002488374710083,
433
  "step": 270
434
  },
435
  {
436
+ "epoch": 0.59,
437
+ "learning_rate": 2.19029145890313e-07,
438
+ "logits/chosen": -2.3714160919189453,
439
+ "logits/rejected": -2.3525238037109375,
440
+ "logps/chosen": -457.384033203125,
441
+ "logps/rejected": -497.83660888671875,
442
+ "loss": 0.0262,
443
+ "rewards/accuracies": 0.71875,
444
+ "rewards/chosen": -1.3860464096069336,
445
+ "rewards/margins": 0.6043224930763245,
446
+ "rewards/rejected": -1.9903688430786133,
447
  "step": 280
448
  },
449
  {
450
+ "epoch": 0.61,
451
+ "learning_rate": 2.0100351342479216e-07,
452
+ "logits/chosen": -2.3231723308563232,
453
+ "logits/rejected": -2.3119874000549316,
454
+ "logps/chosen": -416.6573791503906,
455
+ "logps/rejected": -427.49945068359375,
456
+ "loss": 0.0251,
457
+ "rewards/accuracies": 0.6937500238418579,
458
+ "rewards/chosen": -1.4526689052581787,
459
+ "rewards/margins": 0.5181004405021667,
460
+ "rewards/rejected": -1.9707695245742798,
461
  "step": 290
462
  },
463
  {
464
+ "epoch": 0.63,
465
+ "learning_rate": 1.8323929841460178e-07,
466
+ "logits/chosen": -2.3192086219787598,
467
+ "logits/rejected": -2.2759287357330322,
468
+ "logps/chosen": -440.58935546875,
469
+ "logps/rejected": -448.8992614746094,
470
+ "loss": 0.0248,
471
+ "rewards/accuracies": 0.706250011920929,
472
+ "rewards/chosen": -1.4383071660995483,
473
+ "rewards/margins": 0.5324681997299194,
474
+ "rewards/rejected": -1.9707752466201782,
475
+ "step": 300
476
+ },
477
+ {
478
+ "epoch": 0.63,
479
+ "eval_logits/chosen": -2.339747190475464,
480
+ "eval_logits/rejected": -2.319640874862671,
481
+ "eval_logps/chosen": -395.36798095703125,
482
+ "eval_logps/rejected": -460.886962890625,
483
+ "eval_loss": 0.025441216304898262,
484
+ "eval_rewards/accuracies": 0.69140625,
485
+ "eval_rewards/chosen": -1.383283257484436,
486
+ "eval_rewards/margins": 0.6520535945892334,
487
+ "eval_rewards/rejected": -2.03533673286438,
488
+ "eval_runtime": 53.7578,
489
+ "eval_samples_per_second": 37.204,
490
+ "eval_steps_per_second": 0.595,
491
  "step": 300
492
  },
493
  {
494
+ "epoch": 0.65,
495
+ "learning_rate": 1.6583128063291573e-07,
496
+ "logits/chosen": -2.256671667098999,
497
+ "logits/rejected": -2.2072629928588867,
498
+ "logps/chosen": -441.985107421875,
499
+ "logps/rejected": -454.2076721191406,
500
+ "loss": 0.0255,
501
+ "rewards/accuracies": 0.6625000238418579,
502
+ "rewards/chosen": -1.5002824068069458,
503
+ "rewards/margins": 0.5682298541069031,
504
+ "rewards/rejected": -2.068512439727783,
505
  "step": 310
506
  },
507
  {
508
+ "epoch": 0.67,
509
+ "learning_rate": 1.488723393865766e-07,
510
+ "logits/chosen": -2.318877696990967,
511
+ "logits/rejected": -2.311671495437622,
512
+ "logps/chosen": -368.1661071777344,
513
+ "logps/rejected": -419.93499755859375,
514
+ "loss": 0.0257,
515
+ "rewards/accuracies": 0.731249988079071,
516
+ "rewards/chosen": -1.1376068592071533,
517
+ "rewards/margins": 0.6408455967903137,
518
+ "rewards/rejected": -1.7784525156021118,
519
  "step": 320
520
  },
521
  {
522
+ "epoch": 0.69,
523
+ "learning_rate": 1.3245295796480788e-07,
524
+ "logits/chosen": -2.307875156402588,
525
+ "logits/rejected": -2.287741184234619,
526
+ "logps/chosen": -388.35369873046875,
527
+ "logps/rejected": -418.2744140625,
528
+ "loss": 0.0264,
529
+ "rewards/accuracies": 0.6312500238418579,
530
+ "rewards/chosen": -1.2828859090805054,
531
+ "rewards/margins": 0.3729013502597809,
532
+ "rewards/rejected": -1.6557872295379639,
533
  "step": 330
534
  },
535
  {
536
+ "epoch": 0.71,
537
+ "learning_rate": 1.1666074087171627e-07,
538
+ "logits/chosen": -2.3257226943969727,
539
+ "logits/rejected": -2.3047900199890137,
540
+ "logps/chosen": -400.9947204589844,
541
+ "logps/rejected": -466.18438720703125,
542
+ "loss": 0.0244,
543
+ "rewards/accuracies": 0.6812499761581421,
544
+ "rewards/chosen": -1.4033421277999878,
545
+ "rewards/margins": 0.6501402854919434,
546
+ "rewards/rejected": -2.0534825325012207,
547
  "step": 340
548
  },
549
  {
550
+ "epoch": 0.73,
551
+ "learning_rate": 1.0157994641835734e-07,
552
+ "logits/chosen": -2.2791500091552734,
553
+ "logits/rejected": -2.228987216949463,
554
+ "logps/chosen": -438.2213439941406,
555
+ "logps/rejected": -482.71185302734375,
556
+ "loss": 0.0231,
557
+ "rewards/accuracies": 0.737500011920929,
558
+ "rewards/chosen": -1.380692720413208,
559
+ "rewards/margins": 0.6693974733352661,
560
+ "rewards/rejected": -2.0500900745391846,
561
  "step": 350
562
  },
563
  {
564
+ "epoch": 0.75,
565
+ "learning_rate": 8.729103716819111e-08,
566
+ "logits/chosen": -2.3203213214874268,
567
+ "logits/rejected": -2.2689433097839355,
568
+ "logps/chosen": -447.54644775390625,
569
+ "logps/rejected": -481.5086364746094,
570
+ "loss": 0.0244,
571
+ "rewards/accuracies": 0.7437499761581421,
572
+ "rewards/chosen": -1.4591137170791626,
573
+ "rewards/margins": 0.7254992723464966,
574
+ "rewards/rejected": -2.184612989425659,
575
  "step": 360
576
  },
577
  {
578
+ "epoch": 0.77,
579
+ "learning_rate": 7.387025063449081e-08,
580
+ "logits/chosen": -2.2618608474731445,
581
+ "logits/rejected": -2.2370240688323975,
582
+ "logps/chosen": -463.3082580566406,
583
+ "logps/rejected": -520.4660034179688,
584
+ "loss": 0.0211,
585
+ "rewards/accuracies": 0.71875,
586
+ "rewards/chosen": -1.6034772396087646,
587
+ "rewards/margins": 0.8156415820121765,
588
+ "rewards/rejected": -2.419118881225586,
589
  "step": 370
590
  },
591
  {
592
+ "epoch": 0.79,
593
+ "learning_rate": 6.138919252022435e-08,
594
+ "logits/chosen": -2.198676347732544,
595
+ "logits/rejected": -2.1428768634796143,
596
+ "logps/chosen": -469.5831604003906,
597
+ "logps/rejected": -499.9231872558594,
598
+ "loss": 0.0227,
599
+ "rewards/accuracies": 0.706250011920929,
600
+ "rewards/chosen": -1.7702577114105225,
601
+ "rewards/margins": 0.7163289189338684,
602
+ "rewards/rejected": -2.4865870475769043,
603
  "step": 380
604
  },
605
  {
606
+ "epoch": 0.82,
607
+ "learning_rate": 4.991445467064689e-08,
608
+ "logits/chosen": -2.2201147079467773,
609
+ "logits/rejected": -2.207123041152954,
610
+ "logps/chosen": -446.3898010253906,
611
+ "logps/rejected": -482.3876953125,
612
+ "loss": 0.0202,
613
+ "rewards/accuracies": 0.65625,
614
+ "rewards/chosen": -1.8155654668807983,
615
+ "rewards/margins": 0.5150431394577026,
616
+ "rewards/rejected": -2.330608367919922,
617
  "step": 390
618
  },
619
  {
620
+ "epoch": 0.84,
621
+ "learning_rate": 3.9507259776993954e-08,
622
+ "logits/chosen": -2.2568085193634033,
623
+ "logits/rejected": -2.2447478771209717,
624
+ "logps/chosen": -449.7113342285156,
625
+ "logps/rejected": -495.636474609375,
626
+ "loss": 0.0194,
627
+ "rewards/accuracies": 0.706250011920929,
628
+ "rewards/chosen": -1.930235505104065,
629
+ "rewards/margins": 0.5124340653419495,
630
+ "rewards/rejected": -2.442669630050659,
631
+ "step": 400
632
+ },
633
+ {
634
+ "epoch": 0.84,
635
+ "eval_logits/chosen": -2.262547254562378,
636
+ "eval_logits/rejected": -2.237802267074585,
637
+ "eval_logps/chosen": -439.0294189453125,
638
+ "eval_logps/rejected": -509.7351379394531,
639
+ "eval_loss": 0.019433461129665375,
640
+ "eval_rewards/accuracies": 0.703125,
641
+ "eval_rewards/chosen": -1.8198977708816528,
642
+ "eval_rewards/margins": 0.7039206027984619,
643
+ "eval_rewards/rejected": -2.523818254470825,
644
+ "eval_runtime": 53.7532,
645
+ "eval_samples_per_second": 37.207,
646
+ "eval_steps_per_second": 0.595,
647
  "step": 400
648
  },
649
+ {
650
+ "epoch": 0.86,
651
+ "learning_rate": 3.022313472693447e-08,
652
+ "logits/chosen": -2.219531536102295,
653
+ "logits/rejected": -2.177171468734741,
654
+ "logps/chosen": -435.68316650390625,
655
+ "logps/rejected": -480.1519470214844,
656
+ "loss": 0.0193,
657
+ "rewards/accuracies": 0.675000011920929,
658
+ "rewards/chosen": -1.8031377792358398,
659
+ "rewards/margins": 0.5790697336196899,
660
+ "rewards/rejected": -2.3822076320648193,
661
+ "step": 410
662
+ },
663
+ {
664
+ "epoch": 0.88,
665
+ "learning_rate": 2.2111614344599684e-08,
666
+ "logits/chosen": -2.261929988861084,
667
+ "logits/rejected": -2.20975661277771,
668
+ "logps/chosen": -446.5043029785156,
669
+ "logps/rejected": -468.5169982910156,
670
+ "loss": 0.0187,
671
+ "rewards/accuracies": 0.6499999761581421,
672
+ "rewards/chosen": -1.7549480199813843,
673
+ "rewards/margins": 0.5446823239326477,
674
+ "rewards/rejected": -2.2996301651000977,
675
+ "step": 420
676
+ },
677
+ {
678
+ "epoch": 0.9,
679
+ "learning_rate": 1.521597710086439e-08,
680
+ "logits/chosen": -2.2168891429901123,
681
+ "logits/rejected": -2.1971139907836914,
682
+ "logps/chosen": -465.6109313964844,
683
+ "logps/rejected": -488.16912841796875,
684
+ "loss": 0.0192,
685
+ "rewards/accuracies": 0.7562500238418579,
686
+ "rewards/chosen": -1.8082774877548218,
687
+ "rewards/margins": 0.5722709894180298,
688
+ "rewards/rejected": -2.3805484771728516,
689
+ "step": 430
690
+ },
691
+ {
692
+ "epoch": 0.92,
693
+ "learning_rate": 9.57301420397924e-09,
694
+ "logits/chosen": -2.231231212615967,
695
+ "logits/rejected": -2.209394931793213,
696
+ "logps/chosen": -448.60986328125,
697
+ "logps/rejected": -509.45770263671875,
698
+ "loss": 0.0187,
699
+ "rewards/accuracies": 0.6875,
700
+ "rewards/chosen": -1.8875834941864014,
701
+ "rewards/margins": 0.6209200620651245,
702
+ "rewards/rejected": -2.5085036754608154,
703
+ "step": 440
704
+ },
705
+ {
706
+ "epoch": 0.94,
707
+ "learning_rate": 5.212833302556258e-09,
708
+ "logits/chosen": -2.2720608711242676,
709
+ "logits/rejected": -2.2489547729492188,
710
+ "logps/chosen": -436.8863830566406,
711
+ "logps/rejected": -479.250244140625,
712
+ "loss": 0.0196,
713
+ "rewards/accuracies": 0.675000011920929,
714
+ "rewards/chosen": -1.7134939432144165,
715
+ "rewards/margins": 0.5517303347587585,
716
+ "rewards/rejected": -2.2652244567871094,
717
+ "step": 450
718
+ },
719
+ {
720
+ "epoch": 0.96,
721
+ "learning_rate": 2.158697848236607e-09,
722
+ "logits/chosen": -2.2524914741516113,
723
+ "logits/rejected": -2.1975879669189453,
724
+ "logps/chosen": -454.9847717285156,
725
+ "logps/rejected": -471.0829162597656,
726
+ "loss": 0.0196,
727
+ "rewards/accuracies": 0.6937500238418579,
728
+ "rewards/chosen": -1.871111273765564,
729
+ "rewards/margins": 0.5668479204177856,
730
+ "rewards/rejected": -2.4379591941833496,
731
+ "step": 460
732
+ },
733
+ {
734
+ "epoch": 0.98,
735
+ "learning_rate": 4.269029751107489e-10,
736
+ "logits/chosen": -2.2418127059936523,
737
+ "logits/rejected": -2.2091026306152344,
738
+ "logps/chosen": -438.8106384277344,
739
+ "logps/rejected": -487.1082458496094,
740
+ "loss": 0.0201,
741
+ "rewards/accuracies": 0.71875,
742
+ "rewards/chosen": -1.6759856939315796,
743
+ "rewards/margins": 0.6907237768173218,
744
+ "rewards/rejected": -2.3667094707489014,
745
+ "step": 470
746
+ },
747
  {
748
  "epoch": 1.0,
749
+ "step": 478,
750
  "total_flos": 0.0,
751
+ "train_loss": 0.03868693582495386,
752
+ "train_runtime": 3950.8818,
753
+ "train_samples_per_second": 15.474,
754
+ "train_steps_per_second": 0.121
755
  }
756
  ],
757
  "logging_steps": 10,
758
+ "max_steps": 478,
759
  "num_train_epochs": 1,
760
  "save_steps": 1000,
761
  "total_flos": 0.0,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:19e290310c5cdc37c4c14a531cf89528ab0416cf966910b48c267456558d7457
3
- size 6456
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa0d4dc1074b63bfb44f5057f5f2e0432b683d2eeab8a75dfa36408b5eb7136d
3
+ size 5944