RikkiXu commited on
Commit
e3f2a18
1 Parent(s): f36cc5d

Model save

Browse files
README.md CHANGED
@@ -1,4 +1,5 @@
1
  ---
 
2
  tags:
3
  - trl
4
  - dpo
@@ -13,7 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
13
 
14
  # zephyr-7b-dpo-full
15
 
16
- This model was trained from scratch on the None dataset.
17
 
18
  ## Model description
19
 
@@ -32,7 +33,7 @@ More information needed
32
  ### Training hyperparameters
33
 
34
  The following hyperparameters were used during training:
35
- - learning_rate: 1e-08
36
  - train_batch_size: 8
37
  - eval_batch_size: 8
38
  - seed: 42
@@ -44,7 +45,7 @@ The following hyperparameters were used during training:
44
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
45
  - lr_scheduler_type: cosine
46
  - lr_scheduler_warmup_ratio: 0.1
47
- - num_epochs: 2
48
 
49
  ### Training results
50
 
 
1
  ---
2
+ base_model: princeton-nlp/Mistral-7B-Base-SFT-DPO
3
  tags:
4
  - trl
5
  - dpo
 
14
 
15
  # zephyr-7b-dpo-full
16
 
17
+ This model is a fine-tuned version of [princeton-nlp/Mistral-7B-Base-SFT-DPO](https://huggingface.co/princeton-nlp/Mistral-7B-Base-SFT-DPO) on the None dataset.
18
 
19
  ## Model description
20
 
 
33
  ### Training hyperparameters
34
 
35
  The following hyperparameters were used during training:
36
+ - learning_rate: 5e-07
37
  - train_batch_size: 8
38
  - eval_batch_size: 8
39
  - seed: 42
 
45
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
46
  - lr_scheduler_type: cosine
47
  - lr_scheduler_warmup_ratio: 0.1
48
+ - num_epochs: 1
49
 
50
  ### Training results
51
 
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
- "epoch": 2.0,
3
- "train_loss": 0.26489045179408527,
4
- "train_runtime": 4520.7899,
5
- "train_samples": 47095,
6
- "train_samples_per_second": 20.835,
7
- "train_steps_per_second": 0.081
8
  }
 
1
  {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.33391942269093283,
4
+ "train_runtime": 5319.9853,
5
+ "train_samples": 47302,
6
+ "train_samples_per_second": 8.891,
7
+ "train_steps_per_second": 0.035
8
  }
config.json CHANGED
@@ -20,7 +20,7 @@
20
  "sliding_window": 4096,
21
  "tie_word_embeddings": false,
22
  "torch_dtype": "bfloat16",
23
- "transformers_version": "4.41.1",
24
  "use_cache": false,
25
  "vocab_size": 32000
26
  }
 
20
  "sliding_window": 4096,
21
  "tie_word_embeddings": false,
22
  "torch_dtype": "bfloat16",
23
+ "transformers_version": "4.39.3",
24
  "use_cache": false,
25
  "vocab_size": 32000
26
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:89021f925795319ec6cd79c57490ff100b4d209128279330c40d869a1257ea07
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:082f207ea835a592e1e8eb2cd383fe7080164e426d45e909c8d42aca68d2441d
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:cc9bb6d1ff6e5e594487f490ec146b7666201b9e396d8c36acadcf28b8944e2e
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f1ab93164931c699c31d8f9a6662aa4d21ae5163cf4ca656ec82228d11a555b
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d64a4a2eb8fd994c6c2e00218546b04c07ec10f9918e36094d7621932e29d265
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:241e2b2864b464b12cb35da89659de3b3cce018b44fd5b62ae428621c0077ca8
3
  size 4540516344
runs/Jun21_00-12-13_n136-100-194/events.out.tfevents.1718899955.n136-100-194.494165.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c97c12876d1c4d23198b5d9f91d061f458ebd51aac69372e75f566ce403f78b3
3
- size 12321
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30db6263136339554ee965a3c30d4aa91ded54ecf93706de542e54b6c0c8595d
3
+ size 18155
tokenizer.json CHANGED
@@ -134,7 +134,6 @@
134
  "end_of_word_suffix": null,
135
  "fuse_unk": true,
136
  "byte_fallback": true,
137
- "ignore_merges": false,
138
  "vocab": {
139
  "<unk>": 0,
140
  "<s>": 1,
 
134
  "end_of_word_suffix": null,
135
  "fuse_unk": true,
136
  "byte_fallback": true,
 
137
  "vocab": {
138
  "<unk>": 0,
139
  "<s>": 1,
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
- "epoch": 2.0,
3
- "train_loss": 0.26489045179408527,
4
- "train_runtime": 4520.7899,
5
- "train_samples": 47095,
6
- "train_samples_per_second": 20.835,
7
- "train_steps_per_second": 0.081
8
  }
 
1
  {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.33391942269093283,
4
+ "train_runtime": 5319.9853,
5
+ "train_samples": 47302,
6
+ "train_samples_per_second": 8.891,
7
+ "train_steps_per_second": 0.035
8
  }
trainer_state.json CHANGED
@@ -1,22 +1,22 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 2.0,
5
  "eval_steps": 500,
6
- "global_step": 368,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.01,
13
- "grad_norm": 2733.5875966596914,
14
- "learning_rate": 2.702702702702703e-10,
15
- "logits/chosen": -1.3332719802856445,
16
- "logits/rejected": -1.246394395828247,
17
- "logps/chosen": -286.9539794921875,
18
- "logps/rejected": -263.3782958984375,
19
- "loss": 0.7283,
20
  "rewards/accuracies": 0.0,
21
  "rewards/chosen": 0.0,
22
  "rewards/margins": 0.0,
@@ -25,558 +25,288 @@
25
  },
26
  {
27
  "epoch": 0.05,
28
- "grad_norm": 3426.28374639058,
29
- "learning_rate": 2.702702702702703e-09,
30
- "logits/chosen": -1.617490530014038,
31
- "logits/rejected": -1.3964743614196777,
32
- "logps/chosen": -342.53607177734375,
33
- "logps/rejected": -294.5452575683594,
34
- "loss": 0.9019,
35
- "rewards/accuracies": 0.4375,
36
- "rewards/chosen": -0.0025859144516289234,
37
- "rewards/margins": 0.014665775932371616,
38
- "rewards/rejected": -0.017251690849661827,
39
  "step": 10
40
  },
41
  {
42
  "epoch": 0.11,
43
- "grad_norm": 3068.517544136213,
44
- "learning_rate": 5.405405405405406e-09,
45
- "logits/chosen": -1.4905732870101929,
46
- "logits/rejected": -1.3132953643798828,
47
- "logps/chosen": -314.7499084472656,
48
- "logps/rejected": -279.27752685546875,
49
- "loss": 0.9225,
50
- "rewards/accuracies": 0.4937500059604645,
51
- "rewards/chosen": 0.03222837299108505,
52
- "rewards/margins": -0.024351513013243675,
53
- "rewards/rejected": 0.05657988786697388,
54
  "step": 20
55
  },
56
  {
57
  "epoch": 0.16,
58
- "grad_norm": 3019.197796031677,
59
- "learning_rate": 8.108108108108109e-09,
60
- "logits/chosen": -1.5479624271392822,
61
- "logits/rejected": -1.3802028894424438,
62
- "logps/chosen": -324.89044189453125,
63
- "logps/rejected": -286.2395324707031,
64
- "loss": 0.9562,
65
- "rewards/accuracies": 0.5218750238418579,
66
- "rewards/chosen": 0.031223665922880173,
67
- "rewards/margins": -0.0476018562912941,
68
- "rewards/rejected": 0.07882551848888397,
69
  "step": 30
70
  },
71
  {
72
  "epoch": 0.22,
73
- "grad_norm": 2861.837005400921,
74
- "learning_rate": 9.997973265157192e-09,
75
- "logits/chosen": -1.5354044437408447,
76
- "logits/rejected": -1.3576419353485107,
77
- "logps/chosen": -325.43408203125,
78
- "logps/rejected": -285.6204528808594,
79
- "loss": 0.9309,
80
- "rewards/accuracies": 0.484375,
81
- "rewards/chosen": -0.08140890300273895,
82
- "rewards/margins": -0.13951030373573303,
83
- "rewards/rejected": 0.05810140445828438,
84
  "step": 40
85
  },
86
  {
87
  "epoch": 0.27,
88
- "grad_norm": 3285.7135249709568,
89
- "learning_rate": 9.961988113473708e-09,
90
- "logits/chosen": -1.534355640411377,
91
- "logits/rejected": -1.3875898122787476,
92
- "logps/chosen": -337.02044677734375,
93
- "logps/rejected": -297.35101318359375,
94
- "loss": 0.88,
95
- "rewards/accuracies": 0.546875,
96
- "rewards/chosen": 0.0009558796882629395,
97
- "rewards/margins": 0.08057532459497452,
98
- "rewards/rejected": -0.07961944490671158,
99
  "step": 50
100
  },
101
  {
102
- "epoch": 0.33,
103
- "grad_norm": 2547.1665545835035,
104
- "learning_rate": 9.881337335184878e-09,
105
- "logits/chosen": -1.5822935104370117,
106
- "logits/rejected": -1.4333903789520264,
107
- "logps/chosen": -319.79644775390625,
108
- "logps/rejected": -285.0381164550781,
109
- "loss": 0.8105,
110
- "rewards/accuracies": 0.6156250238418579,
111
- "rewards/chosen": 0.09285839647054672,
112
- "rewards/margins": 0.40408092737197876,
113
- "rewards/rejected": -0.31122252345085144,
114
  "step": 60
115
  },
116
  {
117
  "epoch": 0.38,
118
- "grad_norm": 3178.8313195657583,
119
- "learning_rate": 9.756746912994832e-09,
120
- "logits/chosen": -1.5119212865829468,
121
- "logits/rejected": -1.350838541984558,
122
- "logps/chosen": -312.1349182128906,
123
- "logps/rejected": -275.08660888671875,
124
- "loss": 0.7993,
125
- "rewards/accuracies": 0.5874999761581421,
126
- "rewards/chosen": -0.05482473224401474,
127
- "rewards/margins": 0.165395587682724,
128
- "rewards/rejected": -0.22022032737731934,
129
  "step": 70
130
  },
131
  {
132
  "epoch": 0.43,
133
- "grad_norm": 2596.199281277795,
134
- "learning_rate": 9.589338354885628e-09,
135
- "logits/chosen": -1.5992329120635986,
136
- "logits/rejected": -1.4463211297988892,
137
- "logps/chosen": -323.2821960449219,
138
- "logps/rejected": -288.0993347167969,
139
- "loss": 0.7772,
140
- "rewards/accuracies": 0.574999988079071,
141
- "rewards/chosen": 0.20231203734874725,
142
- "rewards/margins": 0.4638887345790863,
143
- "rewards/rejected": -0.26157671213150024,
144
  "step": 80
145
  },
146
  {
147
  "epoch": 0.49,
148
- "grad_norm": 2404.162696635418,
149
- "learning_rate": 9.380618598797472e-09,
150
- "logits/chosen": -1.6108148097991943,
151
- "logits/rejected": -1.4147026538848877,
152
- "logps/chosen": -319.95526123046875,
153
- "logps/rejected": -281.7666015625,
154
- "loss": 0.7649,
155
- "rewards/accuracies": 0.640625,
156
- "rewards/chosen": 0.2500740885734558,
157
- "rewards/margins": 0.5574880838394165,
158
- "rewards/rejected": -0.3074139356613159,
159
  "step": 90
160
  },
161
  {
162
  "epoch": 0.54,
163
- "grad_norm": 2500.271614922207,
164
- "learning_rate": 9.132466447838596e-09,
165
- "logits/chosen": -1.542976975440979,
166
- "logits/rejected": -1.3676128387451172,
167
- "logps/chosen": -321.9007263183594,
168
- "logps/rejected": -282.65899658203125,
169
- "loss": 0.7305,
170
- "rewards/accuracies": 0.659375011920929,
171
- "rewards/chosen": 0.4254188537597656,
172
- "rewards/margins": 0.763160228729248,
173
- "rewards/rejected": -0.3377414047718048,
174
  "step": 100
175
  },
176
  {
177
- "epoch": 0.6,
178
- "grad_norm": 2525.572173445101,
179
- "learning_rate": 8.847115658129039e-09,
180
- "logits/chosen": -1.512939453125,
181
- "logits/rejected": -1.3849382400512695,
182
- "logps/chosen": -318.14813232421875,
183
- "logps/rejected": -287.1947937011719,
184
- "loss": 0.7164,
185
- "rewards/accuracies": 0.628125011920929,
186
- "rewards/chosen": 0.3916184604167938,
187
- "rewards/margins": 0.657262921333313,
188
- "rewards/rejected": -0.265644371509552,
189
  "step": 110
190
  },
191
  {
192
  "epoch": 0.65,
193
- "grad_norm": 2498.220455730566,
194
- "learning_rate": 8.527134831514116e-09,
195
- "logits/chosen": -1.5739517211914062,
196
- "logits/rejected": -1.41860032081604,
197
- "logps/chosen": -331.3175354003906,
198
- "logps/rejected": -297.8718566894531,
199
- "loss": 0.7018,
200
- "rewards/accuracies": 0.637499988079071,
201
- "rewards/chosen": 0.6175383925437927,
202
- "rewards/margins": 0.6906081438064575,
203
- "rewards/rejected": -0.07306969165802002,
204
  "step": 120
205
  },
206
  {
207
- "epoch": 0.71,
208
- "grad_norm": 2170.854176631694,
209
- "learning_rate": 8.175404294144481e-09,
210
- "logits/chosen": -1.616276502609253,
211
- "logits/rejected": -1.429518699645996,
212
- "logps/chosen": -317.1609802246094,
213
- "logps/rejected": -271.5557861328125,
214
- "loss": 0.6719,
215
- "rewards/accuracies": 0.6625000238418579,
216
- "rewards/chosen": 0.6955646872520447,
217
- "rewards/margins": 0.8560658693313599,
218
- "rewards/rejected": -0.160501167178154,
219
  "step": 130
220
  },
221
  {
222
  "epoch": 0.76,
223
- "grad_norm": 2147.543587634845,
224
- "learning_rate": 7.79509016905158e-09,
225
- "logits/chosen": -1.5726101398468018,
226
- "logits/rejected": -1.4245671033859253,
227
- "logps/chosen": -331.12109375,
228
- "logps/rejected": -294.2488098144531,
229
- "loss": 0.6686,
230
- "rewards/accuracies": 0.6968749761581421,
231
- "rewards/chosen": 0.9418653249740601,
232
- "rewards/margins": 0.9832620620727539,
233
- "rewards/rejected": -0.04139674827456474,
234
  "step": 140
235
  },
236
  {
237
- "epoch": 0.82,
238
- "grad_norm": 2298.0130679506387,
239
- "learning_rate": 7.389615876105773e-09,
240
- "logits/chosen": -1.5536715984344482,
241
- "logits/rejected": -1.4254592657089233,
242
- "logps/chosen": -314.55267333984375,
243
- "logps/rejected": -291.81536865234375,
244
- "loss": 0.6793,
245
- "rewards/accuracies": 0.675000011920929,
246
- "rewards/chosen": 1.0258944034576416,
247
- "rewards/margins": 0.981005072593689,
248
- "rewards/rejected": 0.04488936811685562,
249
  "step": 150
250
  },
251
  {
252
- "epoch": 0.87,
253
- "grad_norm": 2220.286727308493,
254
- "learning_rate": 6.962631315901861e-09,
255
- "logits/chosen": -1.5181314945220947,
256
- "logits/rejected": -1.4019381999969482,
257
- "logps/chosen": -318.02752685546875,
258
- "logps/rejected": -291.03936767578125,
259
- "loss": 0.6742,
260
- "rewards/accuracies": 0.628125011920929,
261
- "rewards/chosen": 0.9784830808639526,
262
- "rewards/margins": 0.819505512714386,
263
- "rewards/rejected": 0.15897764265537262,
264
  "step": 160
265
  },
266
  {
267
  "epoch": 0.92,
268
- "grad_norm": 2125.4983562302123,
269
- "learning_rate": 6.517980014965139e-09,
270
- "logits/chosen": -1.5958881378173828,
271
- "logits/rejected": -1.4071909189224243,
272
- "logps/chosen": -331.4378356933594,
273
- "logps/rejected": -289.5236511230469,
274
- "loss": 0.6456,
275
- "rewards/accuracies": 0.731249988079071,
276
- "rewards/chosen": 1.1029136180877686,
277
- "rewards/margins": 1.160766363143921,
278
- "rewards/rejected": -0.05785265564918518,
279
  "step": 170
280
  },
281
  {
282
- "epoch": 0.98,
283
- "grad_norm": 2116.9259500110184,
284
- "learning_rate": 6.059664528022266e-09,
285
- "logits/chosen": -1.5962104797363281,
286
- "logits/rejected": -1.445967197418213,
287
- "logps/chosen": -315.10467529296875,
288
- "logps/rejected": -276.73443603515625,
289
- "loss": 0.6191,
290
- "rewards/accuracies": 0.6812499761581421,
291
- "rewards/chosen": 1.123002052307129,
292
- "rewards/margins": 1.1910655498504639,
293
- "rewards/rejected": -0.0680634081363678,
294
  "step": 180
295
  },
296
  {
297
- "epoch": 1.03,
298
- "grad_norm": 2067.8979230397717,
299
- "learning_rate": 5.591810408770492e-09,
300
- "logits/chosen": -1.55275559425354,
301
- "logits/rejected": -1.3787180185317993,
302
- "logps/chosen": -315.572509765625,
303
- "logps/rejected": -278.71087646484375,
304
- "loss": 0.6052,
305
- "rewards/accuracies": 0.71875,
306
- "rewards/chosen": 1.2037546634674072,
307
- "rewards/margins": 1.2858160734176636,
308
- "rewards/rejected": -0.0820615291595459,
309
- "step": 190
310
- },
311
- {
312
- "epoch": 1.09,
313
- "grad_norm": 2203.6189854484824,
314
- "learning_rate": 5.118629073464423e-09,
315
- "logits/chosen": -1.5673738718032837,
316
- "logits/rejected": -1.3565856218338013,
317
- "logps/chosen": -325.91680908203125,
318
- "logps/rejected": -282.65869140625,
319
- "loss": 0.6024,
320
- "rewards/accuracies": 0.746874988079071,
321
- "rewards/chosen": 1.4020355939865112,
322
- "rewards/margins": 1.1880966424942017,
323
- "rewards/rejected": 0.21393892168998718,
324
- "step": 200
325
- },
326
- {
327
- "epoch": 1.14,
328
- "grad_norm": 2263.504332795979,
329
- "learning_rate": 4.644379891605983e-09,
330
- "logits/chosen": -1.611310601234436,
331
- "logits/rejected": -1.4343440532684326,
332
- "logps/chosen": -324.752197265625,
333
- "logps/rejected": -291.36102294921875,
334
- "loss": 0.5985,
335
- "rewards/accuracies": 0.699999988079071,
336
- "rewards/chosen": 1.2598803043365479,
337
- "rewards/margins": 1.2701908349990845,
338
- "rewards/rejected": -0.010310685262084007,
339
- "step": 210
340
- },
341
- {
342
- "epoch": 1.2,
343
- "grad_norm": 2324.309417872748,
344
- "learning_rate": 4.173331844980362e-09,
345
- "logits/chosen": -1.5291264057159424,
346
- "logits/rejected": -1.4033840894699097,
347
- "logps/chosen": -323.9982604980469,
348
- "logps/rejected": -293.4136047363281,
349
- "loss": 0.5948,
350
- "rewards/accuracies": 0.7281249761581421,
351
- "rewards/chosen": 1.1985571384429932,
352
- "rewards/margins": 1.105764627456665,
353
- "rewards/rejected": 0.09279236942529678,
354
- "step": 220
355
- },
356
- {
357
- "epoch": 1.25,
358
- "grad_norm": 2275.313721629071,
359
- "learning_rate": 3.7097251001664824e-09,
360
- "logits/chosen": -1.5342741012573242,
361
- "logits/rejected": -1.3754017353057861,
362
- "logps/chosen": -323.9897766113281,
363
- "logps/rejected": -287.0173645019531,
364
- "loss": 0.577,
365
- "rewards/accuracies": 0.71875,
366
- "rewards/chosen": 1.2511075735092163,
367
- "rewards/margins": 1.3087403774261475,
368
- "rewards/rejected": -0.05763290077447891,
369
- "step": 230
370
- },
371
- {
372
- "epoch": 1.3,
373
- "grad_norm": 2261.889907648677,
374
- "learning_rate": 3.2577328404292057e-09,
375
- "logits/chosen": -1.5497777462005615,
376
- "logits/rejected": -1.4208284616470337,
377
- "logps/chosen": -312.53802490234375,
378
- "logps/rejected": -285.97076416015625,
379
- "loss": 0.5741,
380
- "rewards/accuracies": 0.699999988079071,
381
- "rewards/chosen": 1.4049217700958252,
382
- "rewards/margins": 1.3057136535644531,
383
- "rewards/rejected": 0.09920807182788849,
384
- "step": 240
385
- },
386
- {
387
- "epoch": 1.36,
388
- "grad_norm": 2520.220972155196,
389
- "learning_rate": 2.821423700565763e-09,
390
- "logits/chosen": -1.5996572971343994,
391
- "logits/rejected": -1.4216984510421753,
392
- "logps/chosen": -350.76129150390625,
393
- "logps/rejected": -306.58831787109375,
394
- "loss": 0.5681,
395
- "rewards/accuracies": 0.762499988079071,
396
- "rewards/chosen": 1.537647008895874,
397
- "rewards/margins": 1.4706511497497559,
398
- "rewards/rejected": 0.06699595600366592,
399
- "step": 250
400
- },
401
- {
402
- "epoch": 1.41,
403
- "grad_norm": 2185.2135747623915,
404
- "learning_rate": 2.4047251428513483e-09,
405
- "logits/chosen": -1.61586594581604,
406
- "logits/rejected": -1.4618706703186035,
407
- "logps/chosen": -325.3050537109375,
408
- "logps/rejected": -291.10345458984375,
409
- "loss": 0.5977,
410
- "rewards/accuracies": 0.731249988079071,
411
- "rewards/chosen": 1.5201152563095093,
412
- "rewards/margins": 1.4326369762420654,
413
- "rewards/rejected": 0.08747831732034683,
414
- "step": 260
415
- },
416
- {
417
- "epoch": 1.47,
418
- "grad_norm": 1735.3140228188304,
419
- "learning_rate": 2.011388103757442e-09,
420
- "logits/chosen": -1.5243465900421143,
421
- "logits/rejected": -1.3802506923675537,
422
- "logps/chosen": -316.4330139160156,
423
- "logps/rejected": -285.81353759765625,
424
- "loss": 0.5429,
425
- "rewards/accuracies": 0.7406250238418579,
426
- "rewards/chosen": 1.550986647605896,
427
- "rewards/margins": 1.45218026638031,
428
- "rewards/rejected": 0.09880634397268295,
429
- "step": 270
430
- },
431
- {
432
- "epoch": 1.52,
433
- "grad_norm": 2121.5830866823144,
434
- "learning_rate": 1.644953229677474e-09,
435
- "logits/chosen": -1.6015859842300415,
436
- "logits/rejected": -1.4193016290664673,
437
- "logps/chosen": -326.1202087402344,
438
- "logps/rejected": -284.7384033203125,
439
- "loss": 0.5887,
440
- "rewards/accuracies": 0.746874988079071,
441
- "rewards/chosen": 1.580999732017517,
442
- "rewards/margins": 1.3716144561767578,
443
- "rewards/rejected": 0.20938535034656525,
444
- "step": 280
445
- },
446
- {
447
- "epoch": 1.58,
448
- "grad_norm": 2103.020456118981,
449
- "learning_rate": 1.308719005590957e-09,
450
- "logits/chosen": -1.509340524673462,
451
- "logits/rejected": -1.3944005966186523,
452
- "logps/chosen": -318.451416015625,
453
- "logps/rejected": -282.4563293457031,
454
- "loss": 0.5721,
455
- "rewards/accuracies": 0.731249988079071,
456
- "rewards/chosen": 1.603921890258789,
457
- "rewards/margins": 1.4211392402648926,
458
- "rewards/rejected": 0.182782843708992,
459
- "step": 290
460
- },
461
- {
462
- "epoch": 1.63,
463
- "grad_norm": 2010.5734100649909,
464
- "learning_rate": 1.005712063557776e-09,
465
- "logits/chosen": -1.6272541284561157,
466
- "logits/rejected": -1.4480578899383545,
467
- "logps/chosen": -324.20068359375,
468
- "logps/rejected": -290.54803466796875,
469
- "loss": 0.5898,
470
- "rewards/accuracies": 0.6968749761581421,
471
- "rewards/chosen": 1.3664501905441284,
472
- "rewards/margins": 1.1471518278121948,
473
- "rewards/rejected": 0.21929831802845,
474
- "step": 300
475
- },
476
- {
477
- "epoch": 1.68,
478
- "grad_norm": 1862.455491029429,
479
- "learning_rate": 7.386599383124321e-10,
480
- "logits/chosen": -1.563561201095581,
481
- "logits/rejected": -1.3803369998931885,
482
- "logps/chosen": -321.889404296875,
483
- "logps/rejected": -285.8083190917969,
484
- "loss": 0.5879,
485
- "rewards/accuracies": 0.7124999761581421,
486
- "rewards/chosen": 1.4487113952636719,
487
- "rewards/margins": 1.3910987377166748,
488
- "rewards/rejected": 0.057612527161836624,
489
- "step": 310
490
- },
491
- {
492
- "epoch": 1.74,
493
- "grad_norm": 1875.0391267528262,
494
- "learning_rate": 5.099665152003929e-10,
495
- "logits/chosen": -1.5980346202850342,
496
- "logits/rejected": -1.3878109455108643,
497
- "logps/chosen": -333.843994140625,
498
- "logps/rejected": -289.8874816894531,
499
- "loss": 0.5634,
500
- "rewards/accuracies": 0.7593749761581421,
501
- "rewards/chosen": 1.5412708520889282,
502
- "rewards/margins": 1.3988701105117798,
503
- "rewards/rejected": 0.14240065217018127,
504
- "step": 320
505
- },
506
- {
507
- "epoch": 1.79,
508
- "grad_norm": 2232.545936783362,
509
- "learning_rate": 3.216903914633745e-10,
510
- "logits/chosen": -1.5563807487487793,
511
- "logits/rejected": -1.4342092275619507,
512
- "logps/chosen": -325.34674072265625,
513
- "logps/rejected": -296.1554870605469,
514
- "loss": 0.5762,
515
- "rewards/accuracies": 0.706250011920929,
516
- "rewards/chosen": 1.3614461421966553,
517
- "rewards/margins": 1.3264009952545166,
518
- "rewards/rejected": 0.03504505008459091,
519
- "step": 330
520
- },
521
- {
522
- "epoch": 1.85,
523
- "grad_norm": 1895.2431252651277,
524
- "learning_rate": 1.7552634565570324e-10,
525
- "logits/chosen": -1.5503065586090088,
526
- "logits/rejected": -1.381874680519104,
527
- "logps/chosen": -329.953857421875,
528
- "logps/rejected": -292.89447021484375,
529
- "loss": 0.5681,
530
- "rewards/accuracies": 0.7718750238418579,
531
- "rewards/chosen": 1.7590471506118774,
532
- "rewards/margins": 1.6621748208999634,
533
- "rewards/rejected": 0.09687252342700958,
534
- "step": 340
535
- },
536
- {
537
- "epoch": 1.9,
538
- "grad_norm": 2070.257116146559,
539
- "learning_rate": 7.279008199590543e-11,
540
- "logits/chosen": -1.541084885597229,
541
- "logits/rejected": -1.378144383430481,
542
- "logps/chosen": -326.51507568359375,
543
- "logps/rejected": -291.95123291015625,
544
- "loss": 0.5797,
545
- "rewards/accuracies": 0.7281249761581421,
546
- "rewards/chosen": 1.5987285375595093,
547
- "rewards/margins": 1.4538962841033936,
548
- "rewards/rejected": 0.1448323279619217,
549
- "step": 350
550
- },
551
- {
552
- "epoch": 1.96,
553
- "grad_norm": 2166.9724743285583,
554
- "learning_rate": 1.4406386978128017e-11,
555
- "logits/chosen": -1.6209802627563477,
556
- "logits/rejected": -1.42485511302948,
557
- "logps/chosen": -331.2142639160156,
558
- "logps/rejected": -291.69842529296875,
559
- "loss": 0.5582,
560
- "rewards/accuracies": 0.778124988079071,
561
- "rewards/chosen": 1.724250078201294,
562
- "rewards/margins": 1.5519315004348755,
563
- "rewards/rejected": 0.1723184883594513,
564
- "step": 360
565
- },
566
- {
567
- "epoch": 2.0,
568
- "step": 368,
569
  "total_flos": 0.0,
570
- "train_loss": 0.26489045179408527,
571
- "train_runtime": 4520.7899,
572
- "train_samples_per_second": 20.835,
573
- "train_steps_per_second": 0.081
574
  }
575
  ],
576
  "logging_steps": 10,
577
- "max_steps": 368,
578
  "num_input_tokens_seen": 0,
579
- "num_train_epochs": 2,
580
  "save_steps": 100,
581
  "total_flos": 0.0,
582
  "train_batch_size": 8,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
  "eval_steps": 500,
6
+ "global_step": 185,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.01,
13
+ "grad_norm": 171.98892218238854,
14
+ "learning_rate": 2.6315789473684208e-08,
15
+ "logits/chosen": -0.1266070306301117,
16
+ "logits/rejected": 0.7204304933547974,
17
+ "logps/chosen": -319.01666259765625,
18
+ "logps/rejected": -252.47039794921875,
19
+ "loss": 0.6916,
20
  "rewards/accuracies": 0.0,
21
  "rewards/chosen": 0.0,
22
  "rewards/margins": 0.0,
 
25
  },
26
  {
27
  "epoch": 0.05,
28
+ "grad_norm": 158.2614639136714,
29
+ "learning_rate": 2.631578947368421e-07,
30
+ "logits/chosen": -0.3861861824989319,
31
+ "logits/rejected": 0.33749374747276306,
32
+ "logps/chosen": -266.4891052246094,
33
+ "logps/rejected": -224.11000061035156,
34
+ "loss": 0.6758,
35
+ "rewards/accuracies": 0.5520833134651184,
36
+ "rewards/chosen": -0.03102089650928974,
37
+ "rewards/margins": 0.034922875463962555,
38
+ "rewards/rejected": -0.06594377011060715,
39
  "step": 10
40
  },
41
  {
42
  "epoch": 0.11,
43
+ "grad_norm": 104.41587535161224,
44
+ "learning_rate": 4.999552306674344e-07,
45
+ "logits/chosen": -0.24374540150165558,
46
+ "logits/rejected": 0.8117060661315918,
47
+ "logps/chosen": -289.02911376953125,
48
+ "logps/rejected": -250.653564453125,
49
+ "loss": 0.478,
50
+ "rewards/accuracies": 0.793749988079071,
51
+ "rewards/chosen": -0.4349571764469147,
52
+ "rewards/margins": 1.263426661491394,
53
+ "rewards/rejected": -1.6983836889266968,
54
  "step": 20
55
  },
56
  {
57
  "epoch": 0.16,
58
+ "grad_norm": 98.86884631406178,
59
+ "learning_rate": 4.946022852363932e-07,
60
+ "logits/chosen": -0.2871348261833191,
61
+ "logits/rejected": 0.6740838289260864,
62
+ "logps/chosen": -281.1429748535156,
63
+ "logps/rejected": -271.7496032714844,
64
+ "loss": 0.4067,
65
+ "rewards/accuracies": 0.800000011920929,
66
+ "rewards/chosen": -1.6509501934051514,
67
+ "rewards/margins": 2.906687021255493,
68
+ "rewards/rejected": -4.5576372146606445,
69
  "step": 30
70
  },
71
  {
72
  "epoch": 0.22,
73
+ "grad_norm": 95.68866413164287,
74
+ "learning_rate": 4.805146507594034e-07,
75
+ "logits/chosen": -0.5090769529342651,
76
+ "logits/rejected": 0.5341213345527649,
77
+ "logps/chosen": -283.4405517578125,
78
+ "logps/rejected": -268.97686767578125,
79
+ "loss": 0.371,
80
+ "rewards/accuracies": 0.8500000238418579,
81
+ "rewards/chosen": -2.4618725776672363,
82
+ "rewards/margins": 3.340365171432495,
83
+ "rewards/rejected": -5.802238464355469,
84
  "step": 40
85
  },
86
  {
87
  "epoch": 0.27,
88
+ "grad_norm": 103.23233983894589,
89
+ "learning_rate": 4.581953932909403e-07,
90
+ "logits/chosen": -0.4626421332359314,
91
+ "logits/rejected": 0.5320831537246704,
92
+ "logps/chosen": -313.1284484863281,
93
+ "logps/rejected": -299.7115173339844,
94
+ "loss": 0.335,
95
+ "rewards/accuracies": 0.862500011920929,
96
+ "rewards/chosen": -2.947516441345215,
97
+ "rewards/margins": 3.4062907695770264,
98
+ "rewards/rejected": -6.353806495666504,
99
  "step": 50
100
  },
101
  {
102
+ "epoch": 0.32,
103
+ "grad_norm": 91.31338474042249,
104
+ "learning_rate": 4.284415281717847e-07,
105
+ "logits/chosen": -0.3830726444721222,
106
+ "logits/rejected": 0.7034914493560791,
107
+ "logps/chosen": -302.44549560546875,
108
+ "logps/rejected": -295.2908020019531,
109
+ "loss": 0.2941,
110
+ "rewards/accuracies": 0.890625,
111
+ "rewards/chosen": -2.427272081375122,
112
+ "rewards/margins": 3.3501389026641846,
113
+ "rewards/rejected": -5.777410507202148,
114
  "step": 60
115
  },
116
  {
117
  "epoch": 0.38,
118
+ "grad_norm": 108.21771645007362,
119
+ "learning_rate": 3.923155588020165e-07,
120
+ "logits/chosen": -0.050761766731739044,
121
+ "logits/rejected": 1.1738256216049194,
122
+ "logps/chosen": -279.0822448730469,
123
+ "logps/rejected": -271.3674011230469,
124
+ "loss": 0.3118,
125
+ "rewards/accuracies": 0.859375,
126
+ "rewards/chosen": -2.9542946815490723,
127
+ "rewards/margins": 3.31215238571167,
128
+ "rewards/rejected": -6.266446590423584,
129
  "step": 70
130
  },
131
  {
132
  "epoch": 0.43,
133
+ "grad_norm": 110.48344039822193,
134
+ "learning_rate": 3.511075348989692e-07,
135
+ "logits/chosen": -0.02379416488111019,
136
+ "logits/rejected": 0.9985305666923523,
137
+ "logps/chosen": -291.3994140625,
138
+ "logps/rejected": -278.81207275390625,
139
+ "loss": 0.3145,
140
+ "rewards/accuracies": 0.862500011920929,
141
+ "rewards/chosen": -1.9188703298568726,
142
+ "rewards/margins": 3.239673614501953,
143
+ "rewards/rejected": -5.158544063568115,
144
  "step": 80
145
  },
146
  {
147
  "epoch": 0.49,
148
+ "grad_norm": 94.44668679211257,
149
+ "learning_rate": 3.062889851306735e-07,
150
+ "logits/chosen": 0.15241345763206482,
151
+ "logits/rejected": 1.204730749130249,
152
+ "logps/chosen": -285.8970947265625,
153
+ "logps/rejected": -274.2763671875,
154
+ "loss": 0.3256,
155
+ "rewards/accuracies": 0.859375,
156
+ "rewards/chosen": -2.3829667568206787,
157
+ "rewards/margins": 3.3176727294921875,
158
+ "rewards/rejected": -5.700639724731445,
159
  "step": 90
160
  },
161
  {
162
  "epoch": 0.54,
163
+ "grad_norm": 72.25131318661623,
164
+ "learning_rate": 2.594603691794176e-07,
165
+ "logits/chosen": 0.017316246405243874,
166
+ "logits/rejected": 1.112657070159912,
167
+ "logps/chosen": -291.90631103515625,
168
+ "logps/rejected": -277.5140686035156,
169
+ "loss": 0.2907,
170
+ "rewards/accuracies": 0.846875011920929,
171
+ "rewards/chosen": -2.7051641941070557,
172
+ "rewards/margins": 3.0991756916046143,
173
+ "rewards/rejected": -5.804339408874512,
174
  "step": 100
175
  },
176
  {
177
+ "epoch": 0.59,
178
+ "grad_norm": 106.17479973453032,
179
+ "learning_rate": 2.1229392570965654e-07,
180
+ "logits/chosen": 0.5481065511703491,
181
+ "logits/rejected": 1.4057379961013794,
182
+ "logps/chosen": -290.0019226074219,
183
+ "logps/rejected": -288.4178161621094,
184
+ "loss": 0.2795,
185
+ "rewards/accuracies": 0.859375,
186
+ "rewards/chosen": -3.1080322265625,
187
+ "rewards/margins": 3.114968776702881,
188
+ "rewards/rejected": -6.223001003265381,
189
  "step": 110
190
  },
191
  {
192
  "epoch": 0.65,
193
+ "grad_norm": 88.25415485320248,
194
+ "learning_rate": 1.6647395712565254e-07,
195
+ "logits/chosen": 0.10530638694763184,
196
+ "logits/rejected": 1.3136330842971802,
197
+ "logps/chosen": -303.7025451660156,
198
+ "logps/rejected": -291.4312438964844,
199
+ "loss": 0.3024,
200
+ "rewards/accuracies": 0.8812500238418579,
201
+ "rewards/chosen": -2.87852144241333,
202
+ "rewards/margins": 3.439791440963745,
203
+ "rewards/rejected": -6.318312644958496,
204
  "step": 120
205
  },
206
  {
207
+ "epoch": 0.7,
208
+ "grad_norm": 81.41509800140894,
209
+ "learning_rate": 1.2363668353585485e-07,
210
+ "logits/chosen": 0.025721266865730286,
211
+ "logits/rejected": 1.1706856489181519,
212
+ "logps/chosen": -291.2774963378906,
213
+ "logps/rejected": -280.7757873535156,
214
+ "loss": 0.2712,
215
+ "rewards/accuracies": 0.903124988079071,
216
+ "rewards/chosen": -2.8241302967071533,
217
+ "rewards/margins": 3.6137948036193848,
218
+ "rewards/rejected": -6.437924385070801,
219
  "step": 130
220
  },
221
  {
222
  "epoch": 0.76,
223
+ "grad_norm": 76.56961564493653,
224
+ "learning_rate": 8.53118137245516e-08,
225
+ "logits/chosen": 0.24798288941383362,
226
+ "logits/rejected": 1.3128881454467773,
227
+ "logps/chosen": -298.71783447265625,
228
+ "logps/rejected": -297.16790771484375,
229
+ "loss": 0.2607,
230
+ "rewards/accuracies": 0.8656250238418579,
231
+ "rewards/chosen": -2.95615553855896,
232
+ "rewards/margins": 3.7294158935546875,
233
+ "rewards/rejected": -6.685571193695068,
234
  "step": 140
235
  },
236
  {
237
+ "epoch": 0.81,
238
+ "grad_norm": 101.60579173655283,
239
+ "learning_rate": 5.2867919617408553e-08,
240
+ "logits/chosen": 0.16610342264175415,
241
+ "logits/rejected": 1.297738790512085,
242
+ "logps/chosen": -296.17230224609375,
243
+ "logps/rejected": -285.56707763671875,
244
+ "loss": 0.2777,
245
+ "rewards/accuracies": 0.903124988079071,
246
+ "rewards/chosen": -2.7571194171905518,
247
+ "rewards/margins": 3.536668062210083,
248
+ "rewards/rejected": -6.293786525726318,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.86,
253
+ "grad_norm": 92.310593955402,
254
+ "learning_rate": 2.7463564905650853e-08,
255
+ "logits/chosen": 0.06046704202890396,
256
+ "logits/rejected": 1.0854153633117676,
257
+ "logps/chosen": -297.1445007324219,
258
+ "logps/rejected": -291.33868408203125,
259
+ "loss": 0.2684,
260
+ "rewards/accuracies": 0.893750011920929,
261
+ "rewards/chosen": -2.6816515922546387,
262
+ "rewards/margins": 3.552661418914795,
263
+ "rewards/rejected": -6.234313011169434,
264
  "step": 160
265
  },
266
  {
267
  "epoch": 0.92,
268
+ "grad_norm": 83.71834684366553,
269
+ "learning_rate": 1.0005933014019307e-08,
270
+ "logits/chosen": 0.15604642033576965,
271
+ "logits/rejected": 1.338841199874878,
272
+ "logps/chosen": -298.0588684082031,
273
+ "logps/rejected": -293.54638671875,
274
+ "loss": 0.2745,
275
+ "rewards/accuracies": 0.903124988079071,
276
+ "rewards/chosen": -3.0692405700683594,
277
+ "rewards/margins": 3.527927875518799,
278
+ "rewards/rejected": -6.59716796875,
279
  "step": 170
280
  },
281
  {
282
+ "epoch": 0.97,
283
+ "grad_norm": 78.86616344216218,
284
+ "learning_rate": 1.1184317978602808e-09,
285
+ "logits/chosen": -0.07575028389692307,
286
+ "logits/rejected": 1.0216057300567627,
287
+ "logps/chosen": -288.5888366699219,
288
+ "logps/rejected": -287.2474670410156,
289
+ "loss": 0.3031,
290
+ "rewards/accuracies": 0.90625,
291
+ "rewards/chosen": -2.831172466278076,
292
+ "rewards/margins": 3.852785587310791,
293
+ "rewards/rejected": -6.683958530426025,
294
  "step": 180
295
  },
296
  {
297
+ "epoch": 1.0,
298
+ "step": 185,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
299
  "total_flos": 0.0,
300
+ "train_loss": 0.33391942269093283,
301
+ "train_runtime": 5319.9853,
302
+ "train_samples_per_second": 8.891,
303
+ "train_steps_per_second": 0.035
304
  }
305
  ],
306
  "logging_steps": 10,
307
+ "max_steps": 185,
308
  "num_input_tokens_seen": 0,
309
+ "num_train_epochs": 1,
310
  "save_steps": 100,
311
  "total_flos": 0.0,
312
  "train_batch_size": 8,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7ac969fef84df010a18a3f4d0a9620668cc46f72adfaf3b07d1d383dfb0d5734
3
- size 6520
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:684c529dee2a83fa137d0e6a638952a379fc861d4c597982d59871fc123d5d7b
3
+ size 6264