Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wzhouad commited on
Commit
8a4c724
•
1 Parent(s): 5c36073

Model save

Browse files
Files changed (8) hide show
  1. README.md +17 -20
  2. all_results.json +4 -4
  3. generation_config.json +1 -1
  4. model-00001-of-00003.safetensors +1 -1
  5. model-00002-of-00003.safetensors +1 -1
  6. model-00003-of-00003.safetensors +1 -1
  7. train_results.json +4 -4
  8. trainer_state.json +479 -685
README.md CHANGED
@@ -17,18 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
- - Loss: 0.1422
21
- - Rewards/chosen: -1.3154
22
- - Rewards/rejected: -2.2768
23
- - Rewards/accuracies: 0.7617
24
- - Rewards/margins: 0.9613
25
- - Logps/rejected: -483.9327
26
- - Logps/chosen: -386.7366
27
- - Logits/rejected: -2.1695
28
- - Logits/chosen: -2.2036
29
- - Debug/policy Weights: 0.2815
30
- - Debug/losses: 0.1397
31
- - Debug/raw Losses: 0.4727
32
 
33
  ## Model description
34
 
@@ -63,17 +60,17 @@ The following hyperparameters were used during training:
63
 
64
  ### Training results
65
 
66
- | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Debug/policy Weights | Debug/losses | Debug/raw Losses |
67
- |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------------------:|:------------:|:----------------:|
68
- | 0.1781 | 0.21 | 100 | 0.2007 | -0.6478 | -1.1693 | 0.7344 | 0.5214 | -373.1867 | -319.9806 | -2.6910 | -2.7080 | 0.3512 | 0.1953 | 0.5590 |
69
- | 0.1616 | 0.42 | 200 | 0.1669 | -0.8830 | -1.6003 | 0.7109 | 0.7173 | -416.2844 | -343.4914 | -2.4277 | -2.4499 | 0.3174 | 0.1671 | 0.5079 |
70
- | 0.1343 | 0.63 | 300 | 0.1368 | -1.5021 | -2.3715 | 0.7578 | 0.8695 | -493.4114 | -405.4042 | -2.2283 | -2.2618 | 0.2666 | 0.1365 | 0.4953 |
71
- | 0.1398 | 0.84 | 400 | 0.1422 | -1.3154 | -2.2768 | 0.7617 | 0.9613 | -483.9327 | -386.7366 | -2.1695 | -2.2036 | 0.2815 | 0.1397 | 0.4727 |
72
 
73
 
74
  ### Framework versions
75
 
76
- - Transformers 4.39.3
77
  - Pytorch 2.1.2+cu121
78
  - Datasets 2.14.6
79
- - Tokenizers 0.15.2
 
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
+ - Loss: 239.1993
21
+ - Rewards/chosen: 0.3530
22
+ - Rewards/rejected: 0.3216
23
+ - Rewards/accuracies: 0.5625
24
+ - Rewards/margins: 0.0314
25
+ - Logps/rejected: -225.1900
26
+ - Logps/chosen: -221.7349
27
+ - Logits/rejected: -2.6730
28
+ - Logits/chosen: -2.6975
 
 
 
29
 
30
  ## Model description
31
 
 
60
 
61
  ### Training results
62
 
63
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 256.7688 | 0.21 | 100 | 245.7276 | 0.2947 | 0.2688 | 0.5742 | 0.0259 | -230.4730 | -227.5649 | -2.7670 | -2.7863 |
66
+ | 239.4804 | 0.42 | 200 | 241.5078 | 0.3319 | 0.3036 | 0.5742 | 0.0283 | -226.9933 | -223.8462 | -2.7082 | -2.7290 |
67
+ | 240.5041 | 0.63 | 300 | 239.8053 | 0.3474 | 0.3163 | 0.5625 | 0.0310 | -225.7186 | -222.3044 | -2.6807 | -2.7042 |
68
+ | 236.8453 | 0.84 | 400 | 239.1993 | 0.3530 | 0.3216 | 0.5625 | 0.0314 | -225.1900 | -221.7349 | -2.6730 | -2.6975 |
69
 
70
 
71
  ### Framework versions
72
 
73
+ - Transformers 4.35.2
74
  - Pytorch 2.1.2+cu121
75
  - Datasets 2.14.6
76
+ - Tokenizers 0.14.1
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.17903520272865456,
4
- "train_runtime": 4631.245,
5
  "train_samples": 61134,
6
- "train_samples_per_second": 13.2,
7
- "train_steps_per_second": 0.103
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 243.0746703846185,
4
+ "train_runtime": 4321.456,
5
  "train_samples": 61134,
6
+ "train_samples_per_second": 14.147,
7
+ "train_steps_per_second": 0.111
8
  }
generation_config.json CHANGED
@@ -2,5 +2,5 @@
2
  "_from_model_config": true,
3
  "bos_token_id": 1,
4
  "eos_token_id": 2,
5
- "transformers_version": "4.39.3"
6
  }
 
2
  "_from_model_config": true,
3
  "bos_token_id": 1,
4
  "eos_token_id": 2,
5
+ "transformers_version": "4.35.2"
6
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:18d00ba6a8e8636533d5f9f026b67bade132d932eb7e25cf20a5471d14cf4938
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:277c398953854fe1ef1e607c555edb8d9805bb10b2e6804985b60941cb61f34b
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a726c7d9b0e02053f892980f475aa8bdf053ba38518c5ec8f29a48cd43f1e728
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8501af37d60bcc3a6e88c289f7e5e7cb5fca9b418c5ad3ee8c7ae02e8ddfcb6f
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:abb64bdbf0bdca4349e17eb7d2424fa102b61d7e91947dde62627f2995cbf4b8
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a22eb70eee889fd9ec807b42dce6f68fc73c8facbc6852410e1ae6436844039
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.17903520272865456,
4
- "train_runtime": 4631.245,
5
  "train_samples": 61134,
6
- "train_samples_per_second": 13.2,
7
- "train_steps_per_second": 0.103
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 243.0746703846185,
4
+ "train_runtime": 4321.456,
5
  "train_samples": 61134,
6
+ "train_samples_per_second": 14.147,
7
+ "train_steps_per_second": 0.111
8
  }
trainer_state.json CHANGED
@@ -9,17 +9,13 @@
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
- "debug/losses": 0.36003828048706055,
13
- "debug/policy_weights": 0.5194254517555237,
14
- "debug/raw_losses": 0.6931471824645996,
15
  "epoch": 0.0,
16
- "grad_norm": 5.3582401111914875,
17
  "learning_rate": 1.0416666666666666e-08,
18
- "logits/chosen": -2.888590097427368,
19
- "logits/rejected": -2.871957540512085,
20
- "logps/chosen": -261.88616943359375,
21
- "logps/rejected": -355.31793212890625,
22
- "loss": 0.3648,
23
  "rewards/accuracies": 0.0,
24
  "rewards/chosen": 0.0,
25
  "rewards/margins": 0.0,
@@ -27,944 +23,742 @@
27
  "step": 1
28
  },
29
  {
30
- "debug/losses": 0.3548344671726227,
31
- "debug/policy_weights": 0.5118452906608582,
32
- "debug/raw_losses": 0.6932389736175537,
33
  "epoch": 0.02,
34
- "grad_norm": 5.014402110225361,
35
  "learning_rate": 1.0416666666666667e-07,
36
- "logits/chosen": -2.7435038089752197,
37
- "logits/rejected": -2.7258925437927246,
38
- "logps/chosen": -256.5353088378906,
39
- "logps/rejected": -244.49546813964844,
40
- "loss": 0.3619,
41
- "rewards/accuracies": 0.4236111044883728,
42
- "rewards/chosen": -8.211319800466299e-05,
43
- "rewards/margins": -0.00017834355821833014,
44
- "rewards/rejected": 9.623030200600624e-05,
45
  "step": 10
46
  },
47
  {
48
- "debug/losses": 0.3572663962841034,
49
- "debug/policy_weights": 0.5156484842300415,
50
- "debug/raw_losses": 0.6928108930587769,
51
  "epoch": 0.04,
52
- "grad_norm": 4.715098712970233,
53
  "learning_rate": 2.0833333333333333e-07,
54
- "logits/chosen": -2.8007731437683105,
55
- "logits/rejected": -2.7729618549346924,
56
- "logps/chosen": -271.37860107421875,
57
- "logps/rejected": -288.34271240234375,
58
- "loss": 0.3584,
59
  "rewards/accuracies": 0.543749988079071,
60
- "rewards/chosen": 0.0006374852382577956,
61
- "rewards/margins": 0.0006833201623521745,
62
- "rewards/rejected": -4.583501868182793e-05,
63
  "step": 20
64
  },
65
  {
66
- "debug/losses": 0.35951218008995056,
67
- "debug/policy_weights": 0.521370530128479,
68
- "debug/raw_losses": 0.6895479559898376,
69
  "epoch": 0.06,
70
- "grad_norm": 4.7982052271779825,
71
  "learning_rate": 3.1249999999999997e-07,
72
- "logits/chosen": -2.810387134552002,
73
- "logits/rejected": -2.773030996322632,
74
- "logps/chosen": -291.4071350097656,
75
- "logps/rejected": -258.0267639160156,
76
- "loss": 0.3579,
77
- "rewards/accuracies": 0.6312500238418579,
78
- "rewards/chosen": 0.004485988523811102,
79
- "rewards/margins": 0.00730940094217658,
80
- "rewards/rejected": -0.002823411952704191,
81
  "step": 30
82
  },
83
  {
84
- "debug/losses": 0.35061347484588623,
85
- "debug/policy_weights": 0.5189496278762817,
86
- "debug/raw_losses": 0.6759483218193054,
87
  "epoch": 0.08,
88
- "grad_norm": 4.841334528975048,
89
  "learning_rate": 4.1666666666666667e-07,
90
- "logits/chosen": -2.7438368797302246,
91
- "logits/rejected": -2.7376155853271484,
92
- "logps/chosen": -252.50436401367188,
93
- "logps/rejected": -240.8105010986328,
94
- "loss": 0.3573,
95
- "rewards/accuracies": 0.762499988079071,
96
- "rewards/chosen": 0.015405787155032158,
97
- "rewards/margins": 0.03552238270640373,
98
- "rewards/rejected": -0.020116599276661873,
99
  "step": 40
100
  },
101
  {
102
- "debug/losses": 0.34173911809921265,
103
- "debug/policy_weights": 0.5125550627708435,
104
- "debug/raw_losses": 0.666517436504364,
105
  "epoch": 0.1,
106
- "grad_norm": 5.135483442753442,
107
  "learning_rate": 4.999733114418725e-07,
108
- "logits/chosen": -2.6830456256866455,
109
- "logits/rejected": -2.6514101028442383,
110
- "logps/chosen": -269.3827819824219,
111
- "logps/rejected": -287.2772216796875,
112
- "loss": 0.3437,
113
- "rewards/accuracies": 0.6812499761581421,
114
- "rewards/chosen": 0.0019381232559680939,
115
- "rewards/margins": 0.05897800251841545,
116
- "rewards/rejected": -0.05703987926244736,
117
  "step": 50
118
  },
119
  {
120
- "debug/losses": 0.3377828598022461,
121
- "debug/policy_weights": 0.5224286317825317,
122
- "debug/raw_losses": 0.6448591947555542,
123
  "epoch": 0.13,
124
- "grad_norm": 5.5606740216520905,
125
  "learning_rate": 4.990398100856366e-07,
126
- "logits/chosen": -2.745028018951416,
127
- "logits/rejected": -2.713804244995117,
128
- "logps/chosen": -266.05279541015625,
129
- "logps/rejected": -305.28271484375,
130
- "loss": 0.3305,
131
- "rewards/accuracies": 0.6937500238418579,
132
- "rewards/chosen": -0.02193317376077175,
133
- "rewards/margins": 0.1177050918340683,
134
- "rewards/rejected": -0.1396382749080658,
135
  "step": 60
136
  },
137
  {
138
- "debug/losses": 0.28029459714889526,
139
- "debug/policy_weights": 0.47101029753685,
140
- "debug/raw_losses": 0.6030661463737488,
141
  "epoch": 0.15,
142
- "grad_norm": 6.284287160526739,
143
  "learning_rate": 4.967775735898179e-07,
144
- "logits/chosen": -2.7072081565856934,
145
- "logits/rejected": -2.7113521099090576,
146
- "logps/chosen": -272.530029296875,
147
- "logps/rejected": -289.87713623046875,
148
- "loss": 0.2826,
149
- "rewards/accuracies": 0.7875000238418579,
150
- "rewards/chosen": -0.1557193100452423,
151
- "rewards/margins": 0.24685576558113098,
152
- "rewards/rejected": -0.4025750756263733,
153
  "step": 70
154
  },
155
  {
156
- "debug/losses": 0.26233458518981934,
157
- "debug/policy_weights": 0.42553257942199707,
158
- "debug/raw_losses": 0.6218484044075012,
159
  "epoch": 0.17,
160
- "grad_norm": 6.8893773770621305,
161
  "learning_rate": 4.931986719649298e-07,
162
- "logits/chosen": -2.800093412399292,
163
- "logits/rejected": -2.7768216133117676,
164
- "logps/chosen": -360.3621520996094,
165
- "logps/rejected": -329.7616882324219,
166
- "loss": 0.2516,
167
- "rewards/accuracies": 0.6875,
168
- "rewards/chosen": -0.45188745856285095,
169
- "rewards/margins": 0.26955321431159973,
170
- "rewards/rejected": -0.7214406728744507,
171
  "step": 80
172
  },
173
  {
174
- "debug/losses": 0.18998686969280243,
175
- "debug/policy_weights": 0.34327536821365356,
176
- "debug/raw_losses": 0.5527450442314148,
177
  "epoch": 0.19,
178
- "grad_norm": 6.5870040213670435,
179
  "learning_rate": 4.883222001996351e-07,
180
- "logits/chosen": -2.724050998687744,
181
- "logits/rejected": -2.690390110015869,
182
- "logps/chosen": -303.0259704589844,
183
- "logps/rejected": -344.49505615234375,
184
- "loss": 0.2198,
185
- "rewards/accuracies": 0.768750011920929,
186
- "rewards/chosen": -0.5087189078330994,
187
- "rewards/margins": 0.4743797779083252,
188
- "rewards/rejected": -0.9830986857414246,
189
  "step": 90
190
  },
191
  {
192
- "debug/losses": 0.16577747464179993,
193
- "debug/policy_weights": 0.29914408922195435,
194
- "debug/raw_losses": 0.5552432537078857,
195
  "epoch": 0.21,
196
- "grad_norm": 7.249323730073561,
197
  "learning_rate": 4.821741763807186e-07,
198
- "logits/chosen": -2.6653432846069336,
199
- "logits/rejected": -2.658141613006592,
200
- "logps/chosen": -325.9325866699219,
201
- "logps/rejected": -375.12127685546875,
202
- "loss": 0.1781,
203
- "rewards/accuracies": 0.706250011920929,
204
- "rewards/chosen": -0.7297804951667786,
205
- "rewards/margins": 0.511761486530304,
206
- "rewards/rejected": -1.2415419816970825,
207
  "step": 100
208
  },
209
  {
210
  "epoch": 0.21,
211
- "eval_debug/losses": 0.19525206089019775,
212
- "eval_debug/policy_weights": 0.3511899709701538,
213
- "eval_debug/raw_losses": 0.5590351819992065,
214
- "eval_logits/chosen": -2.70804500579834,
215
- "eval_logits/rejected": -2.6910133361816406,
216
- "eval_logps/chosen": -319.9805603027344,
217
- "eval_logps/rejected": -373.18670654296875,
218
- "eval_loss": 0.20066915452480316,
219
- "eval_rewards/accuracies": 0.734375,
220
- "eval_rewards/chosen": -0.6478448510169983,
221
- "eval_rewards/margins": 0.5214487910270691,
222
- "eval_rewards/rejected": -1.1692936420440674,
223
- "eval_runtime": 43.2006,
224
- "eval_samples_per_second": 46.296,
225
- "eval_steps_per_second": 0.741,
226
  "step": 100
227
  },
228
  {
229
- "debug/losses": 0.1987285315990448,
230
- "debug/policy_weights": 0.34678488969802856,
231
- "debug/raw_losses": 0.5721240043640137,
232
  "epoch": 0.23,
233
- "grad_norm": 6.9123559213202554,
234
  "learning_rate": 4.747874028753375e-07,
235
- "logits/chosen": -2.7045469284057617,
236
- "logits/rejected": -2.672999858856201,
237
- "logps/chosen": -369.9187927246094,
238
- "logps/rejected": -373.6562194824219,
239
- "loss": 0.1986,
240
- "rewards/accuracies": 0.706250011920929,
241
- "rewards/chosen": -0.7313551902770996,
242
- "rewards/margins": 0.47859472036361694,
243
- "rewards/rejected": -1.2099498510360718,
244
  "step": 110
245
  },
246
  {
247
- "debug/losses": 0.18334314227104187,
248
- "debug/policy_weights": 0.3257436156272888,
249
- "debug/raw_losses": 0.5537685751914978,
250
  "epoch": 0.25,
251
- "grad_norm": 7.316158832496888,
252
  "learning_rate": 4.662012913161997e-07,
253
- "logits/chosen": -2.6026859283447266,
254
- "logits/rejected": -2.600971221923828,
255
- "logps/chosen": -346.50555419921875,
256
- "logps/rejected": -379.53887939453125,
257
- "loss": 0.1846,
258
- "rewards/accuracies": 0.731249988079071,
259
- "rewards/chosen": -0.7285534739494324,
260
- "rewards/margins": 0.5237767100334167,
261
- "rewards/rejected": -1.2523301839828491,
262
  "step": 120
263
  },
264
  {
265
- "debug/losses": 0.17840833961963654,
266
- "debug/policy_weights": 0.3271704614162445,
267
- "debug/raw_losses": 0.5529276132583618,
268
  "epoch": 0.27,
269
- "grad_norm": 6.11630399118371,
270
  "learning_rate": 4.5646165232345103e-07,
271
- "logits/chosen": -2.5919995307922363,
272
- "logits/rejected": -2.598027467727661,
273
- "logps/chosen": -346.44610595703125,
274
- "logps/rejected": -391.03778076171875,
275
- "loss": 0.1758,
276
- "rewards/accuracies": 0.731249988079071,
277
- "rewards/chosen": -0.7799400687217712,
278
- "rewards/margins": 0.5067718029022217,
279
- "rewards/rejected": -1.2867119312286377,
280
  "step": 130
281
  },
282
  {
283
- "debug/losses": 0.18315494060516357,
284
- "debug/policy_weights": 0.3452090620994568,
285
- "debug/raw_losses": 0.56131511926651,
286
  "epoch": 0.29,
287
- "grad_norm": 6.845717287191781,
288
  "learning_rate": 4.456204510851956e-07,
289
- "logits/chosen": -2.572934627532959,
290
- "logits/rejected": -2.5713508129119873,
291
- "logps/chosen": -390.03253173828125,
292
- "logps/rejected": -440.16461181640625,
293
- "loss": 0.1668,
294
- "rewards/accuracies": 0.7250000238418579,
295
- "rewards/chosen": -0.8578819036483765,
296
- "rewards/margins": 0.5912094712257385,
297
- "rewards/rejected": -1.4490914344787598,
298
  "step": 140
299
  },
300
  {
301
- "debug/losses": 0.19362398982048035,
302
- "debug/policy_weights": 0.33687275648117065,
303
- "debug/raw_losses": 0.559166669845581,
304
  "epoch": 0.31,
305
- "grad_norm": 7.029648911332738,
306
  "learning_rate": 4.337355301007335e-07,
307
- "logits/chosen": -2.512932300567627,
308
- "logits/rejected": -2.4797143936157227,
309
- "logps/chosen": -357.82464599609375,
310
- "logps/rejected": -402.2618713378906,
311
- "loss": 0.1771,
312
- "rewards/accuracies": 0.7124999761581421,
313
- "rewards/chosen": -0.8432974815368652,
314
- "rewards/margins": 0.48099932074546814,
315
- "rewards/rejected": -1.3242968320846558,
316
  "step": 150
317
  },
318
  {
319
- "debug/losses": 0.14652669429779053,
320
- "debug/policy_weights": 0.28093475103378296,
321
- "debug/raw_losses": 0.5318498611450195,
322
  "epoch": 0.33,
323
- "grad_norm": 6.358610241024355,
324
  "learning_rate": 4.2087030056579986e-07,
325
- "logits/chosen": -2.464831590652466,
326
- "logits/rejected": -2.4293084144592285,
327
- "logps/chosen": -370.14971923828125,
328
- "logps/rejected": -408.4079284667969,
329
- "loss": 0.1583,
330
- "rewards/accuracies": 0.7124999761581421,
331
- "rewards/chosen": -1.0606319904327393,
332
- "rewards/margins": 0.596233606338501,
333
- "rewards/rejected": -1.6568657159805298,
334
  "step": 160
335
  },
336
  {
337
- "debug/losses": 0.1372167319059372,
338
- "debug/policy_weights": 0.2504947781562805,
339
- "debug/raw_losses": 0.6000548005104065,
340
  "epoch": 0.36,
341
- "grad_norm": 7.75801911972225,
342
  "learning_rate": 4.070934040463998e-07,
343
- "logits/chosen": -2.361786365509033,
344
- "logits/rejected": -2.332592010498047,
345
- "logps/chosen": -389.86767578125,
346
- "logps/rejected": -423.97698974609375,
347
- "loss": 0.1417,
348
- "rewards/accuracies": 0.675000011920929,
349
- "rewards/chosen": -1.4737131595611572,
350
- "rewards/margins": 0.511743426322937,
351
- "rewards/rejected": -1.9854564666748047,
352
  "step": 170
353
  },
354
  {
355
- "debug/losses": 0.11748126894235611,
356
- "debug/policy_weights": 0.2296670377254486,
357
- "debug/raw_losses": 0.5410301089286804,
358
  "epoch": 0.38,
359
- "grad_norm": 6.142786182484198,
360
  "learning_rate": 3.9247834624635404e-07,
361
- "logits/chosen": -2.3254575729370117,
362
- "logits/rejected": -2.3115153312683105,
363
- "logps/chosen": -391.5261535644531,
364
- "logps/rejected": -429.21875,
365
- "loss": 0.1327,
366
- "rewards/accuracies": 0.706250011920929,
367
- "rewards/chosen": -1.446401834487915,
368
- "rewards/margins": 0.6310542821884155,
369
- "rewards/rejected": -2.07745623588562,
370
  "step": 180
371
  },
372
  {
373
- "debug/losses": 0.16298121213912964,
374
- "debug/policy_weights": 0.2831325829029083,
375
- "debug/raw_losses": 0.5709558725357056,
376
  "epoch": 0.4,
377
- "grad_norm": 6.941055961208667,
378
  "learning_rate": 3.7710310482256523e-07,
379
- "logits/chosen": -2.378763198852539,
380
- "logits/rejected": -2.3435521125793457,
381
- "logps/chosen": -381.296630859375,
382
- "logps/rejected": -435.13494873046875,
383
- "loss": 0.1447,
384
- "rewards/accuracies": 0.675000011920929,
385
- "rewards/chosen": -1.178696870803833,
386
- "rewards/margins": 0.6175613403320312,
387
- "rewards/rejected": -1.7962583303451538,
388
  "step": 190
389
  },
390
  {
391
- "debug/losses": 0.1531984657049179,
392
- "debug/policy_weights": 0.27346447110176086,
393
- "debug/raw_losses": 0.5578541159629822,
394
  "epoch": 0.42,
395
- "grad_norm": 9.54639374222291,
396
  "learning_rate": 3.610497133404795e-07,
397
- "logits/chosen": -2.3584351539611816,
398
- "logits/rejected": -2.362753391265869,
399
- "logps/chosen": -359.6307067871094,
400
- "logps/rejected": -422.2249450683594,
401
- "loss": 0.1616,
402
- "rewards/accuracies": 0.731249988079071,
403
- "rewards/chosen": -1.0809690952301025,
404
- "rewards/margins": 0.6879221796989441,
405
- "rewards/rejected": -1.7688913345336914,
406
  "step": 200
407
  },
408
  {
409
  "epoch": 0.42,
410
- "eval_debug/losses": 0.16714033484458923,
411
- "eval_debug/policy_weights": 0.3173619508743286,
412
- "eval_debug/raw_losses": 0.5079437494277954,
413
- "eval_logits/chosen": -2.44987154006958,
414
- "eval_logits/rejected": -2.4276504516601562,
415
- "eval_logps/chosen": -343.49139404296875,
416
- "eval_logps/rejected": -416.28436279296875,
417
- "eval_loss": 0.16694626212120056,
418
- "eval_rewards/accuracies": 0.7109375,
419
- "eval_rewards/chosen": -0.8829529881477356,
420
- "eval_rewards/margins": 0.7173169851303101,
421
- "eval_rewards/rejected": -1.6002700328826904,
422
- "eval_runtime": 43.2126,
423
- "eval_samples_per_second": 46.283,
424
- "eval_steps_per_second": 0.741,
425
  "step": 200
426
  },
427
  {
428
- "debug/losses": 0.14734311401844025,
429
- "debug/policy_weights": 0.2870820164680481,
430
- "debug/raw_losses": 0.5041711330413818,
431
  "epoch": 0.44,
432
- "grad_norm": 7.104709810051969,
433
  "learning_rate": 3.4440382358952115e-07,
434
- "logits/chosen": -2.3574612140655518,
435
- "logits/rejected": -2.326927900314331,
436
- "logps/chosen": -381.2717590332031,
437
- "logps/rejected": -413.890869140625,
438
- "loss": 0.1636,
439
- "rewards/accuracies": 0.7437499761581421,
440
- "rewards/chosen": -0.9982665777206421,
441
- "rewards/margins": 0.6929588317871094,
442
- "rewards/rejected": -1.6912254095077515,
443
  "step": 210
444
  },
445
  {
446
- "debug/losses": 0.13881739974021912,
447
- "debug/policy_weights": 0.27268368005752563,
448
- "debug/raw_losses": 0.5210752487182617,
449
  "epoch": 0.46,
450
- "grad_norm": 8.340698654196919,
451
  "learning_rate": 3.272542485937368e-07,
452
- "logits/chosen": -2.3073763847351074,
453
- "logits/rejected": -2.288245677947998,
454
- "logps/chosen": -370.5925598144531,
455
- "logps/rejected": -423.53021240234375,
456
- "loss": 0.1549,
457
- "rewards/accuracies": 0.75,
458
- "rewards/chosen": -1.1713535785675049,
459
- "rewards/margins": 0.7361623644828796,
460
- "rewards/rejected": -1.9075158834457397,
461
  "step": 220
462
  },
463
  {
464
- "debug/losses": 0.15816743671894073,
465
- "debug/policy_weights": 0.2902621626853943,
466
- "debug/raw_losses": 0.509421706199646,
467
  "epoch": 0.48,
468
- "grad_norm": 7.2612463171873225,
469
  "learning_rate": 3.096924887558854e-07,
470
- "logits/chosen": -2.339087724685669,
471
- "logits/rejected": -2.31463360786438,
472
- "logps/chosen": -342.2999572753906,
473
- "logps/rejected": -425.86181640625,
474
- "loss": 0.169,
475
- "rewards/accuracies": 0.731249988079071,
476
- "rewards/chosen": -1.0157617330551147,
477
- "rewards/margins": 0.8210276365280151,
478
- "rewards/rejected": -1.8367893695831299,
479
  "step": 230
480
  },
481
  {
482
- "debug/losses": 0.15576457977294922,
483
- "debug/policy_weights": 0.29614073038101196,
484
- "debug/raw_losses": 0.4828997552394867,
485
  "epoch": 0.5,
486
- "grad_norm": 7.345094511114747,
487
  "learning_rate": 2.9181224366319943e-07,
488
- "logits/chosen": -2.3611202239990234,
489
- "logits/rejected": -2.317119836807251,
490
- "logps/chosen": -368.0779113769531,
491
- "logps/rejected": -434.31549072265625,
492
- "loss": 0.1514,
493
- "rewards/accuracies": 0.768750011920929,
494
- "rewards/chosen": -1.1054710149765015,
495
- "rewards/margins": 0.8346185684204102,
496
- "rewards/rejected": -1.9400895833969116,
497
  "step": 240
498
  },
499
  {
500
- "debug/losses": 0.12131881713867188,
501
- "debug/policy_weights": 0.2461313009262085,
502
- "debug/raw_losses": 0.46923789381980896,
503
  "epoch": 0.52,
504
- "grad_norm": 6.902697741869489,
505
  "learning_rate": 2.7370891215954565e-07,
506
- "logits/chosen": -2.2654645442962646,
507
- "logits/rejected": -2.2106964588165283,
508
- "logps/chosen": -434.3672790527344,
509
- "logps/rejected": -491.00750732421875,
510
- "loss": 0.1285,
511
- "rewards/accuracies": 0.78125,
512
- "rewards/chosen": -1.4400012493133545,
513
- "rewards/margins": 0.9503369331359863,
514
- "rewards/rejected": -2.390338182449341,
515
  "step": 250
516
  },
517
  {
518
- "debug/losses": 0.1332876831293106,
519
- "debug/policy_weights": 0.27705293893814087,
520
- "debug/raw_losses": 0.5086624622344971,
521
  "epoch": 0.54,
522
- "grad_norm": 7.179715362550834,
523
  "learning_rate": 2.55479083351317e-07,
524
- "logits/chosen": -2.316641330718994,
525
- "logits/rejected": -2.3014323711395264,
526
- "logps/chosen": -410.114990234375,
527
- "logps/rejected": -454.8689880371094,
528
- "loss": 0.1406,
529
- "rewards/accuracies": 0.762499988079071,
530
- "rewards/chosen": -1.2414360046386719,
531
- "rewards/margins": 0.8403455018997192,
532
- "rewards/rejected": -2.0817813873291016,
533
  "step": 260
534
  },
535
  {
536
- "debug/losses": 0.168529212474823,
537
- "debug/policy_weights": 0.2988266348838806,
538
- "debug/raw_losses": 0.5682334899902344,
539
  "epoch": 0.56,
540
- "grad_norm": 7.216808591858915,
541
  "learning_rate": 2.3722002126275822e-07,
542
- "logits/chosen": -2.3377718925476074,
543
- "logits/rejected": -2.3152408599853516,
544
- "logps/chosen": -379.34661865234375,
545
- "logps/rejected": -430.7207946777344,
546
- "loss": 0.1519,
547
- "rewards/accuracies": 0.675000011920929,
548
- "rewards/chosen": -1.0926826000213623,
549
- "rewards/margins": 0.6951464414596558,
550
- "rewards/rejected": -1.7878293991088867,
551
  "step": 270
552
  },
553
  {
554
- "debug/losses": 0.15156733989715576,
555
- "debug/policy_weights": 0.2692407965660095,
556
- "debug/raw_losses": 0.5413981080055237,
557
  "epoch": 0.59,
558
- "grad_norm": 9.464551455783996,
559
  "learning_rate": 2.19029145890313e-07,
560
- "logits/chosen": -2.295574426651001,
561
- "logits/rejected": -2.2587995529174805,
562
- "logps/chosen": -383.4193420410156,
563
- "logps/rejected": -444.91680908203125,
564
- "loss": 0.1498,
565
- "rewards/accuracies": 0.731249988079071,
566
- "rewards/chosen": -1.302856683731079,
567
- "rewards/margins": 0.7735105156898499,
568
- "rewards/rejected": -2.076367139816284,
569
  "step": 280
570
  },
571
  {
572
- "debug/losses": 0.14380133152008057,
573
- "debug/policy_weights": 0.26454827189445496,
574
- "debug/raw_losses": 0.5405210256576538,
575
  "epoch": 0.61,
576
- "grad_norm": 7.542791313020671,
577
  "learning_rate": 2.0100351342479216e-07,
578
- "logits/chosen": -2.3062846660614014,
579
- "logits/rejected": -2.2651028633117676,
580
- "logps/chosen": -386.890625,
581
- "logps/rejected": -450.9478454589844,
582
- "loss": 0.1352,
583
- "rewards/accuracies": 0.737500011920929,
584
- "rewards/chosen": -1.4489481449127197,
585
- "rewards/margins": 0.7288294434547424,
586
- "rewards/rejected": -2.1777775287628174,
587
  "step": 290
588
  },
589
  {
590
- "debug/losses": 0.14622345566749573,
591
- "debug/policy_weights": 0.2636463940143585,
592
- "debug/raw_losses": 0.536266028881073,
593
  "epoch": 0.63,
594
- "grad_norm": 7.687631612230176,
595
  "learning_rate": 1.8323929841460178e-07,
596
- "logits/chosen": -2.2653095722198486,
597
- "logits/rejected": -2.2190821170806885,
598
- "logps/chosen": -444.52960205078125,
599
- "logps/rejected": -489.06817626953125,
600
- "loss": 0.1343,
601
- "rewards/accuracies": 0.699999988079071,
602
- "rewards/chosen": -1.5472103357315063,
603
- "rewards/margins": 0.8187928199768066,
604
- "rewards/rejected": -2.3660032749176025,
605
  "step": 300
606
  },
607
  {
608
  "epoch": 0.63,
609
- "eval_debug/losses": 0.13651318848133087,
610
- "eval_debug/policy_weights": 0.26656585931777954,
611
- "eval_debug/raw_losses": 0.49531808495521545,
612
- "eval_logits/chosen": -2.2618136405944824,
613
- "eval_logits/rejected": -2.228300094604492,
614
- "eval_logps/chosen": -405.40423583984375,
615
- "eval_logps/rejected": -493.411376953125,
616
- "eval_loss": 0.13676287233829498,
617
- "eval_rewards/accuracies": 0.7578125,
618
- "eval_rewards/chosen": -1.5020815134048462,
619
- "eval_rewards/margins": 0.8694581985473633,
620
- "eval_rewards/rejected": -2.371539831161499,
621
- "eval_runtime": 43.5081,
622
- "eval_samples_per_second": 45.968,
623
- "eval_steps_per_second": 0.735,
624
  "step": 300
625
  },
626
  {
627
- "debug/losses": 0.1321336030960083,
628
- "debug/policy_weights": 0.2647426724433899,
629
- "debug/raw_losses": 0.4956802725791931,
630
  "epoch": 0.65,
631
- "grad_norm": 7.197171827503445,
632
  "learning_rate": 1.6583128063291573e-07,
633
- "logits/chosen": -2.186096429824829,
634
- "logits/rejected": -2.185868501663208,
635
- "logps/chosen": -443.67803955078125,
636
- "logps/rejected": -486.25091552734375,
637
- "loss": 0.1268,
638
- "rewards/accuracies": 0.768750011920929,
639
- "rewards/chosen": -1.5781536102294922,
640
- "rewards/margins": 0.7728461027145386,
641
- "rewards/rejected": -2.3510000705718994,
642
  "step": 310
643
  },
644
  {
645
- "debug/losses": 0.13546636700630188,
646
- "debug/policy_weights": 0.2598869204521179,
647
- "debug/raw_losses": 0.5187257528305054,
648
  "epoch": 0.67,
649
- "grad_norm": 7.989288502873238,
650
  "learning_rate": 1.488723393865766e-07,
651
- "logits/chosen": -2.2410237789154053,
652
- "logits/rejected": -2.2062346935272217,
653
- "logps/chosen": -433.8523864746094,
654
- "logps/rejected": -467.3929138183594,
655
- "loss": 0.1308,
656
- "rewards/accuracies": 0.7124999761581421,
657
- "rewards/chosen": -1.493381142616272,
658
- "rewards/margins": 0.7878578901290894,
659
- "rewards/rejected": -2.2812390327453613,
660
  "step": 320
661
  },
662
  {
663
- "debug/losses": 0.14906349778175354,
664
- "debug/policy_weights": 0.2727554440498352,
665
- "debug/raw_losses": 0.5552006959915161,
666
  "epoch": 0.69,
667
- "grad_norm": 6.170487059058624,
668
  "learning_rate": 1.3245295796480788e-07,
669
- "logits/chosen": -2.236531972885132,
670
- "logits/rejected": -2.1864519119262695,
671
- "logps/chosen": -390.4073791503906,
672
- "logps/rejected": -459.62347412109375,
673
- "loss": 0.1411,
674
- "rewards/accuracies": 0.7124999761581421,
675
- "rewards/chosen": -1.373223066329956,
676
- "rewards/margins": 0.6836794018745422,
677
- "rewards/rejected": -2.0569024085998535,
678
  "step": 330
679
  },
680
  {
681
- "debug/losses": 0.1494503617286682,
682
- "debug/policy_weights": 0.28068724274635315,
683
- "debug/raw_losses": 0.4996885359287262,
684
  "epoch": 0.71,
685
- "grad_norm": 9.416700000227674,
686
  "learning_rate": 1.1666074087171627e-07,
687
- "logits/chosen": -2.2197887897491455,
688
- "logits/rejected": -2.1604628562927246,
689
- "logps/chosen": -418.9659118652344,
690
- "logps/rejected": -511.0225524902344,
691
- "loss": 0.1446,
692
- "rewards/accuracies": 0.800000011920929,
693
- "rewards/chosen": -1.3679134845733643,
694
- "rewards/margins": 1.0408259630203247,
695
- "rewards/rejected": -2.4087395668029785,
696
  "step": 340
697
  },
698
  {
699
- "debug/losses": 0.12328938394784927,
700
- "debug/policy_weights": 0.24812527000904083,
701
- "debug/raw_losses": 0.48738375306129456,
702
  "epoch": 0.73,
703
- "grad_norm": 8.40666805347326,
704
  "learning_rate": 1.0157994641835734e-07,
705
- "logits/chosen": -2.172837018966675,
706
- "logits/rejected": -2.1325600147247314,
707
- "logps/chosen": -398.5860900878906,
708
- "logps/rejected": -471.38397216796875,
709
- "loss": 0.1286,
710
- "rewards/accuracies": 0.7124999761581421,
711
- "rewards/chosen": -1.4896047115325928,
712
- "rewards/margins": 0.9131780862808228,
713
- "rewards/rejected": -2.402782917022705,
714
  "step": 350
715
  },
716
  {
717
- "debug/losses": 0.1280422806739807,
718
- "debug/policy_weights": 0.2567124366760254,
719
- "debug/raw_losses": 0.5079829692840576,
720
  "epoch": 0.75,
721
- "grad_norm": 7.5652554505895955,
722
  "learning_rate": 8.729103716819111e-08,
723
- "logits/chosen": -2.230677366256714,
724
- "logits/rejected": -2.138909101486206,
725
- "logps/chosen": -448.53179931640625,
726
- "logps/rejected": -494.7422790527344,
727
- "loss": 0.1317,
728
- "rewards/accuracies": 0.7437499761581421,
729
- "rewards/chosen": -1.5306211709976196,
730
- "rewards/margins": 0.8633092641830444,
731
- "rewards/rejected": -2.393930435180664,
732
  "step": 360
733
  },
734
  {
735
- "debug/losses": 0.13079939782619476,
736
- "debug/policy_weights": 0.23427358269691467,
737
- "debug/raw_losses": 0.5890456438064575,
738
  "epoch": 0.77,
739
- "grad_norm": 9.87793971826204,
740
  "learning_rate": 7.387025063449081e-08,
741
- "logits/chosen": -2.167722225189209,
742
- "logits/rejected": -2.1336138248443604,
743
- "logps/chosen": -432.41314697265625,
744
- "logps/rejected": -470.364501953125,
745
- "loss": 0.1313,
746
- "rewards/accuracies": 0.71875,
747
- "rewards/chosen": -1.6386140584945679,
748
- "rewards/margins": 0.7472943067550659,
749
- "rewards/rejected": -2.385908365249634,
750
  "step": 370
751
  },
752
  {
753
- "debug/losses": 0.1057448610663414,
754
- "debug/policy_weights": 0.2194373905658722,
755
- "debug/raw_losses": 0.5027541518211365,
756
  "epoch": 0.79,
757
- "grad_norm": 7.002597773553437,
758
  "learning_rate": 6.138919252022435e-08,
759
- "logits/chosen": -2.093229055404663,
760
- "logits/rejected": -2.0845484733581543,
761
- "logps/chosen": -409.2686462402344,
762
- "logps/rejected": -525.614013671875,
763
- "loss": 0.1212,
764
- "rewards/accuracies": 0.75,
765
- "rewards/chosen": -1.7782268524169922,
766
- "rewards/margins": 0.96544349193573,
767
- "rewards/rejected": -2.743670701980591,
768
  "step": 380
769
  },
770
  {
771
- "debug/losses": 0.14649717509746552,
772
- "debug/policy_weights": 0.27705007791519165,
773
- "debug/raw_losses": 0.5246635675430298,
774
  "epoch": 0.82,
775
- "grad_norm": 7.870379900484857,
776
  "learning_rate": 4.991445467064689e-08,
777
- "logits/chosen": -2.1749279499053955,
778
- "logits/rejected": -2.138232946395874,
779
- "logps/chosen": -439.6005859375,
780
- "logps/rejected": -510.23260498046875,
781
- "loss": 0.1384,
782
- "rewards/accuracies": 0.71875,
783
- "rewards/chosen": -1.4645540714263916,
784
- "rewards/margins": 0.8761545419692993,
785
- "rewards/rejected": -2.3407082557678223,
786
  "step": 390
787
  },
788
  {
789
- "debug/losses": 0.1481957882642746,
790
- "debug/policy_weights": 0.25738272070884705,
791
- "debug/raw_losses": 0.5208276510238647,
792
  "epoch": 0.84,
793
- "grad_norm": 7.317947546801369,
794
  "learning_rate": 3.9507259776993954e-08,
795
- "logits/chosen": -2.1648905277252197,
796
- "logits/rejected": -2.1050162315368652,
797
- "logps/chosen": -416.4757385253906,
798
- "logps/rejected": -496.00927734375,
799
- "loss": 0.1398,
800
- "rewards/accuracies": 0.75,
801
- "rewards/chosen": -1.536012887954712,
802
- "rewards/margins": 0.8311750292778015,
803
- "rewards/rejected": -2.367187976837158,
804
  "step": 400
805
  },
806
  {
807
  "epoch": 0.84,
808
- "eval_debug/losses": 0.13974608480930328,
809
- "eval_debug/policy_weights": 0.28152135014533997,
810
- "eval_debug/raw_losses": 0.47273018956184387,
811
- "eval_logits/chosen": -2.2036194801330566,
812
- "eval_logits/rejected": -2.1694607734680176,
813
- "eval_logps/chosen": -386.73663330078125,
814
- "eval_logps/rejected": -483.9327392578125,
815
- "eval_loss": 0.14222779870033264,
816
- "eval_rewards/accuracies": 0.76171875,
817
- "eval_rewards/chosen": -1.3154058456420898,
818
- "eval_rewards/margins": 0.9613481163978577,
819
- "eval_rewards/rejected": -2.2767536640167236,
820
- "eval_runtime": 98.9464,
821
- "eval_samples_per_second": 20.213,
822
- "eval_steps_per_second": 0.323,
823
  "step": 400
824
  },
825
  {
826
- "debug/losses": 0.13129359483718872,
827
- "debug/policy_weights": 0.25963759422302246,
828
- "debug/raw_losses": 0.5042656660079956,
829
  "epoch": 0.86,
830
- "grad_norm": 7.093812921964164,
831
  "learning_rate": 3.022313472693447e-08,
832
- "logits/chosen": -2.2121942043304443,
833
- "logits/rejected": -2.1647019386291504,
834
- "logps/chosen": -432.4383850097656,
835
- "logps/rejected": -489.4990234375,
836
- "loss": 0.1404,
837
- "rewards/accuracies": 0.75,
838
- "rewards/chosen": -1.4152858257293701,
839
- "rewards/margins": 0.8453054428100586,
840
- "rewards/rejected": -2.260591506958008,
841
  "step": 410
842
  },
843
  {
844
- "debug/losses": 0.15901803970336914,
845
- "debug/policy_weights": 0.2908507287502289,
846
- "debug/raw_losses": 0.5441660284996033,
847
  "epoch": 0.88,
848
- "grad_norm": 7.194889894590537,
849
  "learning_rate": 2.2111614344599684e-08,
850
- "logits/chosen": -2.1668639183044434,
851
- "logits/rejected": -2.1467199325561523,
852
- "logps/chosen": -427.835693359375,
853
- "logps/rejected": -492.3506774902344,
854
- "loss": 0.1412,
855
- "rewards/accuracies": 0.6937500238418579,
856
- "rewards/chosen": -1.4115890264511108,
857
- "rewards/margins": 0.840238094329834,
858
- "rewards/rejected": -2.2518272399902344,
859
  "step": 420
860
  },
861
  {
862
- "debug/losses": 0.1401083916425705,
863
- "debug/policy_weights": 0.2716709077358246,
864
- "debug/raw_losses": 0.49931907653808594,
865
  "epoch": 0.9,
866
- "grad_norm": 6.971935536096578,
867
  "learning_rate": 1.521597710086439e-08,
868
- "logits/chosen": -2.106959342956543,
869
- "logits/rejected": -2.051405668258667,
870
- "logps/chosen": -412.853759765625,
871
- "logps/rejected": -481.4043884277344,
872
- "loss": 0.1291,
873
- "rewards/accuracies": 0.7875000238418579,
874
- "rewards/chosen": -1.4160857200622559,
875
- "rewards/margins": 0.8900654911994934,
876
- "rewards/rejected": -2.3061509132385254,
877
  "step": 430
878
  },
879
  {
880
- "debug/losses": 0.14032089710235596,
881
- "debug/policy_weights": 0.28002962470054626,
882
- "debug/raw_losses": 0.4943667948246002,
883
  "epoch": 0.92,
884
- "grad_norm": 7.598363791526828,
885
  "learning_rate": 9.57301420397924e-09,
886
- "logits/chosen": -2.1854939460754395,
887
- "logits/rejected": -2.1297545433044434,
888
- "logps/chosen": -409.72625732421875,
889
- "logps/rejected": -484.60546875,
890
- "loss": 0.1401,
891
- "rewards/accuracies": 0.737500011920929,
892
- "rewards/chosen": -1.338160753250122,
893
- "rewards/margins": 0.8834611773490906,
894
- "rewards/rejected": -2.2216219902038574,
895
  "step": 440
896
  },
897
  {
898
- "debug/losses": 0.1532571017742157,
899
- "debug/policy_weights": 0.28182727098464966,
900
- "debug/raw_losses": 0.5403295159339905,
901
  "epoch": 0.94,
902
- "grad_norm": 7.420969377583743,
903
  "learning_rate": 5.212833302556258e-09,
904
- "logits/chosen": -2.108179807662964,
905
- "logits/rejected": -2.0896925926208496,
906
- "logps/chosen": -421.1512756347656,
907
- "logps/rejected": -522.5540771484375,
908
- "loss": 0.1398,
909
- "rewards/accuracies": 0.699999988079071,
910
- "rewards/chosen": -1.3998409509658813,
911
- "rewards/margins": 0.8198366165161133,
912
- "rewards/rejected": -2.219677448272705,
913
  "step": 450
914
  },
915
  {
916
- "debug/losses": 0.14014151692390442,
917
- "debug/policy_weights": 0.27090734243392944,
918
- "debug/raw_losses": 0.5257419347763062,
919
  "epoch": 0.96,
920
- "grad_norm": 6.950959627699968,
921
  "learning_rate": 2.158697848236607e-09,
922
- "logits/chosen": -2.1658027172088623,
923
- "logits/rejected": -2.1314563751220703,
924
- "logps/chosen": -402.38995361328125,
925
- "logps/rejected": -448.8475646972656,
926
- "loss": 0.1362,
927
- "rewards/accuracies": 0.71875,
928
- "rewards/chosen": -1.3720080852508545,
929
- "rewards/margins": 0.7829836010932922,
930
- "rewards/rejected": -2.154991626739502,
931
  "step": 460
932
  },
933
  {
934
- "debug/losses": 0.14437520503997803,
935
- "debug/policy_weights": 0.27037832140922546,
936
- "debug/raw_losses": 0.5343233942985535,
937
  "epoch": 0.98,
938
- "grad_norm": 8.44671954335882,
939
  "learning_rate": 4.269029751107489e-10,
940
- "logits/chosen": -2.151285171508789,
941
- "logits/rejected": -2.119642972946167,
942
- "logps/chosen": -403.640625,
943
- "logps/rejected": -483.1455078125,
944
- "loss": 0.1371,
945
- "rewards/accuracies": 0.7124999761581421,
946
- "rewards/chosen": -1.3858269453048706,
947
- "rewards/margins": 0.757634699344635,
948
- "rewards/rejected": -2.1434617042541504,
949
  "step": 470
950
  },
951
  {
952
  "epoch": 1.0,
953
  "step": 478,
954
  "total_flos": 0.0,
955
- "train_loss": 0.17903520272865456,
956
- "train_runtime": 4631.245,
957
- "train_samples_per_second": 13.2,
958
- "train_steps_per_second": 0.103
959
  }
960
  ],
961
  "logging_steps": 10,
962
  "max_steps": 478,
963
- "num_input_tokens_seen": 0,
964
  "num_train_epochs": 1,
965
  "save_steps": 100,
966
  "total_flos": 0.0,
967
- "train_batch_size": 8,
968
  "trial_name": null,
969
  "trial_params": null
970
  }
 
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
 
 
 
12
  "epoch": 0.0,
 
13
  "learning_rate": 1.0416666666666666e-08,
14
+ "logits/chosen": -2.8099329471588135,
15
+ "logits/rejected": -2.7572641372680664,
16
+ "logps/chosen": -241.48843383789062,
17
+ "logps/rejected": -197.4517822265625,
18
+ "loss": 271.7943,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
23
  "step": 1
24
  },
25
  {
 
 
 
26
  "epoch": 0.02,
 
27
  "learning_rate": 1.0416666666666667e-07,
28
+ "logits/chosen": -2.83237361907959,
29
+ "logits/rejected": -2.808957815170288,
30
+ "logps/chosen": -292.6072692871094,
31
+ "logps/rejected": -278.4604797363281,
32
+ "loss": 286.0386,
33
+ "rewards/accuracies": 0.4166666567325592,
34
+ "rewards/chosen": 0.0008353570010513067,
35
+ "rewards/margins": -0.0004216647648718208,
36
+ "rewards/rejected": 0.0012570219114422798,
37
  "step": 10
38
  },
39
  {
 
 
 
40
  "epoch": 0.04,
 
41
  "learning_rate": 2.0833333333333333e-07,
42
+ "logits/chosen": -2.81878662109375,
43
+ "logits/rejected": -2.7905821800231934,
44
+ "logps/chosen": -286.19378662109375,
45
+ "logps/rejected": -286.7618103027344,
46
+ "loss": 264.9378,
47
  "rewards/accuracies": 0.543749988079071,
48
+ "rewards/chosen": 0.04205816239118576,
49
+ "rewards/margins": 0.0025776384864002466,
50
+ "rewards/rejected": 0.03948052600026131,
51
  "step": 20
52
  },
53
  {
 
 
 
54
  "epoch": 0.06,
 
55
  "learning_rate": 3.1249999999999997e-07,
56
+ "logits/chosen": -2.7921807765960693,
57
+ "logits/rejected": -2.7616498470306396,
58
+ "logps/chosen": -232.4526824951172,
59
+ "logps/rejected": -212.8272705078125,
60
+ "loss": 266.1199,
61
+ "rewards/accuracies": 0.518750011920929,
62
+ "rewards/chosen": 0.13909421861171722,
63
+ "rewards/margins": 0.004568194039165974,
64
+ "rewards/rejected": 0.13452602922916412,
65
  "step": 30
66
  },
67
  {
 
 
 
68
  "epoch": 0.08,
 
69
  "learning_rate": 4.1666666666666667e-07,
70
+ "logits/chosen": -2.8503639698028564,
71
+ "logits/rejected": -2.819370985031128,
72
+ "logps/chosen": -280.4808654785156,
73
+ "logps/rejected": -243.86935424804688,
74
+ "loss": 258.2878,
75
+ "rewards/accuracies": 0.5687500238418579,
76
+ "rewards/chosen": 0.19040736556053162,
77
+ "rewards/margins": 0.01646682806313038,
78
+ "rewards/rejected": 0.1739405393600464,
79
  "step": 40
80
  },
81
  {
 
 
 
82
  "epoch": 0.1,
 
83
  "learning_rate": 4.999733114418725e-07,
84
+ "logits/chosen": -2.764444589614868,
85
+ "logits/rejected": -2.7427210807800293,
86
+ "logps/chosen": -254.5093231201172,
87
+ "logps/rejected": -240.6798858642578,
88
+ "loss": 244.2991,
89
+ "rewards/accuracies": 0.518750011920929,
90
+ "rewards/chosen": 0.22921113669872284,
91
+ "rewards/margins": -0.0009602505015209317,
92
+ "rewards/rejected": 0.23017136752605438,
93
  "step": 50
94
  },
95
  {
 
 
 
96
  "epoch": 0.13,
 
97
  "learning_rate": 4.990398100856366e-07,
98
+ "logits/chosen": -2.7702980041503906,
99
+ "logits/rejected": -2.742863893508911,
100
+ "logps/chosen": -235.5380401611328,
101
+ "logps/rejected": -211.11770629882812,
102
+ "loss": 255.6714,
103
+ "rewards/accuracies": 0.5562499761581421,
104
+ "rewards/chosen": 0.23538246750831604,
105
+ "rewards/margins": 0.020409177988767624,
106
+ "rewards/rejected": 0.2149733006954193,
107
  "step": 60
108
  },
109
  {
 
 
 
110
  "epoch": 0.15,
 
111
  "learning_rate": 4.967775735898179e-07,
112
+ "logits/chosen": -2.7509877681732178,
113
+ "logits/rejected": -2.7182183265686035,
114
+ "logps/chosen": -239.36514282226562,
115
+ "logps/rejected": -210.6224822998047,
116
+ "loss": 253.836,
117
+ "rewards/accuracies": 0.606249988079071,
118
+ "rewards/chosen": 0.29057446122169495,
119
+ "rewards/margins": 0.04796246066689491,
120
+ "rewards/rejected": 0.24261197447776794,
121
  "step": 70
122
  },
123
  {
 
 
 
124
  "epoch": 0.17,
 
125
  "learning_rate": 4.931986719649298e-07,
126
+ "logits/chosen": -2.699618339538574,
127
+ "logits/rejected": -2.697359800338745,
128
+ "logps/chosen": -241.5849609375,
129
+ "logps/rejected": -241.43533325195312,
130
+ "loss": 261.2479,
131
+ "rewards/accuracies": 0.518750011920929,
132
+ "rewards/chosen": 0.29146844148635864,
133
+ "rewards/margins": 0.030574629083275795,
134
+ "rewards/rejected": 0.2608937919139862,
135
  "step": 80
136
  },
137
  {
 
 
 
138
  "epoch": 0.19,
 
139
  "learning_rate": 4.883222001996351e-07,
140
+ "logits/chosen": -2.7679967880249023,
141
+ "logits/rejected": -2.7326865196228027,
142
+ "logps/chosen": -225.57156372070312,
143
+ "logps/rejected": -208.76693725585938,
144
+ "loss": 252.1535,
145
+ "rewards/accuracies": 0.6187499761581421,
146
+ "rewards/chosen": 0.29392915964126587,
147
+ "rewards/margins": 0.03465462103486061,
148
+ "rewards/rejected": 0.25927454233169556,
149
  "step": 90
150
  },
151
  {
 
 
 
152
  "epoch": 0.21,
 
153
  "learning_rate": 4.821741763807186e-07,
154
+ "logits/chosen": -2.819113254547119,
155
+ "logits/rejected": -2.783386707305908,
156
+ "logps/chosen": -260.7179260253906,
157
+ "logps/rejected": -238.54434204101562,
158
+ "loss": 256.7688,
159
+ "rewards/accuracies": 0.550000011920929,
160
+ "rewards/chosen": 0.28721123933792114,
161
+ "rewards/margins": -0.005946027580648661,
162
+ "rewards/rejected": 0.29315727949142456,
163
  "step": 100
164
  },
165
  {
166
  "epoch": 0.21,
167
+ "eval_logits/chosen": -2.7863247394561768,
168
+ "eval_logits/rejected": -2.767011880874634,
169
+ "eval_logps/chosen": -227.56491088867188,
170
+ "eval_logps/rejected": -230.47303771972656,
171
+ "eval_loss": 245.72764587402344,
172
+ "eval_rewards/accuracies": 0.57421875,
173
+ "eval_rewards/chosen": 0.29474756121635437,
174
+ "eval_rewards/margins": 0.025945277884602547,
175
+ "eval_rewards/rejected": 0.268802285194397,
176
+ "eval_runtime": 53.5253,
177
+ "eval_samples_per_second": 37.365,
178
+ "eval_steps_per_second": 0.598,
 
 
 
179
  "step": 100
180
  },
181
  {
 
 
 
182
  "epoch": 0.23,
 
183
  "learning_rate": 4.747874028753375e-07,
184
+ "logits/chosen": -2.6783273220062256,
185
+ "logits/rejected": -2.644078016281128,
186
+ "logps/chosen": -237.9336395263672,
187
+ "logps/rejected": -188.83895874023438,
188
+ "loss": 240.9402,
189
+ "rewards/accuracies": 0.550000011920929,
190
+ "rewards/chosen": 0.32163843512535095,
191
+ "rewards/margins": -0.008066670037806034,
192
+ "rewards/rejected": 0.3297051787376404,
193
  "step": 110
194
  },
195
  {
 
 
 
196
  "epoch": 0.25,
 
197
  "learning_rate": 4.662012913161997e-07,
198
+ "logits/chosen": -2.746994733810425,
199
+ "logits/rejected": -2.724551200866699,
200
+ "logps/chosen": -240.14242553710938,
201
+ "logps/rejected": -237.6074981689453,
202
+ "loss": 245.7042,
203
+ "rewards/accuracies": 0.53125,
204
+ "rewards/chosen": 0.2870863080024719,
205
+ "rewards/margins": 0.03465163707733154,
206
+ "rewards/rejected": 0.25243470072746277,
207
  "step": 120
208
  },
209
  {
 
 
 
210
  "epoch": 0.27,
 
211
  "learning_rate": 4.5646165232345103e-07,
212
+ "logits/chosen": -2.7405877113342285,
213
+ "logits/rejected": -2.7226414680480957,
214
+ "logps/chosen": -242.31460571289062,
215
+ "logps/rejected": -222.0249786376953,
216
+ "loss": 239.1022,
217
+ "rewards/accuracies": 0.574999988079071,
218
+ "rewards/chosen": 0.3218019902706146,
219
+ "rewards/margins": 0.03552493453025818,
220
+ "rewards/rejected": 0.28627708554267883,
221
  "step": 130
222
  },
223
  {
 
 
 
224
  "epoch": 0.29,
 
225
  "learning_rate": 4.456204510851956e-07,
226
+ "logits/chosen": -2.7363457679748535,
227
+ "logits/rejected": -2.710092067718506,
228
+ "logps/chosen": -248.31982421875,
229
+ "logps/rejected": -242.44064331054688,
230
+ "loss": 240.1779,
231
+ "rewards/accuracies": 0.550000011920929,
232
+ "rewards/chosen": 0.32314616441726685,
233
+ "rewards/margins": 0.03991778939962387,
234
+ "rewards/rejected": 0.2832283675670624,
235
  "step": 140
236
  },
237
  {
 
 
 
238
  "epoch": 0.31,
 
239
  "learning_rate": 4.337355301007335e-07,
240
+ "logits/chosen": -2.6783087253570557,
241
+ "logits/rejected": -2.707118511199951,
242
+ "logps/chosen": -196.87429809570312,
243
+ "logps/rejected": -205.2587890625,
244
+ "loss": 236.8216,
245
+ "rewards/accuracies": 0.46875,
246
+ "rewards/chosen": 0.2980460226535797,
247
+ "rewards/margins": -0.012227327562868595,
248
+ "rewards/rejected": 0.31027334928512573,
249
  "step": 150
250
  },
251
  {
 
 
 
252
  "epoch": 0.33,
 
253
  "learning_rate": 4.2087030056579986e-07,
254
+ "logits/chosen": -2.7062594890594482,
255
+ "logits/rejected": -2.6792685985565186,
256
+ "logps/chosen": -224.0284881591797,
257
+ "logps/rejected": -215.8988494873047,
258
+ "loss": 233.851,
259
+ "rewards/accuracies": 0.518750011920929,
260
+ "rewards/chosen": 0.31943365931510925,
261
+ "rewards/margins": 0.01964866928756237,
262
+ "rewards/rejected": 0.29978498816490173,
263
  "step": 160
264
  },
265
  {
 
 
 
266
  "epoch": 0.36,
 
267
  "learning_rate": 4.070934040463998e-07,
268
+ "logits/chosen": -2.6613574028015137,
269
+ "logits/rejected": -2.6377127170562744,
270
+ "logps/chosen": -249.140869140625,
271
+ "logps/rejected": -229.2135009765625,
272
+ "loss": 238.7709,
273
+ "rewards/accuracies": 0.574999988079071,
274
+ "rewards/chosen": 0.3302595019340515,
275
+ "rewards/margins": 0.02071164920926094,
276
+ "rewards/rejected": 0.30954790115356445,
277
  "step": 170
278
  },
279
  {
 
 
 
280
  "epoch": 0.38,
 
281
  "learning_rate": 3.9247834624635404e-07,
282
+ "logits/chosen": -2.697937488555908,
283
+ "logits/rejected": -2.6982693672180176,
284
+ "logps/chosen": -237.61752319335938,
285
+ "logps/rejected": -222.94534301757812,
286
+ "loss": 240.8094,
287
+ "rewards/accuracies": 0.5375000238418579,
288
+ "rewards/chosen": 0.3156043589115143,
289
+ "rewards/margins": -0.00877897534519434,
290
+ "rewards/rejected": 0.32438334822654724,
291
  "step": 180
292
  },
293
  {
 
 
 
294
  "epoch": 0.4,
 
295
  "learning_rate": 3.7710310482256523e-07,
296
+ "logits/chosen": -2.716660499572754,
297
+ "logits/rejected": -2.6782338619232178,
298
+ "logps/chosen": -256.08953857421875,
299
+ "logps/rejected": -233.2887725830078,
300
+ "loss": 241.1514,
301
+ "rewards/accuracies": 0.612500011920929,
302
+ "rewards/chosen": 0.35306206345558167,
303
+ "rewards/margins": 0.0451970100402832,
304
+ "rewards/rejected": 0.3078650236129761,
305
  "step": 190
306
  },
307
  {
 
 
 
308
  "epoch": 0.42,
 
309
  "learning_rate": 3.610497133404795e-07,
310
+ "logits/chosen": -2.6975326538085938,
311
+ "logits/rejected": -2.715744733810425,
312
+ "logps/chosen": -260.04095458984375,
313
+ "logps/rejected": -232.90744018554688,
314
+ "loss": 239.4804,
315
+ "rewards/accuracies": 0.5562499761581421,
316
+ "rewards/chosen": 0.3612653315067291,
317
+ "rewards/margins": 0.047182150185108185,
318
+ "rewards/rejected": 0.31408315896987915,
319
  "step": 200
320
  },
321
  {
322
  "epoch": 0.42,
323
+ "eval_logits/chosen": -2.7289865016937256,
324
+ "eval_logits/rejected": -2.7082109451293945,
325
+ "eval_logps/chosen": -223.84622192382812,
326
+ "eval_logps/rejected": -226.99331665039062,
327
+ "eval_loss": 241.5078125,
328
+ "eval_rewards/accuracies": 0.57421875,
329
+ "eval_rewards/chosen": 0.33193421363830566,
330
+ "eval_rewards/margins": 0.028334595263004303,
331
+ "eval_rewards/rejected": 0.30359959602355957,
332
+ "eval_runtime": 53.4363,
333
+ "eval_samples_per_second": 37.428,
334
+ "eval_steps_per_second": 0.599,
 
 
 
335
  "step": 200
336
  },
337
  {
 
 
 
338
  "epoch": 0.44,
 
339
  "learning_rate": 3.4440382358952115e-07,
340
+ "logits/chosen": -2.653046131134033,
341
+ "logits/rejected": -2.639833927154541,
342
+ "logps/chosen": -236.7248992919922,
343
+ "logps/rejected": -215.8306427001953,
344
+ "loss": 232.2403,
345
+ "rewards/accuracies": 0.518750011920929,
346
+ "rewards/chosen": 0.3630830645561218,
347
+ "rewards/margins": 0.002032138407230377,
348
+ "rewards/rejected": 0.36105093359947205,
349
  "step": 210
350
  },
351
  {
 
 
 
352
  "epoch": 0.46,
 
353
  "learning_rate": 3.272542485937368e-07,
354
+ "logits/chosen": -2.6577677726745605,
355
+ "logits/rejected": -2.610652446746826,
356
+ "logps/chosen": -234.62515258789062,
357
+ "logps/rejected": -224.5327911376953,
358
+ "loss": 236.0736,
359
+ "rewards/accuracies": 0.5625,
360
+ "rewards/chosen": 0.403405100107193,
361
+ "rewards/margins": 0.025665929540991783,
362
+ "rewards/rejected": 0.3777391314506531,
363
  "step": 220
364
  },
365
  {
 
 
 
366
  "epoch": 0.48,
 
367
  "learning_rate": 3.096924887558854e-07,
368
+ "logits/chosen": -2.7138988971710205,
369
+ "logits/rejected": -2.6695990562438965,
370
+ "logps/chosen": -238.20166015625,
371
+ "logps/rejected": -238.32510375976562,
372
+ "loss": 243.5431,
373
+ "rewards/accuracies": 0.637499988079071,
374
+ "rewards/chosen": 0.36346226930618286,
375
+ "rewards/margins": 0.04655776172876358,
376
+ "rewards/rejected": 0.31690454483032227,
377
  "step": 230
378
  },
379
  {
 
 
 
380
  "epoch": 0.5,
 
381
  "learning_rate": 2.9181224366319943e-07,
382
+ "logits/chosen": -2.6672616004943848,
383
+ "logits/rejected": -2.6338298320770264,
384
+ "logps/chosen": -245.7289581298828,
385
+ "logps/rejected": -222.80068969726562,
386
+ "loss": 241.8247,
387
+ "rewards/accuracies": 0.612500011920929,
388
+ "rewards/chosen": 0.4044111371040344,
389
+ "rewards/margins": 0.09945273399353027,
390
+ "rewards/rejected": 0.30495840311050415,
391
  "step": 240
392
  },
393
  {
 
 
 
394
  "epoch": 0.52,
 
395
  "learning_rate": 2.7370891215954565e-07,
396
+ "logits/chosen": -2.711887836456299,
397
+ "logits/rejected": -2.660736083984375,
398
+ "logps/chosen": -236.48410034179688,
399
+ "logps/rejected": -220.5592803955078,
400
+ "loss": 232.2589,
401
+ "rewards/accuracies": 0.550000011920929,
402
+ "rewards/chosen": 0.3508817255496979,
403
+ "rewards/margins": 0.012243595905601978,
404
+ "rewards/rejected": 0.3386381268501282,
405
  "step": 250
406
  },
407
  {
 
 
 
408
  "epoch": 0.54,
 
409
  "learning_rate": 2.55479083351317e-07,
410
+ "logits/chosen": -2.6873819828033447,
411
+ "logits/rejected": -2.665743827819824,
412
+ "logps/chosen": -226.0983428955078,
413
+ "logps/rejected": -222.646484375,
414
+ "loss": 249.1208,
415
+ "rewards/accuracies": 0.48124998807907104,
416
+ "rewards/chosen": 0.33049115538597107,
417
+ "rewards/margins": -0.025500113144516945,
418
+ "rewards/rejected": 0.35599130392074585,
419
  "step": 260
420
  },
421
  {
 
 
 
422
  "epoch": 0.56,
 
423
  "learning_rate": 2.3722002126275822e-07,
424
+ "logits/chosen": -2.684255599975586,
425
+ "logits/rejected": -2.6710708141326904,
426
+ "logps/chosen": -250.91659545898438,
427
+ "logps/rejected": -232.743896484375,
428
+ "loss": 232.0498,
429
+ "rewards/accuracies": 0.5625,
430
+ "rewards/chosen": 0.3753630220890045,
431
+ "rewards/margins": 0.05215846374630928,
432
+ "rewards/rejected": 0.32320457696914673,
433
  "step": 270
434
  },
435
  {
 
 
 
436
  "epoch": 0.59,
 
437
  "learning_rate": 2.19029145890313e-07,
438
+ "logits/chosen": -2.642585515975952,
439
+ "logits/rejected": -2.6435678005218506,
440
+ "logps/chosen": -271.9496154785156,
441
+ "logps/rejected": -232.7229461669922,
442
+ "loss": 242.3832,
443
+ "rewards/accuracies": 0.5625,
444
+ "rewards/chosen": 0.3579340875148773,
445
+ "rewards/margins": 0.029489517211914062,
446
+ "rewards/rejected": 0.32844457030296326,
447
  "step": 280
448
  },
449
  {
 
 
 
450
  "epoch": 0.61,
 
451
  "learning_rate": 2.0100351342479216e-07,
452
+ "logits/chosen": -2.6854400634765625,
453
+ "logits/rejected": -2.6508984565734863,
454
+ "logps/chosen": -238.96533203125,
455
+ "logps/rejected": -214.7186737060547,
456
+ "loss": 237.8058,
457
+ "rewards/accuracies": 0.518750011920929,
458
+ "rewards/chosen": 0.38798388838768005,
459
+ "rewards/margins": 0.002142349723726511,
460
+ "rewards/rejected": 0.3858415484428406,
461
  "step": 290
462
  },
463
  {
 
 
 
464
  "epoch": 0.63,
 
465
  "learning_rate": 1.8323929841460178e-07,
466
+ "logits/chosen": -2.6979777812957764,
467
+ "logits/rejected": -2.6885297298431396,
468
+ "logps/chosen": -218.0729522705078,
469
+ "logps/rejected": -206.6052703857422,
470
+ "loss": 240.5041,
471
+ "rewards/accuracies": 0.4749999940395355,
472
+ "rewards/chosen": 0.3353363871574402,
473
+ "rewards/margins": -0.021952930837869644,
474
+ "rewards/rejected": 0.35728925466537476,
475
  "step": 300
476
  },
477
  {
478
  "epoch": 0.63,
479
+ "eval_logits/chosen": -2.7041783332824707,
480
+ "eval_logits/rejected": -2.680725574493408,
481
+ "eval_logps/chosen": -222.30435180664062,
482
+ "eval_logps/rejected": -225.71856689453125,
483
+ "eval_loss": 239.8052978515625,
484
+ "eval_rewards/accuracies": 0.5625,
485
+ "eval_rewards/chosen": 0.3473527431488037,
486
+ "eval_rewards/margins": 0.03100587986409664,
487
+ "eval_rewards/rejected": 0.3163468837738037,
488
+ "eval_runtime": 53.5009,
489
+ "eval_samples_per_second": 37.383,
490
+ "eval_steps_per_second": 0.598,
 
 
 
491
  "step": 300
492
  },
493
  {
 
 
 
494
  "epoch": 0.65,
 
495
  "learning_rate": 1.6583128063291573e-07,
496
+ "logits/chosen": -2.681898832321167,
497
+ "logits/rejected": -2.647827625274658,
498
+ "logps/chosen": -249.84933471679688,
499
+ "logps/rejected": -221.06466674804688,
500
+ "loss": 244.5816,
501
+ "rewards/accuracies": 0.581250011920929,
502
+ "rewards/chosen": 0.37856870889663696,
503
+ "rewards/margins": 0.031483568251132965,
504
+ "rewards/rejected": 0.347085177898407,
505
  "step": 310
506
  },
507
  {
 
 
 
508
  "epoch": 0.67,
 
509
  "learning_rate": 1.488723393865766e-07,
510
+ "logits/chosen": -2.6419777870178223,
511
+ "logits/rejected": -2.6089978218078613,
512
+ "logps/chosen": -247.5414581298828,
513
+ "logps/rejected": -228.00442504882812,
514
+ "loss": 227.1146,
515
+ "rewards/accuracies": 0.5625,
516
+ "rewards/chosen": 0.38319897651672363,
517
+ "rewards/margins": 0.010034086182713509,
518
+ "rewards/rejected": 0.3731648921966553,
519
  "step": 320
520
  },
521
  {
 
 
 
522
  "epoch": 0.69,
 
523
  "learning_rate": 1.3245295796480788e-07,
524
+ "logits/chosen": -2.6546921730041504,
525
+ "logits/rejected": -2.619576930999756,
526
+ "logps/chosen": -235.1280059814453,
527
+ "logps/rejected": -228.36550903320312,
528
+ "loss": 235.3102,
529
+ "rewards/accuracies": 0.59375,
530
+ "rewards/chosen": 0.3890138268470764,
531
+ "rewards/margins": 0.030328262597322464,
532
+ "rewards/rejected": 0.35868555307388306,
533
  "step": 330
534
  },
535
  {
 
 
 
536
  "epoch": 0.71,
 
537
  "learning_rate": 1.1666074087171627e-07,
538
+ "logits/chosen": -2.6908440589904785,
539
+ "logits/rejected": -2.6842880249023438,
540
+ "logps/chosen": -221.61752319335938,
541
+ "logps/rejected": -221.1806182861328,
542
+ "loss": 226.7886,
543
+ "rewards/accuracies": 0.5062500238418579,
544
+ "rewards/chosen": 0.3297863006591797,
545
+ "rewards/margins": 0.012335492298007011,
546
+ "rewards/rejected": 0.3174508213996887,
547
  "step": 340
548
  },
549
  {
 
 
 
550
  "epoch": 0.73,
 
551
  "learning_rate": 1.0157994641835734e-07,
552
+ "logits/chosen": -2.6660337448120117,
553
+ "logits/rejected": -2.6440846920013428,
554
+ "logps/chosen": -235.5915069580078,
555
+ "logps/rejected": -227.4123077392578,
556
+ "loss": 235.1443,
557
+ "rewards/accuracies": 0.5249999761581421,
558
+ "rewards/chosen": 0.4086208939552307,
559
+ "rewards/margins": 0.052063293755054474,
560
+ "rewards/rejected": 0.35655760765075684,
561
  "step": 350
562
  },
563
  {
 
 
 
564
  "epoch": 0.75,
 
565
  "learning_rate": 8.729103716819111e-08,
566
+ "logits/chosen": -2.631333112716675,
567
+ "logits/rejected": -2.605966806411743,
568
+ "logps/chosen": -223.34115600585938,
569
+ "logps/rejected": -207.91500854492188,
570
+ "loss": 246.5647,
571
+ "rewards/accuracies": 0.5249999761581421,
572
+ "rewards/chosen": 0.35043418407440186,
573
+ "rewards/margins": -0.0040539586916565895,
574
+ "rewards/rejected": 0.35448816418647766,
575
  "step": 360
576
  },
577
  {
 
 
 
578
  "epoch": 0.77,
 
579
  "learning_rate": 7.387025063449081e-08,
580
+ "logits/chosen": -2.667217254638672,
581
+ "logits/rejected": -2.6636977195739746,
582
+ "logps/chosen": -240.44247436523438,
583
+ "logps/rejected": -231.893798828125,
584
+ "loss": 236.9863,
585
+ "rewards/accuracies": 0.5874999761581421,
586
+ "rewards/chosen": 0.37617605924606323,
587
+ "rewards/margins": 0.05858853459358215,
588
+ "rewards/rejected": 0.3175875246524811,
589
  "step": 370
590
  },
591
  {
 
 
 
592
  "epoch": 0.79,
 
593
  "learning_rate": 6.138919252022435e-08,
594
+ "logits/chosen": -2.6146702766418457,
595
+ "logits/rejected": -2.611351728439331,
596
+ "logps/chosen": -189.4939422607422,
597
+ "logps/rejected": -206.30081176757812,
598
+ "loss": 233.4487,
599
+ "rewards/accuracies": 0.518750011920929,
600
+ "rewards/chosen": 0.3334718644618988,
601
+ "rewards/margins": -0.03314907103776932,
602
+ "rewards/rejected": 0.3666209578514099,
603
  "step": 380
604
  },
605
  {
 
 
 
606
  "epoch": 0.82,
 
607
  "learning_rate": 4.991445467064689e-08,
608
+ "logits/chosen": -2.663173198699951,
609
+ "logits/rejected": -2.634059429168701,
610
+ "logps/chosen": -221.12240600585938,
611
+ "logps/rejected": -204.8622589111328,
612
+ "loss": 241.2531,
613
+ "rewards/accuracies": 0.53125,
614
+ "rewards/chosen": 0.30437472462654114,
615
+ "rewards/margins": -0.002047918038442731,
616
+ "rewards/rejected": 0.3064226508140564,
617
  "step": 390
618
  },
619
  {
 
 
 
620
  "epoch": 0.84,
 
621
  "learning_rate": 3.9507259776993954e-08,
622
+ "logits/chosen": -2.646238088607788,
623
+ "logits/rejected": -2.6048245429992676,
624
+ "logps/chosen": -250.01693725585938,
625
+ "logps/rejected": -215.4422607421875,
626
+ "loss": 236.8453,
627
+ "rewards/accuracies": 0.581250011920929,
628
+ "rewards/chosen": 0.35686779022216797,
629
+ "rewards/margins": 0.03598792105913162,
630
+ "rewards/rejected": 0.3208799362182617,
631
  "step": 400
632
  },
633
  {
634
  "epoch": 0.84,
635
+ "eval_logits/chosen": -2.697472333908081,
636
+ "eval_logits/rejected": -2.673011541366577,
637
+ "eval_logps/chosen": -221.73487854003906,
638
+ "eval_logps/rejected": -225.18995666503906,
639
+ "eval_loss": 239.1992950439453,
640
+ "eval_rewards/accuracies": 0.5625,
641
+ "eval_rewards/chosen": 0.3530477285385132,
642
+ "eval_rewards/margins": 0.031414665281772614,
643
+ "eval_rewards/rejected": 0.3216330409049988,
644
+ "eval_runtime": 53.4218,
645
+ "eval_samples_per_second": 37.438,
646
+ "eval_steps_per_second": 0.599,
 
 
 
647
  "step": 400
648
  },
649
  {
 
 
 
650
  "epoch": 0.86,
 
651
  "learning_rate": 3.022313472693447e-08,
652
+ "logits/chosen": -2.684819459915161,
653
+ "logits/rejected": -2.622917652130127,
654
+ "logps/chosen": -226.3255157470703,
655
+ "logps/rejected": -246.3280487060547,
656
+ "loss": 234.8678,
657
+ "rewards/accuracies": 0.5562499761581421,
658
+ "rewards/chosen": 0.3726418912410736,
659
+ "rewards/margins": 0.02128712832927704,
660
+ "rewards/rejected": 0.3513546884059906,
661
  "step": 410
662
  },
663
  {
 
 
 
664
  "epoch": 0.88,
 
665
  "learning_rate": 2.2111614344599684e-08,
666
+ "logits/chosen": -2.6737873554229736,
667
+ "logits/rejected": -2.638714551925659,
668
+ "logps/chosen": -250.0021514892578,
669
+ "logps/rejected": -255.8992156982422,
670
+ "loss": 246.8406,
671
+ "rewards/accuracies": 0.581250011920929,
672
+ "rewards/chosen": 0.3681090474128723,
673
+ "rewards/margins": 0.0030112355016171932,
674
+ "rewards/rejected": 0.3650978207588196,
675
  "step": 420
676
  },
677
  {
 
 
 
678
  "epoch": 0.9,
 
679
  "learning_rate": 1.521597710086439e-08,
680
+ "logits/chosen": -2.6430375576019287,
681
+ "logits/rejected": -2.6089277267456055,
682
+ "logps/chosen": -255.19985961914062,
683
+ "logps/rejected": -234.4241485595703,
684
+ "loss": 234.6739,
685
+ "rewards/accuracies": 0.5062500238418579,
686
+ "rewards/chosen": 0.3505329489707947,
687
+ "rewards/margins": -0.049122948199510574,
688
+ "rewards/rejected": 0.39965590834617615,
689
  "step": 430
690
  },
691
  {
 
 
 
692
  "epoch": 0.92,
 
693
  "learning_rate": 9.57301420397924e-09,
694
+ "logits/chosen": -2.665576934814453,
695
+ "logits/rejected": -2.637112617492676,
696
+ "logps/chosen": -278.3708190917969,
697
+ "logps/rejected": -230.59423828125,
698
+ "loss": 242.5247,
699
+ "rewards/accuracies": 0.5562499761581421,
700
+ "rewards/chosen": 0.36680158972740173,
701
+ "rewards/margins": 0.01681143045425415,
702
+ "rewards/rejected": 0.34999018907546997,
703
  "step": 440
704
  },
705
  {
 
 
 
706
  "epoch": 0.94,
 
707
  "learning_rate": 5.212833302556258e-09,
708
+ "logits/chosen": -2.6472713947296143,
709
+ "logits/rejected": -2.6246514320373535,
710
+ "logps/chosen": -242.03775024414062,
711
+ "logps/rejected": -210.9549560546875,
712
+ "loss": 239.1569,
713
+ "rewards/accuracies": 0.5625,
714
+ "rewards/chosen": 0.3881983160972595,
715
+ "rewards/margins": -0.0041311681270599365,
716
+ "rewards/rejected": 0.39232948422431946,
717
  "step": 450
718
  },
719
  {
 
 
 
720
  "epoch": 0.96,
 
721
  "learning_rate": 2.158697848236607e-09,
722
+ "logits/chosen": -2.6596500873565674,
723
+ "logits/rejected": -2.6326732635498047,
724
+ "logps/chosen": -259.3081359863281,
725
+ "logps/rejected": -237.2527618408203,
726
+ "loss": 246.6495,
727
+ "rewards/accuracies": 0.5874999761581421,
728
+ "rewards/chosen": 0.4107862412929535,
729
+ "rewards/margins": 0.07695204019546509,
730
+ "rewards/rejected": 0.3338342308998108,
731
  "step": 460
732
  },
733
  {
 
 
 
734
  "epoch": 0.98,
 
735
  "learning_rate": 4.269029751107489e-10,
736
+ "logits/chosen": -2.6772663593292236,
737
+ "logits/rejected": -2.6103363037109375,
738
+ "logps/chosen": -256.0906677246094,
739
+ "logps/rejected": -202.92539978027344,
740
+ "loss": 248.9801,
741
+ "rewards/accuracies": 0.668749988079071,
742
+ "rewards/chosen": 0.42115721106529236,
743
+ "rewards/margins": 0.12000129371881485,
744
+ "rewards/rejected": 0.3011559247970581,
745
  "step": 470
746
  },
747
  {
748
  "epoch": 1.0,
749
  "step": 478,
750
  "total_flos": 0.0,
751
+ "train_loss": 243.0746703846185,
752
+ "train_runtime": 4321.456,
753
+ "train_samples_per_second": 14.147,
754
+ "train_steps_per_second": 0.111
755
  }
756
  ],
757
  "logging_steps": 10,
758
  "max_steps": 478,
 
759
  "num_train_epochs": 1,
760
  "save_steps": 100,
761
  "total_flos": 0.0,
 
762
  "trial_name": null,
763
  "trial_params": null
764
  }