Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wzhouad commited on
Commit
7cbeac1
•
1 Parent(s): 90273df

Model save

Browse files
Files changed (7) hide show
  1. README.md +14 -22
  2. all_results.json +4 -4
  3. model-00001-of-00003.safetensors +1 -1
  4. model-00002-of-00003.safetensors +1 -1
  5. model-00003-of-00003.safetensors +1 -1
  6. train_results.json +4 -4
  7. trainer_state.json +577 -1797
README.md CHANGED
@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
- - Loss: 0.5440
21
- - Rewards/chosen: -2.2940
22
- - Rewards/rejected: -3.0054
23
- - Rewards/accuracies: 0.7090
24
- - Rewards/margins: 0.7114
25
- - Logps/rejected: -451.6765
26
- - Logps/chosen: -373.9785
27
- - Logits/rejected: 0.3244
28
- - Logits/chosen: 0.0742
29
 
30
  ## Model description
31
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
47
  - learning_rate: 5e-07
48
  - train_batch_size: 8
49
  - eval_batch_size: 8
50
- - seed: 42
51
  - distributed_type: multi-GPU
52
  - num_devices: 8
53
  - gradient_accumulation_steps: 2
@@ -62,18 +62,10 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 0.6789 | 0.08 | 100 | 0.6770 | -0.1062 | -0.1422 | 0.5914 | 0.0360 | -165.3552 | -155.1927 | -2.7255 | -2.7337 |
66
- | 0.6062 | 0.16 | 200 | 0.6079 | -1.0212 | -1.3873 | 0.6670 | 0.3660 | -289.8622 | -246.6971 | -2.3696 | -2.3856 |
67
- | 0.5965 | 0.24 | 300 | 0.5907 | -1.3779 | -1.8008 | 0.6623 | 0.4229 | -331.2100 | -282.3621 | -2.2450 | -2.2656 |
68
- | 0.5729 | 0.32 | 400 | 0.5711 | -1.6763 | -2.2404 | 0.6828 | 0.5640 | -375.1720 | -312.2064 | -1.2920 | -1.3760 |
69
- | 0.5645 | 0.4 | 500 | 0.5639 | -2.0721 | -2.6869 | 0.6987 | 0.6147 | -419.8194 | -351.7883 | -0.6091 | -0.7860 |
70
- | 0.5513 | 0.48 | 600 | 0.5582 | -2.9237 | -3.5389 | 0.7108 | 0.6152 | -505.0223 | -436.9386 | 0.1224 | -0.1054 |
71
- | 0.5571 | 0.56 | 700 | 0.5559 | -2.7971 | -3.5456 | 0.7043 | 0.7485 | -505.6961 | -424.2823 | 0.2980 | 0.0356 |
72
- | 0.5609 | 0.64 | 800 | 0.5469 | -2.4314 | -3.0831 | 0.7108 | 0.6517 | -459.4439 | -387.7092 | 0.1922 | -0.0312 |
73
- | 0.5514 | 0.72 | 900 | 0.5474 | -2.4774 | -3.2082 | 0.6996 | 0.7308 | -471.9533 | -392.3096 | 0.5382 | 0.2860 |
74
- | 0.527 | 0.8 | 1000 | 0.5454 | -2.5040 | -3.2071 | 0.7080 | 0.7031 | -471.8454 | -394.9711 | 0.6372 | 0.3871 |
75
- | 0.5487 | 0.88 | 1100 | 0.5444 | -2.2851 | -2.9963 | 0.7090 | 0.7112 | -450.7599 | -373.0831 | 0.4336 | 0.1858 |
76
- | 0.5483 | 0.96 | 1200 | 0.5440 | -2.2940 | -3.0054 | 0.7090 | 0.7114 | -451.6765 | -373.9785 | 0.3244 | 0.0742 |
77
 
78
 
79
  ### Framework versions
 
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
+ - Loss: 0.0568
21
+ - Rewards/chosen: -1.4677
22
+ - Rewards/rejected: -2.2092
23
+ - Rewards/accuracies: 0.7188
24
+ - Rewards/margins: 0.7416
25
+ - Logps/rejected: -478.2772
26
+ - Logps/chosen: -403.8058
27
+ - Logits/rejected: -2.3614
28
+ - Logits/chosen: -2.3747
29
 
30
  ## Model description
31
 
 
47
  - learning_rate: 5e-07
48
  - train_batch_size: 8
49
  - eval_batch_size: 8
50
+ - seed: 1
51
  - distributed_type: multi-GPU
52
  - num_devices: 8
53
  - gradient_accumulation_steps: 2
 
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 0.0933 | 0.21 | 100 | 0.0944 | -0.6897 | -1.0557 | 0.6719 | 0.3660 | -362.9189 | -326.0099 | -2.7165 | -2.7312 |
66
+ | 0.0726 | 0.42 | 200 | 0.0747 | -0.9446 | -1.5908 | 0.7070 | 0.6462 | -416.4328 | -351.5037 | -2.5344 | -2.5482 |
67
+ | 0.0572 | 0.63 | 300 | 0.0588 | -1.3666 | -2.0492 | 0.7266 | 0.6826 | -462.2752 | -393.6982 | -2.4058 | -2.4191 |
68
+ | 0.0607 | 0.84 | 400 | 0.0568 | -1.4677 | -2.2092 | 0.7188 | 0.7416 | -478.2772 | -403.8058 | -2.3614 | -2.3747 |
 
 
 
 
 
 
 
 
69
 
70
 
71
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.5712926928784438,
4
- "train_runtime": 11573.5451,
5
- "train_samples": 160800,
6
- "train_samples_per_second": 13.894,
7
  "train_steps_per_second": 0.109
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.08938389025223305,
4
+ "train_runtime": 4400.7374,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 13.892,
7
  "train_steps_per_second": 0.109
8
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:18e1cec63bd40f863dc594533ae9ac02d7bcdd4f57a17c1ef5d63193122a0814
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc3027d5e43e370751cf6db6b8ee96e7067a75c87ba2e17eba2339a17c14707a
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:565d4244afeda54e7f62be9e162a16c6892085c081422f02c7a001ecce587eb6
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:678e6cf2fffbce8ec0357f72e5df7a84e73f8bdcd6832528db02fe4b91152f6b
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0debf1533b3a9f2ffea91ddec7f947ba3d1c43476aedcef3273235a227bb4ce5
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd09f3b0019a3ec700f2b7395188a82ac1d309369ccaa613deafc147f80f8da6
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.5712926928784438,
4
- "train_runtime": 11573.5451,
5
- "train_samples": 160800,
6
- "train_samples_per_second": 13.894,
7
  "train_steps_per_second": 0.109
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.08938389025223305,
4
+ "train_runtime": 4400.7374,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 13.892,
7
  "train_steps_per_second": 0.109
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.9996020692399522,
5
  "eval_steps": 100,
6
- "global_step": 1256,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 3.968253968253968e-09,
14
- "logits/chosen": -2.7193620204925537,
15
- "logits/rejected": -2.698728084564209,
16
- "logps/chosen": -182.0961456298828,
17
- "logps/rejected": -172.47128295898438,
18
- "loss": 0.6931,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -23,1959 +23,739 @@
23
  "step": 1
24
  },
25
  {
26
- "epoch": 0.01,
27
- "learning_rate": 3.968253968253968e-08,
28
- "logits/chosen": -2.7041964530944824,
29
- "logits/rejected": -2.6794540882110596,
30
- "logps/chosen": -162.45831298828125,
31
- "logps/rejected": -140.5693359375,
32
- "loss": 0.6931,
33
- "rewards/accuracies": 0.5486111044883728,
34
- "rewards/chosen": 0.00032037965138442814,
35
- "rewards/margins": 0.0004935775068588555,
36
- "rewards/rejected": -0.00017319784092251211,
37
  "step": 10
38
  },
39
  {
40
- "epoch": 0.02,
41
- "learning_rate": 7.936507936507936e-08,
42
- "logits/chosen": -2.7177577018737793,
43
- "logits/rejected": -2.7136425971984863,
44
- "logps/chosen": -134.47242736816406,
45
- "logps/rejected": -143.55604553222656,
46
- "loss": 0.6931,
47
- "rewards/accuracies": 0.550000011920929,
48
- "rewards/chosen": 8.780837379163131e-05,
49
- "rewards/margins": 0.00010721785656642169,
50
- "rewards/rejected": -1.940951551659964e-05,
51
  "step": 20
52
  },
53
  {
54
- "epoch": 0.02,
55
- "learning_rate": 1.1904761904761903e-07,
56
- "logits/chosen": -2.6898293495178223,
57
- "logits/rejected": -2.676154613494873,
58
- "logps/chosen": -140.94692993164062,
59
- "logps/rejected": -136.50369262695312,
60
- "loss": 0.6931,
61
- "rewards/accuracies": 0.4375,
62
- "rewards/chosen": 0.0005466601578518748,
63
- "rewards/margins": -0.00021456097601912916,
64
- "rewards/rejected": 0.0007612211629748344,
65
  "step": 30
66
  },
67
  {
68
- "epoch": 0.03,
69
- "learning_rate": 1.5873015873015872e-07,
70
- "logits/chosen": -2.6958394050598145,
71
- "logits/rejected": -2.686532974243164,
72
- "logps/chosen": -134.98963928222656,
73
- "logps/rejected": -144.46652221679688,
74
- "loss": 0.6928,
75
- "rewards/accuracies": 0.581250011920929,
76
- "rewards/chosen": 0.0015748919686302543,
77
- "rewards/margins": 0.0009769219905138016,
78
- "rewards/rejected": 0.0005979698617011309,
79
  "step": 40
80
  },
81
  {
82
- "epoch": 0.04,
83
- "learning_rate": 1.984126984126984e-07,
84
- "logits/chosen": -2.7042899131774902,
85
- "logits/rejected": -2.6861345767974854,
86
- "logps/chosen": -149.71768188476562,
87
- "logps/rejected": -145.0757293701172,
88
- "loss": 0.6921,
89
- "rewards/accuracies": 0.6000000238418579,
90
- "rewards/chosen": 0.005199921317398548,
91
- "rewards/margins": 0.0022330707870423794,
92
- "rewards/rejected": 0.0029668500646948814,
93
  "step": 50
94
  },
95
  {
96
- "epoch": 0.05,
97
- "learning_rate": 2.3809523809523806e-07,
98
- "logits/chosen": -2.705153703689575,
99
- "logits/rejected": -2.685439348220825,
100
- "logps/chosen": -154.3783416748047,
101
- "logps/rejected": -151.54519653320312,
102
- "loss": 0.6912,
103
- "rewards/accuracies": 0.543749988079071,
104
- "rewards/chosen": 0.00569504126906395,
105
- "rewards/margins": 0.0022000311873853207,
106
- "rewards/rejected": 0.003495010081678629,
107
  "step": 60
108
  },
109
  {
110
- "epoch": 0.06,
111
- "learning_rate": 2.7777777777777776e-07,
112
- "logits/chosen": -2.7017154693603516,
113
- "logits/rejected": -2.6924962997436523,
114
- "logps/chosen": -146.3284149169922,
115
- "logps/rejected": -138.79405212402344,
116
- "loss": 0.6885,
117
- "rewards/accuracies": 0.550000011920929,
118
- "rewards/chosen": 0.010588793084025383,
119
- "rewards/margins": 0.010192448273301125,
120
- "rewards/rejected": 0.00039634370477870107,
121
  "step": 70
122
  },
123
  {
124
- "epoch": 0.06,
125
- "learning_rate": 3.1746031746031743e-07,
126
- "logits/chosen": -2.7155232429504395,
127
- "logits/rejected": -2.696071147918701,
128
- "logps/chosen": -141.80067443847656,
129
- "logps/rejected": -147.0068817138672,
130
- "loss": 0.6867,
131
- "rewards/accuracies": 0.5562499761581421,
132
- "rewards/chosen": 0.0049073463305830956,
133
- "rewards/margins": 0.013599385507404804,
134
- "rewards/rejected": -0.008692039176821709,
135
  "step": 80
136
  },
137
  {
138
- "epoch": 0.07,
139
- "learning_rate": 3.5714285714285716e-07,
140
- "logits/chosen": -2.7175304889678955,
141
- "logits/rejected": -2.7080624103546143,
142
- "logps/chosen": -153.12509155273438,
143
- "logps/rejected": -146.53590393066406,
144
- "loss": 0.6847,
145
- "rewards/accuracies": 0.5874999761581421,
146
- "rewards/chosen": -0.028871387243270874,
147
- "rewards/margins": 0.017175236716866493,
148
- "rewards/rejected": -0.046046625822782516,
149
  "step": 90
150
  },
151
  {
152
- "epoch": 0.08,
153
- "learning_rate": 3.968253968253968e-07,
154
- "logits/chosen": -2.7524733543395996,
155
- "logits/rejected": -2.7452526092529297,
156
- "logps/chosen": -163.88070678710938,
157
- "logps/rejected": -163.61032104492188,
158
- "loss": 0.6789,
159
- "rewards/accuracies": 0.612500011920929,
160
- "rewards/chosen": -0.0662173330783844,
161
- "rewards/margins": 0.02977912127971649,
162
- "rewards/rejected": -0.09599645435810089,
163
  "step": 100
164
  },
165
  {
166
- "epoch": 0.08,
167
- "eval_logits/chosen": -2.7336502075195312,
168
- "eval_logits/rejected": -2.7255024909973145,
169
- "eval_logps/chosen": -155.19271850585938,
170
- "eval_logps/rejected": -165.35523986816406,
171
- "eval_loss": 0.6769910454750061,
172
- "eval_rewards/accuracies": 0.5914179086685181,
173
- "eval_rewards/chosen": -0.10619194805622101,
174
- "eval_rewards/margins": 0.03601696714758873,
175
- "eval_rewards/rejected": -0.14220890402793884,
176
- "eval_runtime": 184.3291,
177
- "eval_samples_per_second": 46.395,
178
- "eval_steps_per_second": 0.727,
179
  "step": 100
180
  },
181
  {
182
- "epoch": 0.09,
183
- "learning_rate": 4.365079365079365e-07,
184
- "logits/chosen": -2.738532543182373,
185
- "logits/rejected": -2.7273170948028564,
186
- "logps/chosen": -164.2928009033203,
187
- "logps/rejected": -160.19398498535156,
188
- "loss": 0.6738,
189
- "rewards/accuracies": 0.5625,
190
- "rewards/chosen": -0.16211798787117004,
191
- "rewards/margins": 0.03163355216383934,
192
- "rewards/rejected": -0.19375154376029968,
193
  "step": 110
194
  },
195
  {
196
- "epoch": 0.1,
197
- "learning_rate": 4.761904761904761e-07,
198
- "logits/chosen": -2.7289297580718994,
199
- "logits/rejected": -2.705962657928467,
200
- "logps/chosen": -196.69662475585938,
201
- "logps/rejected": -197.2833251953125,
202
- "loss": 0.661,
203
- "rewards/accuracies": 0.59375,
204
- "rewards/chosen": -0.2917623221874237,
205
- "rewards/margins": 0.08966299891471863,
206
- "rewards/rejected": -0.38142532110214233,
207
  "step": 120
208
  },
209
  {
210
- "epoch": 0.1,
211
- "learning_rate": 4.999845414634076e-07,
212
- "logits/chosen": -2.658005475997925,
213
- "logits/rejected": -2.6317684650421143,
214
- "logps/chosen": -187.4532928466797,
215
- "logps/rejected": -188.37689208984375,
216
- "loss": 0.6542,
217
- "rewards/accuracies": 0.6312500238418579,
218
- "rewards/chosen": -0.3956056833267212,
219
- "rewards/margins": 0.12092368304729462,
220
- "rewards/rejected": -0.5165294408798218,
221
  "step": 130
222
  },
223
  {
224
- "epoch": 0.11,
225
- "learning_rate": 4.998106548810311e-07,
226
- "logits/chosen": -2.6906683444976807,
227
- "logits/rejected": -2.6913747787475586,
228
- "logps/chosen": -199.67568969726562,
229
- "logps/rejected": -253.02487182617188,
230
- "loss": 0.6171,
231
- "rewards/accuracies": 0.7124999761581421,
232
- "rewards/chosen": -0.4227059781551361,
233
- "rewards/margins": 0.27536457777023315,
234
- "rewards/rejected": -0.6980706453323364,
235
  "step": 140
236
  },
237
  {
238
- "epoch": 0.12,
239
- "learning_rate": 4.994436933879359e-07,
240
- "logits/chosen": -2.6662166118621826,
241
- "logits/rejected": -2.644784927368164,
242
- "logps/chosen": -197.07180786132812,
243
- "logps/rejected": -198.4012908935547,
244
- "loss": 0.6395,
245
- "rewards/accuracies": 0.6187499761581421,
246
- "rewards/chosen": -0.3597154915332794,
247
- "rewards/margins": 0.13716872036457062,
248
- "rewards/rejected": -0.49688419699668884,
249
  "step": 150
250
  },
251
  {
252
- "epoch": 0.13,
253
- "learning_rate": 4.988839406031596e-07,
254
- "logits/chosen": -2.647681474685669,
255
- "logits/rejected": -2.6395888328552246,
256
- "logps/chosen": -182.04420471191406,
257
- "logps/rejected": -206.59780883789062,
258
- "loss": 0.629,
259
- "rewards/accuracies": 0.6499999761581421,
260
- "rewards/chosen": -0.3573477864265442,
261
- "rewards/margins": 0.2222837507724762,
262
- "rewards/rejected": -0.579631507396698,
263
  "step": 160
264
  },
265
  {
266
- "epoch": 0.14,
267
- "learning_rate": 4.981318291512395e-07,
268
- "logits/chosen": -2.619232654571533,
269
- "logits/rejected": -2.598362684249878,
270
- "logps/chosen": -227.0933380126953,
271
- "logps/rejected": -230.9747772216797,
272
- "loss": 0.6242,
273
- "rewards/accuracies": 0.637499988079071,
274
- "rewards/chosen": -0.7434185743331909,
275
- "rewards/margins": 0.21749505400657654,
276
- "rewards/rejected": -0.9609137773513794,
277
  "step": 170
278
  },
279
  {
280
- "epoch": 0.14,
281
- "learning_rate": 4.971879403278432e-07,
282
- "logits/chosen": -2.5654754638671875,
283
- "logits/rejected": -2.5364232063293457,
284
- "logps/chosen": -241.6617431640625,
285
- "logps/rejected": -245.66268920898438,
286
- "loss": 0.6151,
287
- "rewards/accuracies": 0.6625000238418579,
288
- "rewards/chosen": -0.7342535257339478,
289
- "rewards/margins": 0.23685339093208313,
290
- "rewards/rejected": -0.9711068868637085,
291
  "step": 180
292
  },
293
  {
294
- "epoch": 0.15,
295
- "learning_rate": 4.960530036504941e-07,
296
- "logits/chosen": -2.5271048545837402,
297
- "logits/rejected": -2.486818790435791,
298
- "logps/chosen": -235.6089630126953,
299
- "logps/rejected": -251.17758178710938,
300
- "loss": 0.6215,
301
- "rewards/accuracies": 0.699999988079071,
302
- "rewards/chosen": -0.807177722454071,
303
- "rewards/margins": 0.28561535477638245,
304
- "rewards/rejected": -1.0927931070327759,
305
  "step": 190
306
  },
307
  {
308
- "epoch": 0.16,
309
- "learning_rate": 4.947278962947386e-07,
310
- "logits/chosen": -2.4217896461486816,
311
- "logits/rejected": -2.413295269012451,
312
- "logps/chosen": -251.0736083984375,
313
- "logps/rejected": -268.6098937988281,
314
- "loss": 0.6062,
315
- "rewards/accuracies": 0.6625000238418579,
316
- "rewards/chosen": -1.086307406425476,
317
- "rewards/margins": 0.24874301254749298,
318
- "rewards/rejected": -1.3350504636764526,
319
- "step": 200
320
- },
321
- {
322
- "epoch": 0.16,
323
- "eval_logits/chosen": -2.3855514526367188,
324
- "eval_logits/rejected": -2.369593858718872,
325
- "eval_logps/chosen": -246.6970672607422,
326
- "eval_logps/rejected": -289.8621826171875,
327
- "eval_loss": 0.6079375743865967,
328
- "eval_rewards/accuracies": 0.66697758436203,
329
- "eval_rewards/chosen": -1.021235704421997,
330
- "eval_rewards/margins": 0.3660426437854767,
331
- "eval_rewards/rejected": -1.3872781991958618,
332
- "eval_runtime": 184.172,
333
- "eval_samples_per_second": 46.435,
334
- "eval_steps_per_second": 0.728,
335
- "step": 200
336
- },
337
- {
338
- "epoch": 0.17,
339
- "learning_rate": 4.932136424161899e-07,
340
- "logits/chosen": -2.3366785049438477,
341
- "logits/rejected": -2.3228511810302734,
342
- "logps/chosen": -266.292236328125,
343
- "logps/rejected": -300.22894287109375,
344
- "loss": 0.5893,
345
- "rewards/accuracies": 0.5874999761581421,
346
- "rewards/chosen": -1.2130026817321777,
347
- "rewards/margins": 0.3487839996814728,
348
- "rewards/rejected": -1.5617868900299072,
349
- "step": 210
350
- },
351
- {
352
- "epoch": 0.18,
353
- "learning_rate": 4.915114123589732e-07,
354
- "logits/chosen": -2.321228504180908,
355
- "logits/rejected": -2.3033699989318848,
356
- "logps/chosen": -336.34161376953125,
357
- "logps/rejected": -373.39935302734375,
358
- "loss": 0.612,
359
- "rewards/accuracies": 0.6812499761581421,
360
- "rewards/chosen": -1.9529145956039429,
361
- "rewards/margins": 0.2863468527793884,
362
- "rewards/rejected": -2.2392613887786865,
363
- "step": 220
364
- },
365
- {
366
- "epoch": 0.18,
367
- "learning_rate": 4.896225217511849e-07,
368
- "logits/chosen": -2.4310107231140137,
369
- "logits/rejected": -2.422048568725586,
370
- "logps/chosen": -291.1025695800781,
371
- "logps/rejected": -328.18963623046875,
372
- "loss": 0.6079,
373
- "rewards/accuracies": 0.6812499761581421,
374
- "rewards/chosen": -1.4314143657684326,
375
- "rewards/margins": 0.3364001214504242,
376
- "rewards/rejected": -1.7678143978118896,
377
- "step": 230
378
- },
379
- {
380
- "epoch": 0.19,
381
- "learning_rate": 4.875484304880629e-07,
382
- "logits/chosen": -2.3412394523620605,
383
- "logits/rejected": -2.309183120727539,
384
- "logps/chosen": -280.8785705566406,
385
- "logps/rejected": -308.54132080078125,
386
- "loss": 0.613,
387
- "rewards/accuracies": 0.612500011920929,
388
- "rewards/chosen": -1.309309482574463,
389
- "rewards/margins": 0.3731766939163208,
390
- "rewards/rejected": -1.6824861764907837,
391
- "step": 240
392
- },
393
- {
394
- "epoch": 0.2,
395
- "learning_rate": 4.852907416036558e-07,
396
- "logits/chosen": -2.415271282196045,
397
- "logits/rejected": -2.4072234630584717,
398
- "logps/chosen": -243.56332397460938,
399
- "logps/rejected": -298.7532043457031,
400
- "loss": 0.591,
401
- "rewards/accuracies": 0.737500011920929,
402
- "rewards/chosen": -0.967076301574707,
403
- "rewards/margins": 0.4581146240234375,
404
- "rewards/rejected": -1.4251911640167236,
405
- "step": 250
406
- },
407
- {
408
- "epoch": 0.21,
409
- "learning_rate": 4.828512000318616e-07,
410
- "logits/chosen": -2.3924427032470703,
411
- "logits/rejected": -2.3613152503967285,
412
- "logps/chosen": -266.86572265625,
413
- "logps/rejected": -304.2983093261719,
414
- "loss": 0.5986,
415
- "rewards/accuracies": 0.668749988079071,
416
- "rewards/chosen": -1.2214807271957397,
417
- "rewards/margins": 0.4553411602973938,
418
- "rewards/rejected": -1.6768219470977783,
419
- "step": 260
420
- },
421
- {
422
- "epoch": 0.21,
423
- "learning_rate": 4.802316912577946e-07,
424
- "logits/chosen": -2.4108529090881348,
425
- "logits/rejected": -2.3902478218078613,
426
- "logps/chosen": -252.7959442138672,
427
- "logps/rejected": -295.266357421875,
428
- "loss": 0.5917,
429
- "rewards/accuracies": 0.7250000238418579,
430
- "rewards/chosen": -1.0824626684188843,
431
- "rewards/margins": 0.39643940329551697,
432
- "rewards/rejected": -1.4789022207260132,
433
- "step": 270
434
- },
435
- {
436
- "epoch": 0.22,
437
- "learning_rate": 4.774342398605221e-07,
438
- "logits/chosen": -2.3505263328552246,
439
- "logits/rejected": -2.2942967414855957,
440
- "logps/chosen": -279.871337890625,
441
- "logps/rejected": -300.4220886230469,
442
- "loss": 0.5979,
443
- "rewards/accuracies": 0.6812499761581421,
444
- "rewards/chosen": -1.1787078380584717,
445
- "rewards/margins": 0.430286169052124,
446
- "rewards/rejected": -1.6089938879013062,
447
- "step": 280
448
- },
449
- {
450
- "epoch": 0.23,
451
- "learning_rate": 4.744610079482978e-07,
452
- "logits/chosen": -2.3269264698028564,
453
- "logits/rejected": -2.2910802364349365,
454
- "logps/chosen": -255.27706909179688,
455
- "logps/rejected": -281.60137939453125,
456
- "loss": 0.5853,
457
- "rewards/accuracies": 0.6875,
458
- "rewards/chosen": -1.1753785610198975,
459
- "rewards/margins": 0.3495523929595947,
460
- "rewards/rejected": -1.5249310731887817,
461
- "step": 290
462
- },
463
- {
464
- "epoch": 0.24,
465
- "learning_rate": 4.713142934875005e-07,
466
- "logits/chosen": -2.2868428230285645,
467
- "logits/rejected": -2.2631592750549316,
468
- "logps/chosen": -284.2200012207031,
469
- "logps/rejected": -322.45269775390625,
470
- "loss": 0.5965,
471
- "rewards/accuracies": 0.675000011920929,
472
- "rewards/chosen": -1.336501955986023,
473
- "rewards/margins": 0.3968800902366638,
474
- "rewards/rejected": -1.733382225036621,
475
- "step": 300
476
- },
477
- {
478
- "epoch": 0.24,
479
- "eval_logits/chosen": -2.265592098236084,
480
- "eval_logits/rejected": -2.244987964630127,
481
- "eval_logps/chosen": -282.3620910644531,
482
- "eval_logps/rejected": -331.2099609375,
483
- "eval_loss": 0.5907339453697205,
484
- "eval_rewards/accuracies": 0.6623134613037109,
485
- "eval_rewards/chosen": -1.3778856992721558,
486
- "eval_rewards/margins": 0.42287060618400574,
487
- "eval_rewards/rejected": -1.8007562160491943,
488
- "eval_runtime": 184.1739,
489
- "eval_samples_per_second": 46.434,
490
- "eval_steps_per_second": 0.728,
491
- "step": 300
492
- },
493
- {
494
- "epoch": 0.25,
495
- "learning_rate": 4.679965285265706e-07,
496
- "logits/chosen": -2.2354235649108887,
497
- "logits/rejected": -2.23685884475708,
498
- "logps/chosen": -277.09283447265625,
499
- "logps/rejected": -347.7145080566406,
500
- "loss": 0.5612,
501
- "rewards/accuracies": 0.6875,
502
- "rewards/chosen": -1.3514426946640015,
503
- "rewards/margins": 0.4907970428466797,
504
- "rewards/rejected": -1.8422397375106812,
505
- "step": 310
506
- },
507
- {
508
- "epoch": 0.25,
509
- "learning_rate": 4.64510277316316e-07,
510
- "logits/chosen": -2.2262344360351562,
511
- "logits/rejected": -2.226029634475708,
512
- "logps/chosen": -271.74212646484375,
513
- "logps/rejected": -332.5010986328125,
514
- "loss": 0.5903,
515
- "rewards/accuracies": 0.612500011920929,
516
- "rewards/chosen": -1.3337775468826294,
517
- "rewards/margins": 0.39512914419174194,
518
- "rewards/rejected": -1.7289068698883057,
519
- "step": 320
520
- },
521
- {
522
- "epoch": 0.26,
523
- "learning_rate": 4.6085823432804137e-07,
524
- "logits/chosen": -2.2451891899108887,
525
- "logits/rejected": -2.2502384185791016,
526
- "logps/chosen": -250.6347198486328,
527
- "logps/rejected": -333.8939208984375,
528
- "loss": 0.5722,
529
- "rewards/accuracies": 0.762499988079071,
530
- "rewards/chosen": -1.1326004266738892,
531
- "rewards/margins": 0.5066065192222595,
532
- "rewards/rejected": -1.639206886291504,
533
- "step": 330
534
- },
535
- {
536
- "epoch": 0.27,
537
- "learning_rate": 4.570432221710314e-07,
538
- "logits/chosen": -2.0656931400299072,
539
- "logits/rejected": -2.0213730335235596,
540
- "logps/chosen": -318.232177734375,
541
- "logps/rejected": -369.13311767578125,
542
- "loss": 0.5766,
543
- "rewards/accuracies": 0.7250000238418579,
544
- "rewards/chosen": -1.584176778793335,
545
- "rewards/margins": 0.5901076197624207,
546
- "rewards/rejected": -2.1742844581604004,
547
- "step": 340
548
- },
549
- {
550
- "epoch": 0.28,
551
- "learning_rate": 4.5306818941099866e-07,
552
- "logits/chosen": -1.9084612131118774,
553
- "logits/rejected": -1.8514792919158936,
554
- "logps/chosen": -316.9821472167969,
555
- "logps/rejected": -352.9412841796875,
556
- "loss": 0.5825,
557
- "rewards/accuracies": 0.65625,
558
- "rewards/chosen": -1.5627154111862183,
559
- "rewards/margins": 0.5152220726013184,
560
- "rewards/rejected": -2.077937364578247,
561
- "step": 350
562
- },
563
- {
564
- "epoch": 0.29,
565
- "learning_rate": 4.4893620829118124e-07,
566
- "logits/chosen": -1.8860156536102295,
567
- "logits/rejected": -1.8301204442977905,
568
- "logps/chosen": -309.8200378417969,
569
- "logps/rejected": -362.0408935546875,
570
- "loss": 0.5755,
571
- "rewards/accuracies": 0.6812499761581421,
572
- "rewards/chosen": -1.5869390964508057,
573
- "rewards/margins": 0.49348369240760803,
574
- "rewards/rejected": -2.080422878265381,
575
- "step": 360
576
- },
577
- {
578
- "epoch": 0.29,
579
- "learning_rate": 4.4465047235785185e-07,
580
- "logits/chosen": -1.6610889434814453,
581
- "logits/rejected": -1.585129737854004,
582
- "logps/chosen": -321.8608703613281,
583
- "logps/rejected": -380.31036376953125,
584
- "loss": 0.5697,
585
- "rewards/accuracies": 0.7250000238418579,
586
- "rewards/chosen": -1.774713158607483,
587
- "rewards/margins": 0.6593302488327026,
588
- "rewards/rejected": -2.4340434074401855,
589
- "step": 370
590
- },
591
- {
592
- "epoch": 0.3,
593
- "learning_rate": 4.40214293992074e-07,
594
- "logits/chosen": -1.385825753211975,
595
- "logits/rejected": -1.31913161277771,
596
- "logps/chosen": -377.07269287109375,
597
- "logps/rejected": -459.5557556152344,
598
- "loss": 0.5818,
599
- "rewards/accuracies": 0.706250011920929,
600
- "rewards/chosen": -2.1890993118286133,
601
- "rewards/margins": 0.7521292567253113,
602
- "rewards/rejected": -2.9412286281585693,
603
- "step": 380
604
- },
605
- {
606
- "epoch": 0.31,
607
- "learning_rate": 4.3563110184961234e-07,
608
- "logits/chosen": -1.5089499950408936,
609
- "logits/rejected": -1.4075387716293335,
610
- "logps/chosen": -338.3626708984375,
611
- "logps/rejected": -396.67578125,
612
- "loss": 0.5584,
613
- "rewards/accuracies": 0.737500011920929,
614
- "rewards/chosen": -1.9042552709579468,
615
- "rewards/margins": 0.5932050347328186,
616
- "rewards/rejected": -2.49746036529541,
617
- "step": 390
618
- },
619
- {
620
- "epoch": 0.32,
621
- "learning_rate": 4.3090443821097566e-07,
622
- "logits/chosen": -1.2587625980377197,
623
- "logits/rejected": -1.2017955780029297,
624
- "logps/chosen": -309.43377685546875,
625
- "logps/rejected": -372.00531005859375,
626
- "loss": 0.5729,
627
- "rewards/accuracies": 0.6812499761581421,
628
- "rewards/chosen": -1.9031288623809814,
629
- "rewards/margins": 0.5602144598960876,
630
- "rewards/rejected": -2.4633431434631348,
631
- "step": 400
632
- },
633
- {
634
- "epoch": 0.32,
635
- "eval_logits/chosen": -1.3760210275650024,
636
- "eval_logits/rejected": -1.2920024394989014,
637
- "eval_logps/chosen": -312.20635986328125,
638
- "eval_logps/rejected": -375.1720275878906,
639
- "eval_loss": 0.5711147785186768,
640
- "eval_rewards/accuracies": 0.6828358173370361,
641
- "eval_rewards/chosen": -1.676328182220459,
642
- "eval_rewards/margins": 0.5640482306480408,
643
- "eval_rewards/rejected": -2.2403764724731445,
644
- "eval_runtime": 184.465,
645
- "eval_samples_per_second": 46.361,
646
- "eval_steps_per_second": 0.726,
647
- "step": 400
648
- },
649
- {
650
- "epoch": 0.33,
651
- "learning_rate": 4.2603795624364195e-07,
652
- "logits/chosen": -1.2894772291183472,
653
- "logits/rejected": -1.23129141330719,
654
- "logps/chosen": -299.457275390625,
655
- "logps/rejected": -370.8555908203125,
656
- "loss": 0.5666,
657
- "rewards/accuracies": 0.6875,
658
- "rewards/chosen": -1.6054102182388306,
659
- "rewards/margins": 0.5984233021736145,
660
- "rewards/rejected": -2.203833818435669,
661
- "step": 410
662
- },
663
- {
664
- "epoch": 0.33,
665
- "learning_rate": 4.210354171785795e-07,
666
- "logits/chosen": -1.022984266281128,
667
- "logits/rejected": -0.9285897016525269,
668
- "logps/chosen": -324.4284973144531,
669
- "logps/rejected": -385.0074157714844,
670
- "loss": 0.5596,
671
- "rewards/accuracies": 0.6875,
672
- "rewards/chosen": -1.921677589416504,
673
- "rewards/margins": 0.5404387712478638,
674
- "rewards/rejected": -2.4621164798736572,
675
- "step": 420
676
- },
677
- {
678
- "epoch": 0.34,
679
- "learning_rate": 4.15900687403248e-07,
680
- "logits/chosen": -0.8059805631637573,
681
- "logits/rejected": -0.7196700572967529,
682
- "logps/chosen": -353.788330078125,
683
- "logps/rejected": -411.4853515625,
684
- "loss": 0.5865,
685
- "rewards/accuracies": 0.643750011920929,
686
- "rewards/chosen": -2.1321234703063965,
687
- "rewards/margins": 0.463266521692276,
688
- "rewards/rejected": -2.5953898429870605,
689
- "step": 430
690
- },
691
- {
692
- "epoch": 0.35,
693
- "learning_rate": 4.1063773547332584e-07,
694
- "logits/chosen": -0.9645301699638367,
695
- "logits/rejected": -0.7601315975189209,
696
- "logps/chosen": -346.8272705078125,
697
- "logps/rejected": -392.2935791015625,
698
- "loss": 0.5591,
699
- "rewards/accuracies": 0.762499988079071,
700
- "rewards/chosen": -1.9153356552124023,
701
- "rewards/margins": 0.5891679525375366,
702
- "rewards/rejected": -2.5045037269592285,
703
- "step": 440
704
- },
705
- {
706
- "epoch": 0.36,
707
- "learning_rate": 4.0525062904547276e-07,
708
- "logits/chosen": -0.608537495136261,
709
- "logits/rejected": -0.47767123579978943,
710
- "logps/chosen": -341.55364990234375,
711
- "logps/rejected": -434.1073303222656,
712
- "loss": 0.5687,
713
- "rewards/accuracies": 0.737500011920929,
714
- "rewards/chosen": -2.105318069458008,
715
- "rewards/margins": 0.6994394659996033,
716
- "rewards/rejected": -2.8047571182250977,
717
- "step": 450
718
- },
719
- {
720
- "epoch": 0.37,
721
- "learning_rate": 3.997435317334988e-07,
722
- "logits/chosen": -0.6356207132339478,
723
- "logits/rejected": -0.25634175539016724,
724
- "logps/chosen": -384.43780517578125,
725
- "logps/rejected": -419.24176025390625,
726
- "loss": 0.5608,
727
- "rewards/accuracies": 0.731249988079071,
728
- "rewards/chosen": -2.2970900535583496,
729
- "rewards/margins": 0.6535100340843201,
730
- "rewards/rejected": -2.9506001472473145,
731
- "step": 460
732
- },
733
- {
734
- "epoch": 0.37,
735
- "learning_rate": 3.941206998903701e-07,
736
- "logits/chosen": -1.0318920612335205,
737
- "logits/rejected": -0.7451022267341614,
738
- "logps/chosen": -338.9430236816406,
739
- "logps/rejected": -384.64111328125,
740
- "loss": 0.5678,
741
- "rewards/accuracies": 0.737500011920929,
742
- "rewards/chosen": -1.9643396139144897,
743
- "rewards/margins": 0.5402536392211914,
744
- "rewards/rejected": -2.5045928955078125,
745
- "step": 470
746
- },
747
- {
748
- "epoch": 0.38,
749
- "learning_rate": 3.8838647931853684e-07,
750
- "logits/chosen": -0.6847028732299805,
751
- "logits/rejected": -0.5548251867294312,
752
- "logps/chosen": -339.61456298828125,
753
- "logps/rejected": -435.32061767578125,
754
- "loss": 0.5814,
755
- "rewards/accuracies": 0.7250000238418579,
756
- "rewards/chosen": -2.1349122524261475,
757
- "rewards/margins": 0.7573872804641724,
758
- "rewards/rejected": -2.8923001289367676,
759
- "step": 480
760
- },
761
- {
762
- "epoch": 0.39,
763
- "learning_rate": 3.825453019111281e-07,
764
- "logits/chosen": -0.5378957986831665,
765
- "logits/rejected": -0.28533270955085754,
766
- "logps/chosen": -363.78570556640625,
767
- "logps/rejected": -430.11749267578125,
768
- "loss": 0.5327,
769
  "rewards/accuracies": 0.699999988079071,
770
- "rewards/chosen": -2.134934425354004,
771
- "rewards/margins": 0.6089809536933899,
772
- "rewards/rejected": -2.743915319442749,
773
- "step": 490
774
- },
775
- {
776
- "epoch": 0.4,
777
- "learning_rate": 3.7660168222660824e-07,
778
- "logits/chosen": -0.6318235397338867,
779
- "logits/rejected": -0.5071814656257629,
780
- "logps/chosen": -350.5252380371094,
781
- "logps/rejected": -421.93353271484375,
782
- "loss": 0.5645,
783
- "rewards/accuracies": 0.6812499761581421,
784
- "rewards/chosen": -2.062009572982788,
785
- "rewards/margins": 0.5333147048950195,
786
- "rewards/rejected": -2.5953242778778076,
787
- "step": 500
788
- },
789
- {
790
- "epoch": 0.4,
791
- "eval_logits/chosen": -0.7860146760940552,
792
- "eval_logits/rejected": -0.6090859770774841,
793
- "eval_logps/chosen": -351.7882995605469,
794
- "eval_logps/rejected": -419.81939697265625,
795
- "eval_loss": 0.5639454126358032,
796
- "eval_rewards/accuracies": 0.6986940503120422,
797
- "eval_rewards/chosen": -2.0721471309661865,
798
- "eval_rewards/margins": 0.6147031188011169,
799
- "eval_rewards/rejected": -2.6868505477905273,
800
- "eval_runtime": 184.5772,
801
- "eval_samples_per_second": 46.333,
802
- "eval_steps_per_second": 0.726,
803
- "step": 500
804
- },
805
- {
806
- "epoch": 0.41,
807
- "learning_rate": 3.705602139995416e-07,
808
- "logits/chosen": -0.7258490920066833,
809
- "logits/rejected": -0.4828409254550934,
810
- "logps/chosen": -388.1371154785156,
811
- "logps/rejected": -422.11181640625,
812
- "loss": 0.574,
813
- "rewards/accuracies": 0.637499988079071,
814
- "rewards/chosen": -2.290266513824463,
815
- "rewards/margins": 0.4104091227054596,
816
- "rewards/rejected": -2.7006754875183105,
817
- "step": 510
818
- },
819
- {
820
- "epoch": 0.41,
821
- "learning_rate": 3.6442556659016475e-07,
822
- "logits/chosen": -0.5335447192192078,
823
- "logits/rejected": -0.33706527948379517,
824
- "logps/chosen": -378.86492919921875,
825
- "logps/rejected": -429.67724609375,
826
- "loss": 0.5608,
827
- "rewards/accuracies": 0.737500011920929,
828
- "rewards/chosen": -2.236337423324585,
829
- "rewards/margins": 0.556148886680603,
830
- "rewards/rejected": -2.7924864292144775,
831
- "step": 520
832
  },
833
  {
834
  "epoch": 0.42,
835
- "learning_rate": 3.582024813755076e-07,
836
- "logits/chosen": -0.39548322558403015,
837
- "logits/rejected": -0.10662730038166046,
838
- "logps/chosen": -368.8847961425781,
839
- "logps/rejected": -473.3500061035156,
840
- "loss": 0.5485,
841
- "rewards/accuracies": 0.731249988079071,
842
- "rewards/chosen": -2.3263449668884277,
843
- "rewards/margins": 0.8236624598503113,
844
- "rewards/rejected": -3.150007724761963,
845
- "step": 530
846
- },
847
- {
848
- "epoch": 0.43,
849
- "learning_rate": 3.5189576808485404e-07,
850
- "logits/chosen": 0.15742243826389313,
851
- "logits/rejected": 0.31491726636886597,
852
- "logps/chosen": -394.34930419921875,
853
- "logps/rejected": -492.82232666015625,
854
- "loss": 0.5478,
855
- "rewards/accuracies": 0.7250000238418579,
856
- "rewards/chosen": -2.6109700202941895,
857
- "rewards/margins": 0.8250136375427246,
858
- "rewards/rejected": -3.435983657836914,
859
- "step": 540
860
  },
861
  {
862
  "epoch": 0.44,
863
- "learning_rate": 3.4551030108237433e-07,
864
- "logits/chosen": -0.2550584375858307,
865
- "logits/rejected": -0.06936412304639816,
866
- "logps/chosen": -406.5508728027344,
867
- "logps/rejected": -448.47576904296875,
868
- "loss": 0.5562,
869
- "rewards/accuracies": 0.6312500238418579,
870
- "rewards/chosen": -2.5152666568756104,
871
- "rewards/margins": 0.4819938540458679,
872
- "rewards/rejected": -2.997260332107544,
873
- "step": 550
874
- },
875
- {
876
- "epoch": 0.45,
877
- "learning_rate": 3.390510155998023e-07,
878
- "logits/chosen": -0.5292027592658997,
879
- "logits/rejected": -0.2619571387767792,
880
- "logps/chosen": -371.6798095703125,
881
- "logps/rejected": -420.7915954589844,
882
- "loss": 0.5492,
883
- "rewards/accuracies": 0.7250000238418579,
884
- "rewards/chosen": -2.1147050857543945,
885
- "rewards/margins": 0.6524336338043213,
886
- "rewards/rejected": -2.7671384811401367,
887
- "step": 560
888
- },
889
- {
890
- "epoch": 0.45,
891
- "learning_rate": 3.325229039220684e-07,
892
- "logits/chosen": -0.5881962776184082,
893
- "logits/rejected": -0.4658876061439514,
894
- "logps/chosen": -343.7039794921875,
895
- "logps/rejected": -406.14178466796875,
896
- "loss": 0.57,
897
  "rewards/accuracies": 0.6875,
898
- "rewards/chosen": -2.0860273838043213,
899
- "rewards/margins": 0.498068630695343,
900
- "rewards/rejected": -2.5840957164764404,
901
- "step": 570
902
  },
903
  {
904
  "epoch": 0.46,
905
- "learning_rate": 3.2593101152883795e-07,
906
- "logits/chosen": -0.6565806269645691,
907
- "logits/rejected": -0.2549567222595215,
908
- "logps/chosen": -374.8047180175781,
909
- "logps/rejected": -430.33221435546875,
910
- "loss": 0.5512,
911
- "rewards/accuracies": 0.7437499761581421,
912
- "rewards/chosen": -2.2211391925811768,
913
- "rewards/margins": 0.6813799142837524,
914
- "rewards/rejected": -2.9025187492370605,
915
- "step": 580
916
- },
917
- {
918
- "epoch": 0.47,
919
- "learning_rate": 3.192804331949349e-07,
920
- "logits/chosen": -0.07184700667858124,
921
- "logits/rejected": 0.1699156016111374,
922
- "logps/chosen": -422.27081298828125,
923
- "logps/rejected": -490.69134521484375,
924
- "loss": 0.535,
925
- "rewards/accuracies": 0.731249988079071,
926
- "rewards/chosen": -2.730973720550537,
927
- "rewards/margins": 0.7726518511772156,
928
- "rewards/rejected": -3.5036251544952393,
929
- "step": 590
930
- },
931
- {
932
- "epoch": 0.48,
933
- "learning_rate": 3.125763090526674e-07,
934
- "logits/chosen": -0.029465889558196068,
935
- "logits/rejected": 0.15842057764530182,
936
- "logps/chosen": -417.373046875,
937
- "logps/rejected": -478.73291015625,
938
- "loss": 0.5513,
939
- "rewards/accuracies": 0.675000011920929,
940
- "rewards/chosen": -2.8006317615509033,
941
- "rewards/margins": 0.6451797485351562,
942
- "rewards/rejected": -3.4458115100860596,
943
- "step": 600
944
  },
945
  {
946
  "epoch": 0.48,
947
- "eval_logits/chosen": -0.10542195290327072,
948
- "eval_logits/rejected": 0.12242482602596283,
949
- "eval_logps/chosen": -436.9386291503906,
950
- "eval_logps/rejected": -505.02227783203125,
951
- "eval_loss": 0.5582411885261536,
952
- "eval_rewards/accuracies": 0.7108209133148193,
953
- "eval_rewards/chosen": -2.9236514568328857,
954
- "eval_rewards/margins": 0.6152271032333374,
955
- "eval_rewards/rejected": -3.5388784408569336,
956
- "eval_runtime": 184.6013,
957
- "eval_samples_per_second": 46.327,
958
- "eval_steps_per_second": 0.726,
959
- "step": 600
960
- },
961
- {
962
- "epoch": 0.49,
963
- "learning_rate": 3.0582382061909623e-07,
964
- "logits/chosen": -0.2445104569196701,
965
- "logits/rejected": -0.018268002197146416,
966
- "logps/chosen": -441.7857971191406,
967
- "logps/rejected": -502.60791015625,
968
- "loss": 0.5594,
969
- "rewards/accuracies": 0.6875,
970
- "rewards/chosen": -2.858261823654175,
971
- "rewards/margins": 0.5510683655738831,
972
- "rewards/rejected": -3.409330368041992,
973
- "step": 610
974
- },
975
- {
976
- "epoch": 0.49,
977
- "learning_rate": 2.9902818679131775e-07,
978
- "logits/chosen": -0.4190225601196289,
979
- "logits/rejected": -0.22823679447174072,
980
- "logps/chosen": -399.03924560546875,
981
- "logps/rejected": -498.6724548339844,
982
- "loss": 0.5499,
983
- "rewards/accuracies": 0.71875,
984
- "rewards/chosen": -2.650449275970459,
985
- "rewards/margins": 0.7673205137252808,
986
- "rewards/rejected": -3.4177703857421875,
987
- "step": 620
988
  },
989
  {
990
  "epoch": 0.5,
991
- "learning_rate": 2.921946598128571e-07,
992
- "logits/chosen": -0.43653860688209534,
993
- "logits/rejected": -0.20837187767028809,
994
- "logps/chosen": -402.82781982421875,
995
- "logps/rejected": -485.4117736816406,
996
- "loss": 0.5739,
997
- "rewards/accuracies": 0.7124999761581421,
998
- "rewards/chosen": -2.573000192642212,
999
- "rewards/margins": 0.7478531002998352,
1000
- "rewards/rejected": -3.3208529949188232,
1001
- "step": 630
1002
- },
1003
- {
1004
- "epoch": 0.51,
1005
- "learning_rate": 2.8532852121428733e-07,
1006
- "logits/chosen": -0.43430274724960327,
1007
- "logits/rejected": -0.13240045309066772,
1008
- "logps/chosen": -397.2491149902344,
1009
- "logps/rejected": -442.12384033203125,
1010
- "loss": 0.5462,
1011
- "rewards/accuracies": 0.7250000238418579,
1012
- "rewards/chosen": -2.4125733375549316,
1013
- "rewards/margins": 0.5821372866630554,
1014
- "rewards/rejected": -2.9947104454040527,
1015
- "step": 640
1016
  },
1017
  {
1018
  "epoch": 0.52,
1019
- "learning_rate": 2.7843507773121414e-07,
1020
- "logits/chosen": -0.4247920513153076,
1021
- "logits/rejected": -0.21372787654399872,
1022
- "logps/chosen": -389.4237976074219,
1023
- "logps/rejected": -458.3169860839844,
1024
- "loss": 0.5373,
1025
- "rewards/accuracies": 0.6812499761581421,
1026
- "rewards/chosen": -2.444688320159912,
1027
- "rewards/margins": 0.7236617207527161,
1028
- "rewards/rejected": -3.1683506965637207,
1029
- "step": 650
1030
- },
1031
- {
1032
- "epoch": 0.53,
1033
- "learning_rate": 2.715196572027789e-07,
1034
- "logits/chosen": -0.6697942614555359,
1035
- "logits/rejected": -0.4933086931705475,
1036
- "logps/chosen": -387.529296875,
1037
- "logps/rejected": -472.73944091796875,
1038
- "loss": 0.5685,
1039
- "rewards/accuracies": 0.7124999761581421,
1040
- "rewards/chosen": -2.3496451377868652,
1041
- "rewards/margins": 0.7728831171989441,
1042
- "rewards/rejected": -3.122528314590454,
1043
- "step": 660
1044
- },
1045
- {
1046
- "epoch": 0.53,
1047
- "learning_rate": 2.645876044538521e-07,
1048
- "logits/chosen": -1.0338900089263916,
1049
- "logits/rejected": -0.8813627362251282,
1050
- "logps/chosen": -372.53118896484375,
1051
- "logps/rejected": -426.54241943359375,
1052
- "loss": 0.5725,
1053
- "rewards/accuracies": 0.675000011920929,
1054
- "rewards/chosen": -2.201908588409424,
1055
- "rewards/margins": 0.5010865926742554,
1056
- "rewards/rejected": -2.7029950618743896,
1057
- "step": 670
1058
  },
1059
  {
1060
  "epoch": 0.54,
1061
- "learning_rate": 2.5764427716409815e-07,
1062
- "logits/chosen": -0.9278701543807983,
1063
- "logits/rejected": -0.7282145023345947,
1064
- "logps/chosen": -347.2828674316406,
1065
- "logps/rejected": -416.9349060058594,
1066
- "loss": 0.5479,
1067
- "rewards/accuracies": 0.7124999761581421,
1068
- "rewards/chosen": -2.0276436805725098,
1069
- "rewards/margins": 0.743033230304718,
1070
- "rewards/rejected": -2.770677089691162,
1071
- "step": 680
1072
- },
1073
- {
1074
- "epoch": 0.55,
1075
- "learning_rate": 2.5069504172710494e-07,
1076
- "logits/chosen": -0.5008482336997986,
1077
- "logits/rejected": -0.34875133633613586,
1078
- "logps/chosen": -373.7621154785156,
1079
- "logps/rejected": -485.12884521484375,
1080
- "loss": 0.5217,
1081
- "rewards/accuracies": 0.762499988079071,
1082
- "rewards/chosen": -2.347053050994873,
1083
- "rewards/margins": 0.9024646878242493,
1084
- "rewards/rejected": -3.2495174407958984,
1085
- "step": 690
1086
- },
1087
- {
1088
- "epoch": 0.56,
1089
- "learning_rate": 2.4374526910277886e-07,
1090
- "logits/chosen": 0.06850005686283112,
1091
- "logits/rejected": 0.41385045647621155,
1092
- "logps/chosen": -411.46246337890625,
1093
- "logps/rejected": -476.6162109375,
1094
- "loss": 0.5571,
1095
- "rewards/accuracies": 0.706250011920929,
1096
- "rewards/chosen": -2.6757898330688477,
1097
- "rewards/margins": 0.8085702657699585,
1098
- "rewards/rejected": -3.4843602180480957,
1099
- "step": 700
1100
  },
1101
  {
1102
  "epoch": 0.56,
1103
- "eval_logits/chosen": 0.035554468631744385,
1104
- "eval_logits/rejected": 0.2980235815048218,
1105
- "eval_logps/chosen": -424.2823486328125,
1106
- "eval_logps/rejected": -505.6960754394531,
1107
- "eval_loss": 0.5558871626853943,
1108
- "eval_rewards/accuracies": 0.704291045665741,
1109
- "eval_rewards/chosen": -2.797088146209717,
1110
- "eval_rewards/margins": 0.748529314994812,
1111
- "eval_rewards/rejected": -3.5456173419952393,
1112
- "eval_runtime": 184.3728,
1113
- "eval_samples_per_second": 46.384,
1114
- "eval_steps_per_second": 0.727,
1115
- "step": 700
1116
- },
1117
- {
1118
- "epoch": 0.57,
1119
- "learning_rate": 2.368003306662104e-07,
1120
- "logits/chosen": 0.07857178151607513,
1121
- "logits/rejected": 0.3302653729915619,
1122
- "logps/chosen": -413.8836975097656,
1123
- "logps/rejected": -535.0875244140625,
1124
- "loss": 0.5287,
1125
- "rewards/accuracies": 0.762499988079071,
1126
- "rewards/chosen": -2.7193782329559326,
1127
- "rewards/margins": 1.0089346170425415,
1128
- "rewards/rejected": -3.7283127307891846,
1129
- "step": 710
1130
- },
1131
- {
1132
- "epoch": 0.57,
1133
- "learning_rate": 2.2986559405621886e-07,
1134
- "logits/chosen": 0.2789291739463806,
1135
- "logits/rejected": 0.4242584705352783,
1136
- "logps/chosen": -422.7801818847656,
1137
- "logps/rejected": -522.7840576171875,
1138
- "loss": 0.5551,
1139
- "rewards/accuracies": 0.731249988079071,
1140
- "rewards/chosen": -2.925621509552002,
1141
- "rewards/margins": 0.8043605089187622,
1142
- "rewards/rejected": -3.729982376098633,
1143
- "step": 720
1144
- },
1145
- {
1146
- "epoch": 0.58,
1147
- "learning_rate": 2.2294641902678443e-07,
1148
- "logits/chosen": -0.19327735900878906,
1149
- "logits/rejected": 0.043265581130981445,
1150
- "logps/chosen": -363.1488342285156,
1151
- "logps/rejected": -470.94970703125,
1152
- "loss": 0.5284,
1153
- "rewards/accuracies": 0.78125,
1154
- "rewards/chosen": -2.39530873298645,
1155
- "rewards/margins": 0.908363938331604,
1156
- "rewards/rejected": -3.3036727905273438,
1157
- "step": 730
1158
  },
1159
  {
1160
  "epoch": 0.59,
1161
- "learning_rate": 2.160481533045751e-07,
1162
- "logits/chosen": -0.37412697076797485,
1163
- "logits/rejected": -0.17320053279399872,
1164
- "logps/chosen": -390.2896423339844,
1165
- "logps/rejected": -428.08099365234375,
1166
- "loss": 0.5572,
1167
- "rewards/accuracies": 0.675000011920929,
1168
- "rewards/chosen": -2.3005330562591553,
1169
- "rewards/margins": 0.48462891578674316,
1170
- "rewards/rejected": -2.7851624488830566,
1171
- "step": 740
1172
- },
1173
- {
1174
- "epoch": 0.6,
1175
- "learning_rate": 2.0917612845576882e-07,
1176
- "logits/chosen": -0.26352375745773315,
1177
- "logits/rejected": -0.0010178961092606187,
1178
- "logps/chosen": -373.3875427246094,
1179
- "logps/rejected": -440.09442138671875,
1180
- "loss": 0.5534,
1181
- "rewards/accuracies": 0.6875,
1182
- "rewards/chosen": -2.3315823078155518,
1183
- "rewards/margins": 0.6843063235282898,
1184
- "rewards/rejected": -3.0158886909484863,
1185
- "step": 750
1186
- },
1187
- {
1188
- "epoch": 0.6,
1189
- "learning_rate": 2.0233565576536564e-07,
1190
- "logits/chosen": -0.3354080021381378,
1191
- "logits/rejected": -0.006600166670978069,
1192
- "logps/chosen": -360.56463623046875,
1193
- "logps/rejected": -440.66961669921875,
1194
- "loss": 0.5328,
1195
- "rewards/accuracies": 0.7437499761581421,
1196
- "rewards/chosen": -2.1626803874969482,
1197
- "rewards/margins": 0.8473829030990601,
1198
- "rewards/rejected": -3.010063409805298,
1199
- "step": 760
1200
  },
1201
  {
1202
  "epoch": 0.61,
1203
- "learning_rate": 1.9553202213217537e-07,
1204
- "logits/chosen": -0.021420275792479515,
1205
- "logits/rejected": 0.19946305453777313,
1206
- "logps/chosen": -389.1043395996094,
1207
- "logps/rejected": -448.04998779296875,
1208
- "loss": 0.5523,
1209
- "rewards/accuracies": 0.731249988079071,
1210
- "rewards/chosen": -2.391838788986206,
1211
- "rewards/margins": 0.6678962707519531,
1212
- "rewards/rejected": -3.059735059738159,
1213
- "step": 770
1214
- },
1215
- {
1216
- "epoch": 0.62,
1217
- "learning_rate": 1.887704859826528e-07,
1218
- "logits/chosen": -0.15253478288650513,
1219
- "logits/rejected": -0.00011998042464256287,
1220
- "logps/chosen": -394.9501953125,
1221
- "logps/rejected": -462.32843017578125,
1222
- "loss": 0.5443,
1223
  "rewards/accuracies": 0.699999988079071,
1224
- "rewards/chosen": -2.476644992828369,
1225
- "rewards/margins": 0.566824734210968,
1226
- "rewards/rejected": -3.0434699058532715,
1227
- "step": 780
1228
  },
1229
  {
1230
  "epoch": 0.63,
1231
- "learning_rate": 1.8205627320673836e-07,
1232
- "logits/chosen": -0.17955633997917175,
1233
- "logits/rejected": 0.18167546391487122,
1234
- "logps/chosen": -390.32244873046875,
1235
- "logps/rejected": -444.895263671875,
1236
- "loss": 0.5566,
1237
- "rewards/accuracies": 0.699999988079071,
1238
- "rewards/chosen": -2.4376220703125,
1239
- "rewards/margins": 0.7008293271064758,
1240
- "rewards/rejected": -3.138451099395752,
1241
- "step": 790
1242
- },
1243
- {
1244
- "epoch": 0.64,
1245
- "learning_rate": 1.7539457311884675e-07,
1246
- "logits/chosen": -0.09838727861642838,
1247
- "logits/rejected": 0.11829495429992676,
1248
- "logps/chosen": -402.4017333984375,
1249
- "logps/rejected": -451.49346923828125,
1250
- "loss": 0.5609,
1251
- "rewards/accuracies": 0.699999988079071,
1252
- "rewards/chosen": -2.4441986083984375,
1253
- "rewards/margins": 0.5067789554595947,
1254
- "rewards/rejected": -2.9509775638580322,
1255
- "step": 800
1256
- },
1257
- {
1258
- "epoch": 0.64,
1259
- "eval_logits/chosen": -0.03116540051996708,
1260
- "eval_logits/rejected": 0.1922437697649002,
1261
- "eval_logps/chosen": -387.7091979980469,
1262
- "eval_logps/rejected": -459.44390869140625,
1263
- "eval_loss": 0.5468714833259583,
1264
- "eval_rewards/accuracies": 0.7108209133148193,
1265
- "eval_rewards/chosen": -2.431356430053711,
1266
- "eval_rewards/margins": 0.6517390012741089,
1267
- "eval_rewards/rejected": -3.0830955505371094,
1268
- "eval_runtime": 184.5893,
1269
- "eval_samples_per_second": 46.33,
1270
- "eval_steps_per_second": 0.726,
1271
- "step": 800
1272
  },
1273
  {
1274
- "epoch": 0.64,
1275
- "learning_rate": 1.687905344471226e-07,
1276
- "logits/chosen": 0.07735608518123627,
1277
- "logits/rejected": 0.3973601460456848,
1278
- "logps/chosen": -408.05999755859375,
1279
- "logps/rejected": -459.011474609375,
1280
- "loss": 0.5384,
1281
- "rewards/accuracies": 0.706250011920929,
1282
- "rewards/chosen": -2.5008435249328613,
1283
- "rewards/margins": 0.6535352468490601,
1284
- "rewards/rejected": -3.154379367828369,
1285
- "step": 810
 
 
1286
  },
1287
  {
1288
  "epoch": 0.65,
1289
- "learning_rate": 1.6224926135406693e-07,
1290
- "logits/chosen": 0.1125444769859314,
1291
- "logits/rejected": 0.3865428566932678,
1292
- "logps/chosen": -404.16058349609375,
1293
- "logps/rejected": -484.68621826171875,
1294
- "loss": 0.5448,
1295
- "rewards/accuracies": 0.706250011920929,
1296
- "rewards/chosen": -2.4405789375305176,
1297
- "rewards/margins": 0.718208909034729,
1298
- "rewards/rejected": -3.158787727355957,
1299
- "step": 820
1300
- },
1301
- {
1302
- "epoch": 0.66,
1303
- "learning_rate": 1.557758094916053e-07,
1304
- "logits/chosen": 0.11989516019821167,
1305
- "logits/rejected": 0.30926594138145447,
1306
- "logps/chosen": -370.29876708984375,
1307
- "logps/rejected": -452.27911376953125,
1308
- "loss": 0.5418,
1309
- "rewards/accuracies": 0.737500011920929,
1310
- "rewards/chosen": -2.3860089778900146,
1311
- "rewards/margins": 0.7260924577713013,
1312
- "rewards/rejected": -3.1121015548706055,
1313
- "step": 830
1314
  },
1315
  {
1316
  "epoch": 0.67,
1317
- "learning_rate": 1.4937518209365108e-07,
1318
- "logits/chosen": -0.14239154756069183,
1319
- "logits/rejected": 0.14250756800174713,
1320
- "logps/chosen": -395.55755615234375,
1321
- "logps/rejected": -447.6368713378906,
1322
- "loss": 0.5573,
1323
- "rewards/accuracies": 0.7250000238418579,
1324
- "rewards/chosen": -2.378154754638672,
1325
- "rewards/margins": 0.6160937547683716,
1326
- "rewards/rejected": -2.994248390197754,
1327
- "step": 840
1328
- },
1329
- {
1330
- "epoch": 0.68,
1331
- "learning_rate": 1.4305232610918045e-07,
1332
- "logits/chosen": -0.16526366770267487,
1333
- "logits/rejected": 0.16432161629199982,
1334
- "logps/chosen": -373.45330810546875,
1335
- "logps/rejected": -436.6773376464844,
1336
- "loss": 0.5415,
1337
- "rewards/accuracies": 0.737500011920929,
1338
- "rewards/chosen": -2.3134028911590576,
1339
- "rewards/margins": 0.774810791015625,
1340
- "rewards/rejected": -3.0882136821746826,
1341
- "step": 850
1342
- },
1343
- {
1344
- "epoch": 0.68,
1345
- "learning_rate": 1.3681212837880977e-07,
1346
- "logits/chosen": -0.1321481615304947,
1347
- "logits/rejected": 0.23287932574748993,
1348
- "logps/chosen": -364.96990966796875,
1349
- "logps/rejected": -447.7923278808594,
1350
- "loss": 0.5396,
1351
- "rewards/accuracies": 0.7875000238418579,
1352
- "rewards/chosen": -2.176964282989502,
1353
- "rewards/margins": 0.8955341577529907,
1354
- "rewards/rejected": -3.0724985599517822,
1355
- "step": 860
1356
  },
1357
  {
1358
  "epoch": 0.69,
1359
- "learning_rate": 1.3065941185782977e-07,
1360
- "logits/chosen": 0.05437428876757622,
1361
- "logits/rejected": 0.2819867432117462,
1362
- "logps/chosen": -383.08599853515625,
1363
- "logps/rejected": -439.3629455566406,
1364
- "loss": 0.5505,
1365
- "rewards/accuracies": 0.637499988079071,
1366
- "rewards/chosen": -2.405247211456299,
1367
- "rewards/margins": 0.5403125882148743,
1368
- "rewards/rejected": -2.9455599784851074,
1369
- "step": 870
1370
- },
1371
- {
1372
- "epoch": 0.7,
1373
- "learning_rate": 1.2459893188861613e-07,
1374
- "logits/chosen": -0.12052659690380096,
1375
- "logits/rejected": 0.12284734100103378,
1376
- "logps/chosen": -367.1181640625,
1377
- "logps/rejected": -468.1044921875,
1378
- "loss": 0.5185,
1379
- "rewards/accuracies": 0.75,
1380
- "rewards/chosen": -2.39152455329895,
1381
- "rewards/margins": 0.9137696027755737,
1382
- "rewards/rejected": -3.3052947521209717,
1383
- "step": 880
1384
  },
1385
  {
1386
  "epoch": 0.71,
1387
- "learning_rate": 1.1863537252529548e-07,
1388
- "logits/chosen": 0.14598000049591064,
1389
- "logits/rejected": 0.38815659284591675,
1390
- "logps/chosen": -397.891357421875,
1391
- "logps/rejected": -472.38677978515625,
1392
- "loss": 0.5323,
1393
- "rewards/accuracies": 0.731249988079071,
1394
- "rewards/chosen": -2.512676239013672,
1395
- "rewards/margins": 0.7713057994842529,
1396
- "rewards/rejected": -3.2839818000793457,
1397
- "step": 890
1398
- },
1399
- {
1400
- "epoch": 0.72,
1401
- "learning_rate": 1.1277334291351145e-07,
1402
- "logits/chosen": 0.15319526195526123,
1403
- "logits/rejected": 0.35974830389022827,
1404
- "logps/chosen": -380.77783203125,
1405
- "logps/rejected": -449.54315185546875,
1406
- "loss": 0.5514,
1407
- "rewards/accuracies": 0.6812499761581421,
1408
- "rewards/chosen": -2.3706305027008057,
1409
- "rewards/margins": 0.6724039912223816,
1410
- "rewards/rejected": -3.043034076690674,
1411
- "step": 900
1412
- },
1413
- {
1414
- "epoch": 0.72,
1415
- "eval_logits/chosen": 0.28598034381866455,
1416
- "eval_logits/rejected": 0.5382024645805359,
1417
- "eval_logps/chosen": -392.3096008300781,
1418
- "eval_logps/rejected": -471.95330810546875,
1419
- "eval_loss": 0.5473664402961731,
1420
- "eval_rewards/accuracies": 0.6996268630027771,
1421
- "eval_rewards/chosen": -2.4773612022399902,
1422
- "eval_rewards/margins": 0.7308279275894165,
1423
- "eval_rewards/rejected": -3.2081892490386963,
1424
- "eval_runtime": 184.5275,
1425
- "eval_samples_per_second": 46.345,
1426
- "eval_steps_per_second": 0.726,
1427
- "step": 900
1428
- },
1429
- {
1430
- "epoch": 0.72,
1431
- "learning_rate": 1.0701737372808431e-07,
1432
- "logits/chosen": 0.15951867401599884,
1433
- "logits/rejected": 0.46630391478538513,
1434
- "logps/chosen": -383.52850341796875,
1435
- "logps/rejected": -467.2303771972656,
1436
- "loss": 0.5362,
1437
- "rewards/accuracies": 0.7124999761581421,
1438
- "rewards/chosen": -2.2782187461853027,
1439
- "rewards/margins": 0.8473943471908569,
1440
- "rewards/rejected": -3.125612735748291,
1441
- "step": 910
1442
  },
1443
  {
1444
  "epoch": 0.73,
1445
- "learning_rate": 1.0137191367132078e-07,
1446
- "logits/chosen": 0.2791319191455841,
1447
- "logits/rejected": 0.45174160599708557,
1448
- "logps/chosen": -372.1945495605469,
1449
- "logps/rejected": -446.6507263183594,
1450
- "loss": 0.5458,
1451
  "rewards/accuracies": 0.706250011920929,
1452
- "rewards/chosen": -2.3160648345947266,
1453
- "rewards/margins": 0.68004310131073,
1454
- "rewards/rejected": -2.996107816696167,
1455
- "step": 920
1456
- },
1457
- {
1458
- "epoch": 0.74,
1459
- "learning_rate": 9.584132603467827e-08,
1460
- "logits/chosen": -0.12192128598690033,
1461
- "logits/rejected": 0.1477951854467392,
1462
- "logps/chosen": -366.48321533203125,
1463
- "logps/rejected": -453.130126953125,
1464
- "loss": 0.5467,
1465
- "rewards/accuracies": 0.75,
1466
- "rewards/chosen": -2.200005531311035,
1467
- "rewards/margins": 0.7978888750076294,
1468
- "rewards/rejected": -2.997894287109375,
1469
- "step": 930
1470
  },
1471
  {
1472
  "epoch": 0.75,
1473
- "learning_rate": 9.042988532644249e-08,
1474
- "logits/chosen": -0.03106372058391571,
1475
- "logits/rejected": 0.07721444219350815,
1476
- "logps/chosen": -344.21270751953125,
1477
- "logps/rejected": -438.11077880859375,
1478
- "loss": 0.5161,
1479
- "rewards/accuracies": 0.71875,
1480
- "rewards/chosen": -2.149094581604004,
1481
- "rewards/margins": 0.7353444695472717,
1482
- "rewards/rejected": -2.884438991546631,
1483
- "step": 940
1484
- },
1485
- {
1486
- "epoch": 0.76,
1487
- "learning_rate": 8.514177396802428e-08,
1488
- "logits/chosen": 0.006801058538258076,
1489
- "logits/rejected": 0.20282092690467834,
1490
- "logps/chosen": -358.15167236328125,
1491
- "logps/rejected": -436.4964294433594,
1492
- "loss": 0.5385,
1493
- "rewards/accuracies": 0.7562500238418579,
1494
- "rewards/chosen": -2.2220425605773926,
1495
- "rewards/margins": 0.7004804611206055,
1496
- "rewards/rejected": -2.922523260116577,
1497
- "step": 950
1498
- },
1499
- {
1500
- "epoch": 0.76,
1501
- "learning_rate": 7.998107906142839e-08,
1502
- "logits/chosen": 0.41448846459388733,
1503
- "logits/rejected": 0.705254852771759,
1504
- "logps/chosen": -371.27801513671875,
1505
- "logps/rejected": -434.56866455078125,
1506
- "loss": 0.5236,
1507
- "rewards/accuracies": 0.7437499761581421,
1508
- "rewards/chosen": -2.2166616916656494,
1509
- "rewards/margins": 0.6714047193527222,
1510
- "rewards/rejected": -2.888066530227661,
1511
- "step": 960
1512
  },
1513
  {
1514
  "epoch": 0.77,
1515
- "learning_rate": 7.495178923039396e-08,
1516
- "logits/chosen": 0.23847150802612305,
1517
- "logits/rejected": 0.48661884665489197,
1518
- "logps/chosen": -366.28179931640625,
1519
- "logps/rejected": -462.679443359375,
1520
- "loss": 0.5459,
1521
- "rewards/accuracies": 0.7437499761581421,
1522
- "rewards/chosen": -2.1916985511779785,
1523
- "rewards/margins": 0.8472123146057129,
1524
- "rewards/rejected": -3.038910388946533,
1525
- "step": 970
1526
- },
1527
- {
1528
- "epoch": 0.78,
1529
- "learning_rate": 7.005779153764682e-08,
1530
- "logits/chosen": 0.41438961029052734,
1531
- "logits/rejected": 0.6912784576416016,
1532
- "logps/chosen": -382.70123291015625,
1533
- "logps/rejected": -461.8614807128906,
1534
- "loss": 0.5453,
1535
- "rewards/accuracies": 0.737500011920929,
1536
- "rewards/chosen": -2.4304287433624268,
1537
- "rewards/margins": 0.7116767764091492,
1538
- "rewards/rejected": -3.1421055793762207,
1539
- "step": 980
1540
  },
1541
  {
1542
  "epoch": 0.79,
1543
- "learning_rate": 6.530286848064698e-08,
1544
- "logits/chosen": 0.36573725938796997,
1545
- "logits/rejected": 0.5834362506866455,
1546
- "logps/chosen": -384.49749755859375,
1547
- "logps/rejected": -466.30096435546875,
1548
- "loss": 0.5528,
1549
  "rewards/accuracies": 0.71875,
1550
- "rewards/chosen": -2.5111565589904785,
1551
- "rewards/margins": 0.7234699130058289,
1552
- "rewards/rejected": -3.234626054763794,
1553
- "step": 990
1554
- },
1555
- {
1556
- "epoch": 0.8,
1557
- "learning_rate": 6.069069506815325e-08,
1558
- "logits/chosen": 0.45530566573143005,
1559
- "logits/rejected": 0.5909157991409302,
1560
- "logps/chosen": -379.1433410644531,
1561
- "logps/rejected": -468.88458251953125,
1562
- "loss": 0.527,
1563
- "rewards/accuracies": 0.762499988079071,
1564
- "rewards/chosen": -2.5361268520355225,
1565
- "rewards/margins": 0.7407721281051636,
1566
- "rewards/rejected": -3.2768986225128174,
1567
- "step": 1000
1568
- },
1569
- {
1570
- "epoch": 0.8,
1571
- "eval_logits/chosen": 0.3871051073074341,
1572
- "eval_logits/rejected": 0.6372014284133911,
1573
- "eval_logps/chosen": -394.97113037109375,
1574
- "eval_logps/rejected": -471.8453674316406,
1575
- "eval_loss": 0.5453863739967346,
1576
- "eval_rewards/accuracies": 0.70802241563797,
1577
- "eval_rewards/chosen": -2.503976345062256,
1578
- "eval_rewards/margins": 0.7031334638595581,
1579
- "eval_rewards/rejected": -3.2071101665496826,
1580
- "eval_runtime": 184.4226,
1581
- "eval_samples_per_second": 46.372,
1582
- "eval_steps_per_second": 0.727,
1583
- "step": 1000
1584
- },
1585
- {
1586
- "epoch": 0.8,
1587
- "learning_rate": 5.6224835979863714e-08,
1588
- "logits/chosen": 0.31174296140670776,
1589
- "logits/rejected": 0.6193565130233765,
1590
- "logps/chosen": -390.387451171875,
1591
- "logps/rejected": -468.4959411621094,
1592
- "loss": 0.5568,
1593
- "rewards/accuracies": 0.71875,
1594
- "rewards/chosen": -2.32766056060791,
1595
- "rewards/margins": 0.68747878074646,
1596
- "rewards/rejected": -3.015139102935791,
1597
- "step": 1010
1598
- },
1599
- {
1600
- "epoch": 0.81,
1601
- "learning_rate": 5.190874281132851e-08,
1602
- "logits/chosen": 0.22277125716209412,
1603
- "logits/rejected": 0.6487134099006653,
1604
- "logps/chosen": -402.0958557128906,
1605
- "logps/rejected": -448.5992736816406,
1606
- "loss": 0.5408,
1607
- "rewards/accuracies": 0.71875,
1608
- "rewards/chosen": -2.359062671661377,
1609
- "rewards/margins": 0.6533006429672241,
1610
- "rewards/rejected": -3.0123631954193115,
1611
- "step": 1020
1612
  },
1613
  {
1614
  "epoch": 0.82,
1615
- "learning_rate": 4.774575140626316e-08,
1616
- "logits/chosen": 0.23170511424541473,
1617
- "logits/rejected": 0.47184085845947266,
1618
- "logps/chosen": -363.46917724609375,
1619
- "logps/rejected": -442.47918701171875,
1620
- "loss": 0.5309,
1621
  "rewards/accuracies": 0.7437499761581421,
1622
- "rewards/chosen": -2.142770290374756,
1623
- "rewards/margins": 0.7513145208358765,
1624
- "rewards/rejected": -2.894084930419922,
1625
- "step": 1030
1626
- },
1627
- {
1628
- "epoch": 0.83,
1629
- "learning_rate": 4.373907927832513e-08,
1630
- "logits/chosen": 0.07573021948337555,
1631
- "logits/rejected": 0.32997313141822815,
1632
- "logps/chosen": -381.45599365234375,
1633
- "logps/rejected": -443.0684509277344,
1634
- "loss": 0.5407,
1635
- "rewards/accuracies": 0.7250000238418579,
1636
- "rewards/chosen": -2.2033116817474365,
1637
- "rewards/margins": 0.710732638835907,
1638
- "rewards/rejected": -2.914044141769409,
1639
- "step": 1040
1640
  },
1641
  {
1642
  "epoch": 0.84,
1643
- "learning_rate": 3.9891823124345665e-08,
1644
- "logits/chosen": 0.23884686827659607,
1645
- "logits/rejected": 0.6128005385398865,
1646
- "logps/chosen": -364.00567626953125,
1647
- "logps/rejected": -433.3273010253906,
1648
- "loss": 0.5471,
1649
- "rewards/accuracies": 0.731249988079071,
1650
- "rewards/chosen": -2.2081527709960938,
1651
- "rewards/margins": 0.7639263868331909,
1652
- "rewards/rejected": -2.972079038619995,
1653
- "step": 1050
1654
  },
1655
  {
1656
  "epoch": 0.84,
1657
- "learning_rate": 3.620695643093924e-08,
1658
- "logits/chosen": 0.21963253617286682,
1659
- "logits/rejected": 0.6894062757492065,
1660
- "logps/chosen": -399.5767517089844,
1661
- "logps/rejected": -452.88909912109375,
1662
- "loss": 0.5154,
1663
- "rewards/accuracies": 0.71875,
1664
- "rewards/chosen": -2.3384335041046143,
1665
- "rewards/margins": 0.7010769844055176,
1666
- "rewards/rejected": -3.0395102500915527,
1667
- "step": 1060
1668
- },
1669
- {
1670
- "epoch": 0.85,
1671
- "learning_rate": 3.268732717634032e-08,
1672
- "logits/chosen": 0.3474286198616028,
1673
- "logits/rejected": 0.695271372795105,
1674
- "logps/chosen": -368.0654602050781,
1675
- "logps/rejected": -431.47222900390625,
1676
- "loss": 0.5499,
1677
- "rewards/accuracies": 0.731249988079071,
1678
- "rewards/chosen": -2.1910276412963867,
1679
- "rewards/margins": 0.7267633080482483,
1680
- "rewards/rejected": -2.9177908897399902,
1681
- "step": 1070
1682
  },
1683
  {
1684
  "epoch": 0.86,
1685
- "learning_rate": 2.9335655629243645e-08,
1686
- "logits/chosen": 0.2347393035888672,
1687
- "logits/rejected": 0.5894696712493896,
1688
- "logps/chosen": -388.94757080078125,
1689
- "logps/rejected": -447.3855895996094,
1690
- "loss": 0.525,
1691
- "rewards/accuracies": 0.78125,
1692
- "rewards/chosen": -2.254683017730713,
1693
- "rewards/margins": 0.7334609031677246,
1694
- "rewards/rejected": -2.9881439208984375,
1695
- "step": 1080
1696
- },
1697
- {
1698
- "epoch": 0.87,
1699
- "learning_rate": 2.6154532246349476e-08,
1700
- "logits/chosen": 0.25378522276878357,
1701
- "logits/rejected": 0.5771256685256958,
1702
- "logps/chosen": -358.50640869140625,
1703
- "logps/rejected": -431.145751953125,
1704
- "loss": 0.5462,
1705
- "rewards/accuracies": 0.7250000238418579,
1706
- "rewards/chosen": -2.1666626930236816,
1707
- "rewards/margins": 0.756801426410675,
1708
- "rewards/rejected": -2.923464059829712,
1709
- "step": 1090
1710
- },
1711
- {
1712
- "epoch": 0.88,
1713
- "learning_rate": 2.31464156702382e-08,
1714
- "logits/chosen": 0.35370689630508423,
1715
- "logits/rejected": 0.5671936273574829,
1716
- "logps/chosen": -363.0,
1717
- "logps/rejected": -438.209228515625,
1718
- "loss": 0.5487,
1719
- "rewards/accuracies": 0.7250000238418579,
1720
- "rewards/chosen": -2.2130119800567627,
1721
- "rewards/margins": 0.7499077916145325,
1722
- "rewards/rejected": -2.9629194736480713,
1723
- "step": 1100
1724
- },
1725
- {
1726
- "epoch": 0.88,
1727
- "eval_logits/chosen": 0.1857856959104538,
1728
- "eval_logits/rejected": 0.43363669514656067,
1729
- "eval_logps/chosen": -373.08306884765625,
1730
- "eval_logps/rejected": -450.7598876953125,
1731
- "eval_loss": 0.5444055199623108,
1732
- "eval_rewards/accuracies": 0.7089552283287048,
1733
- "eval_rewards/chosen": -2.285095453262329,
1734
- "eval_rewards/margins": 0.711159884929657,
1735
- "eval_rewards/rejected": -2.996255397796631,
1736
- "eval_runtime": 184.462,
1737
- "eval_samples_per_second": 46.362,
1738
- "eval_steps_per_second": 0.726,
1739
- "step": 1100
1740
  },
1741
  {
1742
  "epoch": 0.88,
1743
- "learning_rate": 2.031363082912252e-08,
1744
- "logits/chosen": 0.070524200797081,
1745
- "logits/rejected": 0.4635602533817291,
1746
- "logps/chosen": -373.29327392578125,
1747
- "logps/rejected": -426.85552978515625,
1748
- "loss": 0.5513,
1749
- "rewards/accuracies": 0.6937500238418579,
1750
- "rewards/chosen": -2.2541089057922363,
1751
- "rewards/margins": 0.6198171973228455,
1752
- "rewards/rejected": -2.8739261627197266,
1753
- "step": 1110
1754
- },
1755
- {
1756
- "epoch": 0.89,
1757
- "learning_rate": 1.7658367139945228e-08,
1758
- "logits/chosen": 0.2600646913051605,
1759
- "logits/rejected": 0.5517584681510925,
1760
- "logps/chosen": -390.8568115234375,
1761
- "logps/rejected": -462.80828857421875,
1762
- "loss": 0.5471,
1763
- "rewards/accuracies": 0.6875,
1764
- "rewards/chosen": -2.377396821975708,
1765
- "rewards/margins": 0.6719989776611328,
1766
- "rewards/rejected": -3.049395799636841,
1767
- "step": 1120
1768
  },
1769
  {
1770
  "epoch": 0.9,
1771
- "learning_rate": 1.5182676816211632e-08,
1772
- "logits/chosen": 0.04413030296564102,
1773
- "logits/rejected": 0.30151715874671936,
1774
- "logps/chosen": -382.0662536621094,
1775
- "logps/rejected": -447.08673095703125,
1776
- "loss": 0.5431,
1777
- "rewards/accuracies": 0.7124999761581421,
1778
- "rewards/chosen": -2.228654384613037,
1779
- "rewards/margins": 0.6926024556159973,
1780
- "rewards/rejected": -2.9212570190429688,
1781
- "step": 1130
1782
- },
1783
- {
1784
- "epoch": 0.91,
1785
- "learning_rate": 1.2888473281864597e-08,
1786
- "logits/chosen": 0.14212054014205933,
1787
- "logits/rejected": 0.47429710626602173,
1788
- "logps/chosen": -367.8409729003906,
1789
- "logps/rejected": -435.02764892578125,
1790
- "loss": 0.5369,
1791
- "rewards/accuracies": 0.768750011920929,
1792
- "rewards/chosen": -2.2534115314483643,
1793
- "rewards/margins": 0.7273036241531372,
1794
- "rewards/rejected": -2.980715274810791,
1795
- "step": 1140
1796
- },
1797
- {
1798
- "epoch": 0.92,
1799
- "learning_rate": 1.0777529692427679e-08,
1800
- "logits/chosen": 0.04115242511034012,
1801
- "logits/rejected": 0.28970104455947876,
1802
- "logps/chosen": -372.7949523925781,
1803
- "logps/rejected": -456.10675048828125,
1804
- "loss": 0.5265,
1805
- "rewards/accuracies": 0.731249988079071,
1806
- "rewards/chosen": -2.300356388092041,
1807
- "rewards/margins": 0.8059718012809753,
1808
- "rewards/rejected": -3.106328248977661,
1809
- "step": 1150
1810
  },
1811
  {
1812
  "epoch": 0.92,
1813
- "learning_rate": 8.851477564560061e-09,
1814
- "logits/chosen": 0.0867738351225853,
1815
- "logits/rejected": 0.4068300127983093,
1816
- "logps/chosen": -372.08636474609375,
1817
- "logps/rejected": -426.42388916015625,
1818
- "loss": 0.5342,
1819
  "rewards/accuracies": 0.6937500238418579,
1820
- "rewards/chosen": -2.331385850906372,
1821
- "rewards/margins": 0.6490964293479919,
1822
- "rewards/rejected": -2.9804821014404297,
1823
- "step": 1160
1824
- },
1825
- {
1826
- "epoch": 0.93,
1827
- "learning_rate": 7.111805515081531e-09,
1828
- "logits/chosen": 0.02022993005812168,
1829
- "logits/rejected": 0.41968393325805664,
1830
- "logps/chosen": -363.818603515625,
1831
- "logps/rejected": -447.7919006347656,
1832
- "loss": 0.5312,
1833
- "rewards/accuracies": 0.7562500238418579,
1834
- "rewards/chosen": -2.2372307777404785,
1835
- "rewards/margins": 0.8540315628051758,
1836
- "rewards/rejected": -3.0912623405456543,
1837
- "step": 1170
1838
  },
1839
  {
1840
  "epoch": 0.94,
1841
- "learning_rate": 5.559858110443016e-09,
1842
- "logits/chosen": 0.29695388674736023,
1843
- "logits/rejected": 0.714096188545227,
1844
- "logps/chosen": -372.5519714355469,
1845
- "logps/rejected": -442.5354919433594,
1846
- "loss": 0.5383,
1847
- "rewards/accuracies": 0.71875,
1848
- "rewards/chosen": -2.3107995986938477,
1849
- "rewards/margins": 0.8070123791694641,
1850
- "rewards/rejected": -3.117811918258667,
1851
- "step": 1180
1852
- },
1853
- {
1854
- "epoch": 0.95,
1855
- "learning_rate": 4.196834827531276e-09,
1856
- "logits/chosen": 0.140055850148201,
1857
- "logits/rejected": 0.3409932255744934,
1858
- "logps/chosen": -355.64324951171875,
1859
- "logps/rejected": -447.585693359375,
1860
- "loss": 0.5152,
1861
- "rewards/accuracies": 0.737500011920929,
1862
- "rewards/chosen": -2.149151563644409,
1863
- "rewards/margins": 0.7904965877532959,
1864
- "rewards/rejected": -2.939648151397705,
1865
- "step": 1190
1866
- },
1867
- {
1868
- "epoch": 0.96,
1869
- "learning_rate": 3.023789126611137e-09,
1870
- "logits/chosen": 0.03294936567544937,
1871
- "logits/rejected": 0.2933207154273987,
1872
- "logps/chosen": -363.29290771484375,
1873
- "logps/rejected": -435.640380859375,
1874
- "loss": 0.5483,
1875
- "rewards/accuracies": 0.699999988079071,
1876
- "rewards/chosen": -2.281057357788086,
1877
- "rewards/margins": 0.7091296911239624,
1878
- "rewards/rejected": -2.990186929702759,
1879
- "step": 1200
1880
- },
1881
- {
1882
- "epoch": 0.96,
1883
- "eval_logits/chosen": 0.07418080419301987,
1884
- "eval_logits/rejected": 0.32435521483421326,
1885
- "eval_logps/chosen": -373.978515625,
1886
- "eval_logps/rejected": -451.6764831542969,
1887
- "eval_loss": 0.5440130829811096,
1888
- "eval_rewards/accuracies": 0.7089552283287048,
1889
- "eval_rewards/chosen": -2.2940499782562256,
1890
- "eval_rewards/margins": 0.7113713622093201,
1891
- "eval_rewards/rejected": -3.0054211616516113,
1892
- "eval_runtime": 184.3589,
1893
- "eval_samples_per_second": 46.388,
1894
- "eval_steps_per_second": 0.727,
1895
- "step": 1200
1896
  },
1897
  {
1898
  "epoch": 0.96,
1899
- "learning_rate": 2.041627637121929e-09,
1900
- "logits/chosen": 0.10010697692632675,
1901
- "logits/rejected": 0.3795483410358429,
1902
- "logps/chosen": -348.8675231933594,
1903
- "logps/rejected": -437.20361328125,
1904
- "loss": 0.5398,
1905
- "rewards/accuracies": 0.7749999761581421,
1906
- "rewards/chosen": -2.087364673614502,
1907
- "rewards/margins": 0.828387439250946,
1908
- "rewards/rejected": -2.9157521724700928,
1909
- "step": 1210
1910
- },
1911
- {
1912
- "epoch": 0.97,
1913
- "learning_rate": 1.2511094569571668e-09,
1914
- "logits/chosen": 0.09991980344057083,
1915
- "logits/rejected": 0.4467397630214691,
1916
- "logps/chosen": -380.14520263671875,
1917
- "logps/rejected": -440.24658203125,
1918
- "loss": 0.5345,
1919
- "rewards/accuracies": 0.737500011920929,
1920
- "rewards/chosen": -2.253425121307373,
1921
- "rewards/margins": 0.702509880065918,
1922
- "rewards/rejected": -2.955935001373291,
1923
- "step": 1220
1924
  },
1925
  {
1926
  "epoch": 0.98,
1927
- "learning_rate": 6.528455657691112e-10,
1928
- "logits/chosen": 0.11626466363668442,
1929
- "logits/rejected": 0.41348797082901,
1930
- "logps/chosen": -372.7298889160156,
1931
- "logps/rejected": -427.22576904296875,
1932
- "loss": 0.549,
1933
- "rewards/accuracies": 0.762499988079071,
1934
- "rewards/chosen": -2.2800345420837402,
1935
- "rewards/margins": 0.6291176080703735,
1936
- "rewards/rejected": -2.909151792526245,
1937
- "step": 1230
1938
- },
1939
- {
1940
- "epoch": 0.99,
1941
- "learning_rate": 2.4729835275189016e-10,
1942
- "logits/chosen": 0.06715863198041916,
1943
- "logits/rejected": 0.29241910576820374,
1944
- "logps/chosen": -393.8903503417969,
1945
- "logps/rejected": -477.9420471191406,
1946
- "loss": 0.5462,
1947
- "rewards/accuracies": 0.737500011920929,
1948
- "rewards/chosen": -2.323488473892212,
1949
- "rewards/margins": 0.8067766427993774,
1950
- "rewards/rejected": -3.1302647590637207,
1951
- "step": 1240
1952
- },
1953
- {
1954
- "epoch": 0.99,
1955
- "learning_rate": 3.478125926756337e-11,
1956
- "logits/chosen": 0.25983649492263794,
1957
- "logits/rejected": 0.4905417561531067,
1958
- "logps/chosen": -364.73431396484375,
1959
- "logps/rejected": -443.79296875,
1960
- "loss": 0.5474,
1961
- "rewards/accuracies": 0.7250000238418579,
1962
- "rewards/chosen": -2.2353272438049316,
1963
- "rewards/margins": 0.771331787109375,
1964
- "rewards/rejected": -3.0066590309143066,
1965
- "step": 1250
1966
  },
1967
  {
1968
  "epoch": 1.0,
1969
- "step": 1256,
1970
  "total_flos": 0.0,
1971
- "train_loss": 0.5712926928784438,
1972
- "train_runtime": 11573.5451,
1973
- "train_samples_per_second": 13.894,
1974
  "train_steps_per_second": 0.109
1975
  }
1976
  ],
1977
  "logging_steps": 10,
1978
- "max_steps": 1256,
1979
  "num_train_epochs": 1,
1980
  "save_steps": 100,
1981
  "total_flos": 0.0,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
  "eval_steps": 100,
6
+ "global_step": 478,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.0416666666666666e-08,
14
+ "logits/chosen": -2.847970962524414,
15
+ "logits/rejected": -2.79160213470459,
16
+ "logps/chosen": -284.9612731933594,
17
+ "logps/rejected": -276.45928955078125,
18
+ "loss": 0.2385,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
23
  "step": 1
24
  },
25
  {
26
+ "epoch": 0.02,
27
+ "learning_rate": 1.0416666666666667e-07,
28
+ "logits/chosen": -2.7546489238739014,
29
+ "logits/rejected": -2.75274658203125,
30
+ "logps/chosen": -249.91802978515625,
31
+ "logps/rejected": -223.00167846679688,
32
+ "loss": 0.2255,
33
+ "rewards/accuracies": 0.4097222089767456,
34
+ "rewards/chosen": 0.0002972287475131452,
35
+ "rewards/margins": -0.00016528893320355564,
36
+ "rewards/rejected": 0.00046251772437244654,
37
  "step": 10
38
  },
39
  {
40
+ "epoch": 0.04,
41
+ "learning_rate": 2.0833333333333333e-07,
42
+ "logits/chosen": -2.7447714805603027,
43
+ "logits/rejected": -2.7452731132507324,
44
+ "logps/chosen": -257.4560852050781,
45
+ "logps/rejected": -247.54714965820312,
46
+ "loss": 0.2261,
47
+ "rewards/accuracies": 0.574999988079071,
48
+ "rewards/chosen": -5.62463401365676e-06,
49
+ "rewards/margins": 0.0011592062655836344,
50
+ "rewards/rejected": -0.001164830755442381,
51
  "step": 20
52
  },
53
  {
54
+ "epoch": 0.06,
55
+ "learning_rate": 3.1249999999999997e-07,
56
+ "logits/chosen": -2.8000121116638184,
57
+ "logits/rejected": -2.7526602745056152,
58
+ "logps/chosen": -300.5664978027344,
59
+ "logps/rejected": -261.8959045410156,
60
+ "loss": 0.2276,
61
+ "rewards/accuracies": 0.668749988079071,
62
+ "rewards/chosen": 0.00010534003376960754,
63
+ "rewards/margins": 0.008146253414452076,
64
+ "rewards/rejected": -0.008040914312005043,
65
  "step": 30
66
  },
67
  {
68
+ "epoch": 0.08,
69
+ "learning_rate": 4.1666666666666667e-07,
70
+ "logits/chosen": -2.759944200515747,
71
+ "logits/rejected": -2.747865915298462,
72
+ "logps/chosen": -256.8673400878906,
73
+ "logps/rejected": -274.7678527832031,
74
+ "loss": 0.2263,
75
+ "rewards/accuracies": 0.706250011920929,
76
+ "rewards/chosen": -0.004334479570388794,
77
+ "rewards/margins": 0.0225108303129673,
78
+ "rewards/rejected": -0.026845309883356094,
79
  "step": 40
80
  },
81
  {
82
+ "epoch": 0.1,
83
+ "learning_rate": 4.999733114418725e-07,
84
+ "logits/chosen": -2.7562313079833984,
85
+ "logits/rejected": -2.7282943725585938,
86
+ "logps/chosen": -285.93524169921875,
87
+ "logps/rejected": -256.98223876953125,
88
+ "loss": 0.2144,
89
+ "rewards/accuracies": 0.637499988079071,
90
+ "rewards/chosen": -0.038558270782232285,
91
+ "rewards/margins": 0.05423177406191826,
92
+ "rewards/rejected": -0.09279005229473114,
93
  "step": 50
94
  },
95
  {
96
+ "epoch": 0.13,
97
+ "learning_rate": 4.990398100856366e-07,
98
+ "logits/chosen": -2.71358060836792,
99
+ "logits/rejected": -2.7017626762390137,
100
+ "logps/chosen": -286.7482604980469,
101
+ "logps/rejected": -260.0042724609375,
102
+ "loss": 0.1968,
103
+ "rewards/accuracies": 0.668749988079071,
104
+ "rewards/chosen": -0.10863776504993439,
105
+ "rewards/margins": 0.09092625230550766,
106
+ "rewards/rejected": -0.19956400990486145,
107
  "step": 60
108
  },
109
  {
110
+ "epoch": 0.15,
111
+ "learning_rate": 4.967775735898179e-07,
112
+ "logits/chosen": -2.7547988891601562,
113
+ "logits/rejected": -2.7131030559539795,
114
+ "logps/chosen": -284.3356018066406,
115
+ "logps/rejected": -262.32232666015625,
116
+ "loss": 0.1734,
117
+ "rewards/accuracies": 0.675000011920929,
118
+ "rewards/chosen": -0.1720018982887268,
119
+ "rewards/margins": 0.13181492686271667,
120
+ "rewards/rejected": -0.30381685495376587,
121
  "step": 70
122
  },
123
  {
124
+ "epoch": 0.17,
125
+ "learning_rate": 4.931986719649298e-07,
126
+ "logits/chosen": -2.7598519325256348,
127
+ "logits/rejected": -2.7336292266845703,
128
+ "logps/chosen": -283.93646240234375,
129
+ "logps/rejected": -316.5826721191406,
130
+ "loss": 0.1495,
131
+ "rewards/accuracies": 0.643750011920929,
132
+ "rewards/chosen": -0.23583512008190155,
133
+ "rewards/margins": 0.20701651275157928,
134
+ "rewards/rejected": -0.44285163283348083,
135
  "step": 80
136
  },
137
  {
138
+ "epoch": 0.19,
139
+ "learning_rate": 4.883222001996351e-07,
140
+ "logits/chosen": -2.7855494022369385,
141
+ "logits/rejected": -2.7607643604278564,
142
+ "logps/chosen": -322.19610595703125,
143
+ "logps/rejected": -328.61712646484375,
144
+ "loss": 0.1087,
145
+ "rewards/accuracies": 0.706250011920929,
146
+ "rewards/chosen": -0.46829262375831604,
147
+ "rewards/margins": 0.3058927655220032,
148
+ "rewards/rejected": -0.7741854190826416,
149
  "step": 90
150
  },
151
  {
152
+ "epoch": 0.21,
153
+ "learning_rate": 4.821741763807186e-07,
154
+ "logits/chosen": -2.768531322479248,
155
+ "logits/rejected": -2.7363836765289307,
156
+ "logps/chosen": -355.22882080078125,
157
+ "logps/rejected": -360.6455383300781,
158
+ "loss": 0.0933,
159
+ "rewards/accuracies": 0.6937500238418579,
160
+ "rewards/chosen": -0.6626380681991577,
161
+ "rewards/margins": 0.3216686546802521,
162
+ "rewards/rejected": -0.9843066930770874,
163
  "step": 100
164
  },
165
  {
166
+ "epoch": 0.21,
167
+ "eval_logits/chosen": -2.7311501502990723,
168
+ "eval_logits/rejected": -2.7164700031280518,
169
+ "eval_logps/chosen": -326.00994873046875,
170
+ "eval_logps/rejected": -362.9189147949219,
171
+ "eval_loss": 0.0944075807929039,
172
+ "eval_rewards/accuracies": 0.671875,
173
+ "eval_rewards/chosen": -0.689703106880188,
174
+ "eval_rewards/margins": 0.3659532368183136,
175
+ "eval_rewards/rejected": -1.0556563138961792,
176
+ "eval_runtime": 53.6784,
177
+ "eval_samples_per_second": 37.259,
178
+ "eval_steps_per_second": 0.596,
179
  "step": 100
180
  },
181
  {
182
+ "epoch": 0.23,
183
+ "learning_rate": 4.747874028753375e-07,
184
+ "logits/chosen": -2.6889359951019287,
185
+ "logits/rejected": -2.684002637863159,
186
+ "logps/chosen": -335.4092102050781,
187
+ "logps/rejected": -388.6229553222656,
188
+ "loss": 0.0874,
189
+ "rewards/accuracies": 0.7250000238418579,
190
+ "rewards/chosen": -0.7114591002464294,
191
+ "rewards/margins": 0.44765734672546387,
192
+ "rewards/rejected": -1.1591163873672485,
193
  "step": 110
194
  },
195
  {
196
+ "epoch": 0.25,
197
+ "learning_rate": 4.662012913161997e-07,
198
+ "logits/chosen": -2.678748369216919,
199
+ "logits/rejected": -2.6456408500671387,
200
+ "logps/chosen": -351.7953186035156,
201
+ "logps/rejected": -368.17962646484375,
202
+ "loss": 0.0805,
203
+ "rewards/accuracies": 0.643750011920929,
204
+ "rewards/chosen": -0.8425552248954773,
205
+ "rewards/margins": 0.3654478192329407,
206
+ "rewards/rejected": -1.208003044128418,
207
  "step": 120
208
  },
209
  {
210
+ "epoch": 0.27,
211
+ "learning_rate": 4.5646165232345103e-07,
212
+ "logits/chosen": -2.640324831008911,
213
+ "logits/rejected": -2.607522487640381,
214
+ "logps/chosen": -386.58087158203125,
215
+ "logps/rejected": -372.4346618652344,
216
+ "loss": 0.0802,
217
+ "rewards/accuracies": 0.668749988079071,
218
+ "rewards/chosen": -0.9444938898086548,
219
+ "rewards/margins": 0.37742048501968384,
220
+ "rewards/rejected": -1.3219144344329834,
221
  "step": 130
222
  },
223
  {
224
+ "epoch": 0.29,
225
+ "learning_rate": 4.456204510851956e-07,
226
+ "logits/chosen": -2.549278736114502,
227
+ "logits/rejected": -2.5322203636169434,
228
+ "logps/chosen": -339.7931213378906,
229
+ "logps/rejected": -353.47943115234375,
230
+ "loss": 0.0745,
231
+ "rewards/accuracies": 0.6187499761581421,
232
+ "rewards/chosen": -0.9143403172492981,
233
+ "rewards/margins": 0.42283686995506287,
234
+ "rewards/rejected": -1.3371771574020386,
235
  "step": 140
236
  },
237
  {
238
+ "epoch": 0.31,
239
+ "learning_rate": 4.337355301007335e-07,
240
+ "logits/chosen": -2.598860502243042,
241
+ "logits/rejected": -2.588347911834717,
242
+ "logps/chosen": -336.6380615234375,
243
+ "logps/rejected": -346.9957275390625,
244
+ "loss": 0.0911,
245
+ "rewards/accuracies": 0.6625000238418579,
246
+ "rewards/chosen": -0.7140481472015381,
247
+ "rewards/margins": 0.3562600612640381,
248
+ "rewards/rejected": -1.0703080892562866,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.33,
253
+ "learning_rate": 4.2087030056579986e-07,
254
+ "logits/chosen": -2.583531141281128,
255
+ "logits/rejected": -2.569545269012451,
256
+ "logps/chosen": -327.7892150878906,
257
+ "logps/rejected": -342.8360290527344,
258
+ "loss": 0.0878,
259
+ "rewards/accuracies": 0.6937500238418579,
260
+ "rewards/chosen": -0.668094277381897,
261
+ "rewards/margins": 0.4449305534362793,
262
+ "rewards/rejected": -1.1130248308181763,
263
  "step": 160
264
  },
265
  {
266
+ "epoch": 0.36,
267
+ "learning_rate": 4.070934040463998e-07,
268
+ "logits/chosen": -2.5163466930389404,
269
+ "logits/rejected": -2.4945168495178223,
270
+ "logps/chosen": -360.7650451660156,
271
+ "logps/rejected": -385.5032653808594,
272
+ "loss": 0.0759,
273
+ "rewards/accuracies": 0.699999988079071,
274
+ "rewards/chosen": -0.9210854768753052,
275
+ "rewards/margins": 0.42766818404197693,
276
+ "rewards/rejected": -1.3487536907196045,
277
  "step": 170
278
  },
279
  {
280
+ "epoch": 0.38,
281
+ "learning_rate": 3.9247834624635404e-07,
282
+ "logits/chosen": -2.569969654083252,
283
+ "logits/rejected": -2.5289721488952637,
284
+ "logps/chosen": -372.3882751464844,
285
+ "logps/rejected": -406.4942626953125,
286
+ "loss": 0.0648,
287
+ "rewards/accuracies": 0.737500011920929,
288
+ "rewards/chosen": -0.959389328956604,
289
+ "rewards/margins": 0.6261054277420044,
290
+ "rewards/rejected": -1.5854947566986084,
291
  "step": 180
292
  },
293
  {
294
+ "epoch": 0.4,
295
+ "learning_rate": 3.7710310482256523e-07,
296
+ "logits/chosen": -2.4658422470092773,
297
+ "logits/rejected": -2.4459939002990723,
298
+ "logps/chosen": -377.95208740234375,
299
+ "logps/rejected": -404.22808837890625,
300
+ "loss": 0.0677,
301
+ "rewards/accuracies": 0.6937500238418579,
302
+ "rewards/chosen": -1.2073966264724731,
303
+ "rewards/margins": 0.4666738510131836,
304
+ "rewards/rejected": -1.6740705966949463,
305
  "step": 190
306
  },
307
  {
308
+ "epoch": 0.42,
309
+ "learning_rate": 3.610497133404795e-07,
310
+ "logits/chosen": -2.5809519290924072,
311
+ "logits/rejected": -2.5254673957824707,
312
+ "logps/chosen": -427.1067810058594,
313
+ "logps/rejected": -449.2569885253906,
314
+ "loss": 0.0726,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
315
  "rewards/accuracies": 0.699999988079071,
316
+ "rewards/chosen": -0.9253028035163879,
317
+ "rewards/margins": 0.7682465314865112,
318
+ "rewards/rejected": -1.693549394607544,
319
+ "step": 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
320
  },
321
  {
322
  "epoch": 0.42,
323
+ "eval_logits/chosen": -2.5482492446899414,
324
+ "eval_logits/rejected": -2.5344252586364746,
325
+ "eval_logps/chosen": -351.503662109375,
326
+ "eval_logps/rejected": -416.43280029296875,
327
+ "eval_loss": 0.07468212395906448,
328
+ "eval_rewards/accuracies": 0.70703125,
329
+ "eval_rewards/chosen": -0.9446406364440918,
330
+ "eval_rewards/margins": 0.6461541652679443,
331
+ "eval_rewards/rejected": -1.5907946825027466,
332
+ "eval_runtime": 53.6388,
333
+ "eval_samples_per_second": 37.286,
334
+ "eval_steps_per_second": 0.597,
335
+ "step": 200
 
 
 
 
 
 
 
 
 
 
 
 
336
  },
337
  {
338
  "epoch": 0.44,
339
+ "learning_rate": 3.4440382358952115e-07,
340
+ "logits/chosen": -2.523655414581299,
341
+ "logits/rejected": -2.480741500854492,
342
+ "logps/chosen": -377.5237121582031,
343
+ "logps/rejected": -400.14324951171875,
344
+ "loss": 0.0693,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
345
  "rewards/accuracies": 0.6875,
346
+ "rewards/chosen": -1.1031700372695923,
347
+ "rewards/margins": 0.4939765930175781,
348
+ "rewards/rejected": -1.59714674949646,
349
+ "step": 210
350
  },
351
  {
352
  "epoch": 0.46,
353
+ "learning_rate": 3.272542485937368e-07,
354
+ "logits/chosen": -2.495047092437744,
355
+ "logits/rejected": -2.485301971435547,
356
+ "logps/chosen": -411.7909240722656,
357
+ "logps/rejected": -421.6966247558594,
358
+ "loss": 0.0664,
359
+ "rewards/accuracies": 0.6937500238418579,
360
+ "rewards/chosen": -1.172304391860962,
361
+ "rewards/margins": 0.6343176364898682,
362
+ "rewards/rejected": -1.8066219091415405,
363
+ "step": 220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
364
  },
365
  {
366
  "epoch": 0.48,
367
+ "learning_rate": 3.096924887558854e-07,
368
+ "logits/chosen": -2.493622303009033,
369
+ "logits/rejected": -2.494441032409668,
370
+ "logps/chosen": -401.9507751464844,
371
+ "logps/rejected": -458.2899475097656,
372
+ "loss": 0.0671,
373
+ "rewards/accuracies": 0.6812499761581421,
374
+ "rewards/chosen": -1.201406717300415,
375
+ "rewards/margins": 0.4728333353996277,
376
+ "rewards/rejected": -1.6742403507232666,
377
+ "step": 230
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
378
  },
379
  {
380
  "epoch": 0.5,
381
+ "learning_rate": 2.9181224366319943e-07,
382
+ "logits/chosen": -2.4886069297790527,
383
+ "logits/rejected": -2.4639804363250732,
384
+ "logps/chosen": -384.59649658203125,
385
+ "logps/rejected": -447.69488525390625,
386
+ "loss": 0.0661,
387
+ "rewards/accuracies": 0.699999988079071,
388
+ "rewards/chosen": -1.3089364767074585,
389
+ "rewards/margins": 0.5827382802963257,
390
+ "rewards/rejected": -1.8916747570037842,
391
+ "step": 240
 
 
 
 
 
 
 
 
 
 
 
 
 
 
392
  },
393
  {
394
  "epoch": 0.52,
395
+ "learning_rate": 2.7370891215954565e-07,
396
+ "logits/chosen": -2.474219799041748,
397
+ "logits/rejected": -2.432372570037842,
398
+ "logps/chosen": -391.3350830078125,
399
+ "logps/rejected": -424.7400817871094,
400
+ "loss": 0.0617,
401
+ "rewards/accuracies": 0.71875,
402
+ "rewards/chosen": -1.3240981101989746,
403
+ "rewards/margins": 0.569055438041687,
404
+ "rewards/rejected": -1.8931537866592407,
405
+ "step": 250
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
406
  },
407
  {
408
  "epoch": 0.54,
409
+ "learning_rate": 2.55479083351317e-07,
410
+ "logits/chosen": -2.4577202796936035,
411
+ "logits/rejected": -2.4588468074798584,
412
+ "logps/chosen": -388.98602294921875,
413
+ "logps/rejected": -418.14697265625,
414
+ "loss": 0.0678,
415
+ "rewards/accuracies": 0.625,
416
+ "rewards/chosen": -1.2037723064422607,
417
+ "rewards/margins": 0.49300527572631836,
418
+ "rewards/rejected": -1.69677734375,
419
+ "step": 260
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
420
  },
421
  {
422
  "epoch": 0.56,
423
+ "learning_rate": 2.3722002126275822e-07,
424
+ "logits/chosen": -2.416320562362671,
425
+ "logits/rejected": -2.4132323265075684,
426
+ "logps/chosen": -406.05108642578125,
427
+ "logps/rejected": -446.5943908691406,
428
+ "loss": 0.0606,
429
+ "rewards/accuracies": 0.7749999761581421,
430
+ "rewards/chosen": -1.3119760751724243,
431
+ "rewards/margins": 0.5983575582504272,
432
+ "rewards/rejected": -1.9103336334228516,
433
+ "step": 270
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
434
  },
435
  {
436
  "epoch": 0.59,
437
+ "learning_rate": 2.19029145890313e-07,
438
+ "logits/chosen": -2.4188361167907715,
439
+ "logits/rejected": -2.3941617012023926,
440
+ "logps/chosen": -375.2843933105469,
441
+ "logps/rejected": -415.2039489746094,
442
+ "loss": 0.0647,
443
+ "rewards/accuracies": 0.699999988079071,
444
+ "rewards/chosen": -1.3387877941131592,
445
+ "rewards/margins": 0.5111810564994812,
446
+ "rewards/rejected": -1.849968671798706,
447
+ "step": 280
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
448
  },
449
  {
450
  "epoch": 0.61,
451
+ "learning_rate": 2.0100351342479216e-07,
452
+ "logits/chosen": -2.3956875801086426,
453
+ "logits/rejected": -2.3585715293884277,
454
+ "logps/chosen": -364.3868713378906,
455
+ "logps/rejected": -407.68438720703125,
456
+ "loss": 0.0634,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
457
  "rewards/accuracies": 0.699999988079071,
458
+ "rewards/chosen": -1.177756667137146,
459
+ "rewards/margins": 0.5566600561141968,
460
+ "rewards/rejected": -1.7344166040420532,
461
+ "step": 290
462
  },
463
  {
464
  "epoch": 0.63,
465
+ "learning_rate": 1.8323929841460178e-07,
466
+ "logits/chosen": -2.401419162750244,
467
+ "logits/rejected": -2.3725171089172363,
468
+ "logps/chosen": -391.79034423828125,
469
+ "logps/rejected": -445.4104919433594,
470
+ "loss": 0.0572,
471
+ "rewards/accuracies": 0.6625000238418579,
472
+ "rewards/chosen": -1.3552172183990479,
473
+ "rewards/margins": 0.5253753662109375,
474
+ "rewards/rejected": -1.880592703819275,
475
+ "step": 300
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
476
  },
477
  {
478
+ "epoch": 0.63,
479
+ "eval_logits/chosen": -2.419144868850708,
480
+ "eval_logits/rejected": -2.405817985534668,
481
+ "eval_logps/chosen": -393.69818115234375,
482
+ "eval_logps/rejected": -462.2751770019531,
483
+ "eval_loss": 0.0587710402905941,
484
+ "eval_rewards/accuracies": 0.7265625,
485
+ "eval_rewards/chosen": -1.3665854930877686,
486
+ "eval_rewards/margins": 0.6826332211494446,
487
+ "eval_rewards/rejected": -2.0492186546325684,
488
+ "eval_runtime": 53.6424,
489
+ "eval_samples_per_second": 37.284,
490
+ "eval_steps_per_second": 0.597,
491
+ "step": 300
492
  },
493
  {
494
  "epoch": 0.65,
495
+ "learning_rate": 1.6583128063291573e-07,
496
+ "logits/chosen": -2.3361194133758545,
497
+ "logits/rejected": -2.2814993858337402,
498
+ "logps/chosen": -412.4520568847656,
499
+ "logps/rejected": -449.22845458984375,
500
+ "loss": 0.0604,
501
+ "rewards/accuracies": 0.78125,
502
+ "rewards/chosen": -1.3927319049835205,
503
+ "rewards/margins": 0.6278494000434875,
504
+ "rewards/rejected": -2.0205814838409424,
505
+ "step": 310
 
 
 
 
 
 
 
 
 
 
 
 
 
 
506
  },
507
  {
508
  "epoch": 0.67,
509
+ "learning_rate": 1.488723393865766e-07,
510
+ "logits/chosen": -2.3553965091705322,
511
+ "logits/rejected": -2.3439528942108154,
512
+ "logps/chosen": -436.58782958984375,
513
+ "logps/rejected": -426.3726501464844,
514
+ "loss": 0.0574,
515
+ "rewards/accuracies": 0.699999988079071,
516
+ "rewards/chosen": -1.402055263519287,
517
+ "rewards/margins": 0.5076062083244324,
518
+ "rewards/rejected": -1.9096615314483643,
519
+ "step": 320
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
520
  },
521
  {
522
  "epoch": 0.69,
523
+ "learning_rate": 1.3245295796480788e-07,
524
+ "logits/chosen": -2.379216432571411,
525
+ "logits/rejected": -2.3494679927825928,
526
+ "logps/chosen": -380.96612548828125,
527
+ "logps/rejected": -415.45391845703125,
528
+ "loss": 0.0577,
529
+ "rewards/accuracies": 0.7437499761581421,
530
+ "rewards/chosen": -1.2875292301177979,
531
+ "rewards/margins": 0.5307010412216187,
532
+ "rewards/rejected": -1.8182302713394165,
533
+ "step": 330
 
 
 
 
 
 
 
 
 
 
 
 
 
 
534
  },
535
  {
536
  "epoch": 0.71,
537
+ "learning_rate": 1.1666074087171627e-07,
538
+ "logits/chosen": -2.319523334503174,
539
+ "logits/rejected": -2.3065667152404785,
540
+ "logps/chosen": -385.5484313964844,
541
+ "logps/rejected": -441.82684326171875,
542
+ "loss": 0.064,
543
+ "rewards/accuracies": 0.706250011920929,
544
+ "rewards/chosen": -1.3430511951446533,
545
+ "rewards/margins": 0.655988872051239,
546
+ "rewards/rejected": -1.999039888381958,
547
+ "step": 340
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
548
  },
549
  {
550
  "epoch": 0.73,
551
+ "learning_rate": 1.0157994641835734e-07,
552
+ "logits/chosen": -2.3364834785461426,
553
+ "logits/rejected": -2.310567617416382,
554
+ "logps/chosen": -367.62396240234375,
555
+ "logps/rejected": -447.74542236328125,
556
+ "loss": 0.0537,
557
  "rewards/accuracies": 0.706250011920929,
558
+ "rewards/chosen": -1.4167375564575195,
559
+ "rewards/margins": 0.653745174407959,
560
+ "rewards/rejected": -2.0704827308654785,
561
+ "step": 350
 
 
 
 
 
 
 
 
 
 
 
 
 
 
562
  },
563
  {
564
  "epoch": 0.75,
565
+ "learning_rate": 8.729103716819111e-08,
566
+ "logits/chosen": -2.323378801345825,
567
+ "logits/rejected": -2.2844491004943848,
568
+ "logps/chosen": -413.23388671875,
569
+ "logps/rejected": -452.17633056640625,
570
+ "loss": 0.0532,
571
+ "rewards/accuracies": 0.6937500238418579,
572
+ "rewards/chosen": -1.4596569538116455,
573
+ "rewards/margins": 0.6687207221984863,
574
+ "rewards/rejected": -2.128377914428711,
575
+ "step": 360
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
576
  },
577
  {
578
  "epoch": 0.77,
579
+ "learning_rate": 7.387025063449081e-08,
580
+ "logits/chosen": -2.297429323196411,
581
+ "logits/rejected": -2.30240797996521,
582
+ "logps/chosen": -398.3007507324219,
583
+ "logps/rejected": -471.49725341796875,
584
+ "loss": 0.0518,
585
+ "rewards/accuracies": 0.7124999761581421,
586
+ "rewards/chosen": -1.4305323362350464,
587
+ "rewards/margins": 0.74176025390625,
588
+ "rewards/rejected": -2.172292709350586,
589
+ "step": 370
 
 
 
 
 
 
 
 
 
 
 
 
 
 
590
  },
591
  {
592
  "epoch": 0.79,
593
+ "learning_rate": 6.138919252022435e-08,
594
+ "logits/chosen": -2.3351542949676514,
595
+ "logits/rejected": -2.326129198074341,
596
+ "logps/chosen": -421.6412048339844,
597
+ "logps/rejected": -450.1289978027344,
598
+ "loss": 0.052,
599
  "rewards/accuracies": 0.71875,
600
+ "rewards/chosen": -1.5581185817718506,
601
+ "rewards/margins": 0.57490473985672,
602
+ "rewards/rejected": -2.133023738861084,
603
+ "step": 380
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
604
  },
605
  {
606
  "epoch": 0.82,
607
+ "learning_rate": 4.991445467064689e-08,
608
+ "logits/chosen": -2.321873426437378,
609
+ "logits/rejected": -2.3132846355438232,
610
+ "logps/chosen": -397.6571044921875,
611
+ "logps/rejected": -462.237060546875,
612
+ "loss": 0.0501,
613
  "rewards/accuracies": 0.7437499761581421,
614
+ "rewards/chosen": -1.4045766592025757,
615
+ "rewards/margins": 0.7981351613998413,
616
+ "rewards/rejected": -2.202711820602417,
617
+ "step": 390
 
 
 
 
 
 
 
 
 
 
 
 
 
 
618
  },
619
  {
620
  "epoch": 0.84,
621
+ "learning_rate": 3.9507259776993954e-08,
622
+ "logits/chosen": -2.355191707611084,
623
+ "logits/rejected": -2.350708484649658,
624
+ "logps/chosen": -452.8954162597656,
625
+ "logps/rejected": -484.632080078125,
626
+ "loss": 0.0607,
627
+ "rewards/accuracies": 0.6937500238418579,
628
+ "rewards/chosen": -1.527354121208191,
629
+ "rewards/margins": 0.5898515582084656,
630
+ "rewards/rejected": -2.117205858230591,
631
+ "step": 400
632
  },
633
  {
634
  "epoch": 0.84,
635
+ "eval_logits/chosen": -2.374659299850464,
636
+ "eval_logits/rejected": -2.3613595962524414,
637
+ "eval_logps/chosen": -403.8057861328125,
638
+ "eval_logps/rejected": -478.2771911621094,
639
+ "eval_loss": 0.056821852922439575,
640
+ "eval_rewards/accuracies": 0.71875,
641
+ "eval_rewards/chosen": -1.467661738395691,
642
+ "eval_rewards/margins": 0.7415772676467896,
643
+ "eval_rewards/rejected": -2.2092390060424805,
644
+ "eval_runtime": 53.5993,
645
+ "eval_samples_per_second": 37.314,
646
+ "eval_steps_per_second": 0.597,
647
+ "step": 400
 
 
 
 
 
 
 
 
 
 
 
 
648
  },
649
  {
650
  "epoch": 0.86,
651
+ "learning_rate": 3.022313472693447e-08,
652
+ "logits/chosen": -2.3436548709869385,
653
+ "logits/rejected": -2.345236301422119,
654
+ "logps/chosen": -405.0478820800781,
655
+ "logps/rejected": -429.0362854003906,
656
+ "loss": 0.0533,
657
+ "rewards/accuracies": 0.612500011920929,
658
+ "rewards/chosen": -1.5035853385925293,
659
+ "rewards/margins": 0.4621230959892273,
660
+ "rewards/rejected": -1.9657083749771118,
661
+ "step": 410
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
662
  },
663
  {
664
  "epoch": 0.88,
665
+ "learning_rate": 2.2111614344599684e-08,
666
+ "logits/chosen": -2.3887081146240234,
667
+ "logits/rejected": -2.3521924018859863,
668
+ "logps/chosen": -422.1944885253906,
669
+ "logps/rejected": -470.697509765625,
670
+ "loss": 0.0544,
671
+ "rewards/accuracies": 0.75,
672
+ "rewards/chosen": -1.4449588060379028,
673
+ "rewards/margins": 0.7653876543045044,
674
+ "rewards/rejected": -2.2103464603424072,
675
+ "step": 420
 
 
 
 
 
 
 
 
 
 
 
 
 
 
676
  },
677
  {
678
  "epoch": 0.9,
679
+ "learning_rate": 1.521597710086439e-08,
680
+ "logits/chosen": -2.3927550315856934,
681
+ "logits/rejected": -2.3677451610565186,
682
+ "logps/chosen": -415.412109375,
683
+ "logps/rejected": -444.7759704589844,
684
+ "loss": 0.0595,
685
+ "rewards/accuracies": 0.699999988079071,
686
+ "rewards/chosen": -1.404589295387268,
687
+ "rewards/margins": 0.5866460800170898,
688
+ "rewards/rejected": -1.991235375404358,
689
+ "step": 430
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
690
  },
691
  {
692
  "epoch": 0.92,
693
+ "learning_rate": 9.57301420397924e-09,
694
+ "logits/chosen": -2.3254761695861816,
695
+ "logits/rejected": -2.329185724258423,
696
+ "logps/chosen": -426.0096740722656,
697
+ "logps/rejected": -444.96551513671875,
698
+ "loss": 0.0581,
699
  "rewards/accuracies": 0.6937500238418579,
700
+ "rewards/chosen": -1.4774408340454102,
701
+ "rewards/margins": 0.5694506168365479,
702
+ "rewards/rejected": -2.046891450881958,
703
+ "step": 440
 
 
 
 
 
 
 
 
 
 
 
 
 
 
704
  },
705
  {
706
  "epoch": 0.94,
707
+ "learning_rate": 5.212833302556258e-09,
708
+ "logits/chosen": -2.338719367980957,
709
+ "logits/rejected": -2.320690631866455,
710
+ "logps/chosen": -425.0148010253906,
711
+ "logps/rejected": -486.3421325683594,
712
+ "loss": 0.0554,
713
+ "rewards/accuracies": 0.668749988079071,
714
+ "rewards/chosen": -1.4839718341827393,
715
+ "rewards/margins": 0.7307707071304321,
716
+ "rewards/rejected": -2.214742660522461,
717
+ "step": 450
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
718
  },
719
  {
720
  "epoch": 0.96,
721
+ "learning_rate": 2.158697848236607e-09,
722
+ "logits/chosen": -2.314551830291748,
723
+ "logits/rejected": -2.3122870922088623,
724
+ "logps/chosen": -407.9598693847656,
725
+ "logps/rejected": -448.6959533691406,
726
+ "loss": 0.0527,
727
+ "rewards/accuracies": 0.7124999761581421,
728
+ "rewards/chosen": -1.426579236984253,
729
+ "rewards/margins": 0.5909186601638794,
730
+ "rewards/rejected": -2.0174977779388428,
731
+ "step": 460
 
 
 
 
 
 
 
 
 
 
 
 
 
 
732
  },
733
  {
734
  "epoch": 0.98,
735
+ "learning_rate": 4.269029751107489e-10,
736
+ "logits/chosen": -2.3605387210845947,
737
+ "logits/rejected": -2.3362832069396973,
738
+ "logps/chosen": -426.3868713378906,
739
+ "logps/rejected": -446.27276611328125,
740
+ "loss": 0.0562,
741
+ "rewards/accuracies": 0.699999988079071,
742
+ "rewards/chosen": -1.4052413702011108,
743
+ "rewards/margins": 0.7002570033073425,
744
+ "rewards/rejected": -2.1054985523223877,
745
+ "step": 470
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
746
  },
747
  {
748
  "epoch": 1.0,
749
+ "step": 478,
750
  "total_flos": 0.0,
751
+ "train_loss": 0.08938389025223305,
752
+ "train_runtime": 4400.7374,
753
+ "train_samples_per_second": 13.892,
754
  "train_steps_per_second": 0.109
755
  }
756
  ],
757
  "logging_steps": 10,
758
+ "max_steps": 478,
759
  "num_train_epochs": 1,
760
  "save_steps": 100,
761
  "total_flos": 0.0,