Hugging Face's logo

wzhouad
/
zephyr-7b-dpo-full

Text Generation
Transformers
Safetensors
llama
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wzhouad commited on
Commit
b8685c5
•
1 Parent(s): a6f0bf6

Model save

Browse files
Files changed (8) hide show
  1. README.md +14 -22
  2. all_results.json +5 -5
  3. model-00001-of-00003.safetensors +1 -1
  4. model-00002-of-00003.safetensors +1 -1
  5. model-00003-of-00003.safetensors +1 -1
  6. train_results.json +5 -5
  7. trainer_state.json +578 -1798
  8. training_args.bin +1 -1
README.md CHANGED
@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
- - Loss: 0.5440
21
- - Rewards/chosen: -2.2940
22
- - Rewards/rejected: -3.0054
23
- - Rewards/accuracies: 0.7090
24
- - Rewards/margins: 0.7114
25
- - Logps/rejected: -451.6765
26
- - Logps/chosen: -373.9785
27
- - Logits/rejected: 0.3244
28
- - Logits/chosen: 0.0742
29
 
30
  ## Model description
31
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
47
  - learning_rate: 5e-07
48
  - train_batch_size: 8
49
  - eval_batch_size: 8
50
- - seed: 42
51
  - distributed_type: multi-GPU
52
  - num_devices: 8
53
  - gradient_accumulation_steps: 2
@@ -62,18 +62,10 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 0.6789 | 0.08 | 100 | 0.6770 | -0.1062 | -0.1422 | 0.5914 | 0.0360 | -165.3552 | -155.1927 | -2.7255 | -2.7337 |
66
- | 0.6062 | 0.16 | 200 | 0.6079 | -1.0212 | -1.3873 | 0.6670 | 0.3660 | -289.8622 | -246.6971 | -2.3696 | -2.3856 |
67
- | 0.5965 | 0.24 | 300 | 0.5907 | -1.3779 | -1.8008 | 0.6623 | 0.4229 | -331.2100 | -282.3621 | -2.2450 | -2.2656 |
68
- | 0.5729 | 0.32 | 400 | 0.5711 | -1.6763 | -2.2404 | 0.6828 | 0.5640 | -375.1720 | -312.2064 | -1.2920 | -1.3760 |
69
- | 0.5645 | 0.4 | 500 | 0.5639 | -2.0721 | -2.6869 | 0.6987 | 0.6147 | -419.8194 | -351.7883 | -0.6091 | -0.7860 |
70
- | 0.5513 | 0.48 | 600 | 0.5582 | -2.9237 | -3.5389 | 0.7108 | 0.6152 | -505.0223 | -436.9386 | 0.1224 | -0.1054 |
71
- | 0.5571 | 0.56 | 700 | 0.5559 | -2.7971 | -3.5456 | 0.7043 | 0.7485 | -505.6961 | -424.2823 | 0.2980 | 0.0356 |
72
- | 0.5609 | 0.64 | 800 | 0.5469 | -2.4314 | -3.0831 | 0.7108 | 0.6517 | -459.4439 | -387.7092 | 0.1922 | -0.0312 |
73
- | 0.5514 | 0.72 | 900 | 0.5474 | -2.4774 | -3.2082 | 0.6996 | 0.7308 | -471.9533 | -392.3096 | 0.5382 | 0.2860 |
74
- | 0.527 | 0.8 | 1000 | 0.5454 | -2.5040 | -3.2071 | 0.7080 | 0.7031 | -471.8454 | -394.9711 | 0.6372 | 0.3871 |
75
- | 0.5487 | 0.88 | 1100 | 0.5444 | -2.2851 | -2.9963 | 0.7090 | 0.7112 | -450.7599 | -373.0831 | 0.4336 | 0.1858 |
76
- | 0.5483 | 0.96 | 1200 | 0.5440 | -2.2940 | -3.0054 | 0.7090 | 0.7114 | -451.6765 | -373.9785 | 0.3244 | 0.0742 |
77
 
78
 
79
  ### Framework versions
 
17
 
18
  This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
+ - Loss: 0.0510
21
+ - Rewards/chosen: -1.2696
22
+ - Rewards/rejected: -1.9807
23
+ - Rewards/accuracies: 0.7227
24
+ - Rewards/margins: 0.7111
25
+ - Logps/rejected: -455.4246
26
+ - Logps/chosen: -384.0027
27
+ - Logits/rejected: -2.3534
28
+ - Logits/chosen: -2.3725
29
 
30
  ## Model description
31
 
 
47
  - learning_rate: 5e-07
48
  - train_batch_size: 8
49
  - eval_batch_size: 8
50
+ - seed: 5
51
  - distributed_type: multi-GPU
52
  - num_devices: 8
53
  - gradient_accumulation_steps: 2
 
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 0.1041 | 0.21 | 100 | 0.1028 | -0.4071 | -0.7890 | 0.7383 | 0.3819 | -336.2553 | -297.7511 | -2.7675 | -2.7878 |
66
+ | 0.0503 | 0.42 | 200 | 0.0505 | -1.2848 | -1.8438 | 0.7070 | 0.5590 | -441.7352 | -385.5240 | -2.5374 | -2.5561 |
67
+ | 0.0486 | 0.63 | 300 | 0.0545 | -1.2021 | -1.8852 | 0.7266 | 0.6832 | -445.8765 | -377.2457 | -2.3894 | -2.4070 |
68
+ | 0.0435 | 0.84 | 400 | 0.0510 | -1.2696 | -1.9807 | 0.7227 | 0.7111 | -455.4246 | -384.0027 | -2.3534 | -2.3725 |
 
 
 
 
 
 
 
 
69
 
70
 
71
  ### Framework versions
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.5712926928784438,
4
- "train_runtime": 11451.0976,
5
- "train_samples": 160800,
6
- "train_samples_per_second": 14.042,
7
- "train_steps_per_second": 0.11
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.07493206359851311,
4
+ "train_runtime": 4367.2364,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 13.998,
7
+ "train_steps_per_second": 0.109
8
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:18e1cec63bd40f863dc594533ae9ac02d7bcdd4f57a17c1ef5d63193122a0814
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5beb9bb6eebd3480becc778a231d13bf7e92ffeaf3581085810b65282e93015b
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:565d4244afeda54e7f62be9e162a16c6892085c081422f02c7a001ecce587eb6
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6cb07dc736dfdb0d4c456dbf5a0d8eb6a110689251795b8807c08792ea10ea5
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0debf1533b3a9f2ffea91ddec7f947ba3d1c43476aedcef3273235a227bb4ce5
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3fd0edf230898ac29210705331ab88d51e0298ee569f4884284dc824fd1d363
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.5712926928784438,
4
- "train_runtime": 11451.0976,
5
- "train_samples": 160800,
6
- "train_samples_per_second": 14.042,
7
- "train_steps_per_second": 0.11
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.07493206359851311,
4
+ "train_runtime": 4367.2364,
5
+ "train_samples": 61134,
6
+ "train_samples_per_second": 13.998,
7
+ "train_steps_per_second": 0.109
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.9996020692399522,
5
  "eval_steps": 100,
6
- "global_step": 1256,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
- "learning_rate": 3.968253968253968e-09,
14
- "logits/chosen": -2.7193620204925537,
15
- "logits/rejected": -2.698728084564209,
16
- "logps/chosen": -182.0961456298828,
17
- "logps/rejected": -172.47128295898438,
18
- "loss": 0.6931,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -23,1959 +23,739 @@
23
  "step": 1
24
  },
25
  {
26
- "epoch": 0.01,
27
- "learning_rate": 3.968253968253968e-08,
28
- "logits/chosen": -2.7041964530944824,
29
- "logits/rejected": -2.6794540882110596,
30
- "logps/chosen": -162.45831298828125,
31
- "logps/rejected": -140.5693359375,
32
- "loss": 0.6931,
33
- "rewards/accuracies": 0.5486111044883728,
34
- "rewards/chosen": 0.00032037965138442814,
35
- "rewards/margins": 0.0004935775068588555,
36
- "rewards/rejected": -0.00017319784092251211,
37
  "step": 10
38
  },
39
  {
40
- "epoch": 0.02,
41
- "learning_rate": 7.936507936507936e-08,
42
- "logits/chosen": -2.7177577018737793,
43
- "logits/rejected": -2.7136425971984863,
44
- "logps/chosen": -134.47242736816406,
45
- "logps/rejected": -143.55604553222656,
46
- "loss": 0.6931,
47
- "rewards/accuracies": 0.550000011920929,
48
- "rewards/chosen": 8.780837379163131e-05,
49
- "rewards/margins": 0.00010721785656642169,
50
- "rewards/rejected": -1.940951551659964e-05,
51
  "step": 20
52
  },
53
  {
54
- "epoch": 0.02,
55
- "learning_rate": 1.1904761904761903e-07,
56
- "logits/chosen": -2.6898293495178223,
57
- "logits/rejected": -2.676154613494873,
58
- "logps/chosen": -140.94692993164062,
59
- "logps/rejected": -136.50369262695312,
60
- "loss": 0.6931,
61
- "rewards/accuracies": 0.4375,
62
- "rewards/chosen": 0.0005466601578518748,
63
- "rewards/margins": -0.00021456097601912916,
64
- "rewards/rejected": 0.0007612211629748344,
65
  "step": 30
66
  },
67
  {
68
- "epoch": 0.03,
69
- "learning_rate": 1.5873015873015872e-07,
70
- "logits/chosen": -2.6958394050598145,
71
- "logits/rejected": -2.686532974243164,
72
- "logps/chosen": -134.98963928222656,
73
- "logps/rejected": -144.46652221679688,
74
- "loss": 0.6928,
75
- "rewards/accuracies": 0.581250011920929,
76
- "rewards/chosen": 0.0015748919686302543,
77
- "rewards/margins": 0.0009769219905138016,
78
- "rewards/rejected": 0.0005979698617011309,
79
  "step": 40
80
  },
81
  {
82
- "epoch": 0.04,
83
- "learning_rate": 1.984126984126984e-07,
84
- "logits/chosen": -2.7042899131774902,
85
- "logits/rejected": -2.6861345767974854,
86
- "logps/chosen": -149.71768188476562,
87
- "logps/rejected": -145.0757293701172,
88
- "loss": 0.6921,
89
- "rewards/accuracies": 0.6000000238418579,
90
- "rewards/chosen": 0.005199921317398548,
91
- "rewards/margins": 0.0022330707870423794,
92
- "rewards/rejected": 0.0029668500646948814,
93
  "step": 50
94
  },
95
  {
96
- "epoch": 0.05,
97
- "learning_rate": 2.3809523809523806e-07,
98
- "logits/chosen": -2.705153703689575,
99
- "logits/rejected": -2.685439348220825,
100
- "logps/chosen": -154.3783416748047,
101
- "logps/rejected": -151.54519653320312,
102
- "loss": 0.6912,
103
- "rewards/accuracies": 0.543749988079071,
104
- "rewards/chosen": 0.00569504126906395,
105
- "rewards/margins": 0.0022000311873853207,
106
- "rewards/rejected": 0.003495010081678629,
107
  "step": 60
108
  },
109
  {
110
- "epoch": 0.06,
111
- "learning_rate": 2.7777777777777776e-07,
112
- "logits/chosen": -2.7017154693603516,
113
- "logits/rejected": -2.6924962997436523,
114
- "logps/chosen": -146.3284149169922,
115
- "logps/rejected": -138.79405212402344,
116
- "loss": 0.6885,
117
- "rewards/accuracies": 0.550000011920929,
118
- "rewards/chosen": 0.010588793084025383,
119
- "rewards/margins": 0.010192448273301125,
120
- "rewards/rejected": 0.00039634370477870107,
121
  "step": 70
122
  },
123
  {
124
- "epoch": 0.06,
125
- "learning_rate": 3.1746031746031743e-07,
126
- "logits/chosen": -2.7155232429504395,
127
- "logits/rejected": -2.696071147918701,
128
- "logps/chosen": -141.80067443847656,
129
- "logps/rejected": -147.0068817138672,
130
- "loss": 0.6867,
131
- "rewards/accuracies": 0.5562499761581421,
132
- "rewards/chosen": 0.0049073463305830956,
133
- "rewards/margins": 0.013599385507404804,
134
- "rewards/rejected": -0.008692039176821709,
135
  "step": 80
136
  },
137
  {
138
- "epoch": 0.07,
139
- "learning_rate": 3.5714285714285716e-07,
140
- "logits/chosen": -2.7175304889678955,
141
- "logits/rejected": -2.7080624103546143,
142
- "logps/chosen": -153.12509155273438,
143
- "logps/rejected": -146.53590393066406,
144
- "loss": 0.6847,
145
- "rewards/accuracies": 0.5874999761581421,
146
- "rewards/chosen": -0.028871387243270874,
147
- "rewards/margins": 0.017175236716866493,
148
- "rewards/rejected": -0.046046625822782516,
149
  "step": 90
150
  },
151
  {
152
- "epoch": 0.08,
153
- "learning_rate": 3.968253968253968e-07,
154
- "logits/chosen": -2.7524733543395996,
155
- "logits/rejected": -2.7452526092529297,
156
- "logps/chosen": -163.88070678710938,
157
- "logps/rejected": -163.61032104492188,
158
- "loss": 0.6789,
159
- "rewards/accuracies": 0.612500011920929,
160
- "rewards/chosen": -0.0662173330783844,
161
- "rewards/margins": 0.02977912127971649,
162
- "rewards/rejected": -0.09599645435810089,
163
  "step": 100
164
  },
165
  {
166
- "epoch": 0.08,
167
- "eval_logits/chosen": -2.7336502075195312,
168
- "eval_logits/rejected": -2.7255024909973145,
169
- "eval_logps/chosen": -155.19271850585938,
170
- "eval_logps/rejected": -165.35523986816406,
171
- "eval_loss": 0.6769910454750061,
172
- "eval_rewards/accuracies": 0.5914179086685181,
173
- "eval_rewards/chosen": -0.10619194805622101,
174
- "eval_rewards/margins": 0.03601696714758873,
175
- "eval_rewards/rejected": -0.14220890402793884,
176
- "eval_runtime": 184.217,
177
- "eval_samples_per_second": 46.424,
178
- "eval_steps_per_second": 0.727,
179
  "step": 100
180
  },
181
  {
182
- "epoch": 0.09,
183
- "learning_rate": 4.365079365079365e-07,
184
- "logits/chosen": -2.738532543182373,
185
- "logits/rejected": -2.7273170948028564,
186
- "logps/chosen": -164.2928009033203,
187
- "logps/rejected": -160.19398498535156,
188
- "loss": 0.6738,
189
- "rewards/accuracies": 0.5625,
190
- "rewards/chosen": -0.16211798787117004,
191
- "rewards/margins": 0.03163355216383934,
192
- "rewards/rejected": -0.19375154376029968,
193
  "step": 110
194
  },
195
  {
196
- "epoch": 0.1,
197
- "learning_rate": 4.761904761904761e-07,
198
- "logits/chosen": -2.7289297580718994,
199
- "logits/rejected": -2.705962657928467,
200
- "logps/chosen": -196.69662475585938,
201
- "logps/rejected": -197.2833251953125,
202
- "loss": 0.661,
203
- "rewards/accuracies": 0.59375,
204
- "rewards/chosen": -0.2917623221874237,
205
- "rewards/margins": 0.08966299891471863,
206
- "rewards/rejected": -0.38142532110214233,
207
  "step": 120
208
  },
209
  {
210
- "epoch": 0.1,
211
- "learning_rate": 4.999845414634076e-07,
212
- "logits/chosen": -2.658005475997925,
213
- "logits/rejected": -2.6317684650421143,
214
- "logps/chosen": -187.4532928466797,
215
- "logps/rejected": -188.37689208984375,
216
- "loss": 0.6542,
217
- "rewards/accuracies": 0.6312500238418579,
218
- "rewards/chosen": -0.3956056833267212,
219
- "rewards/margins": 0.12092368304729462,
220
- "rewards/rejected": -0.5165294408798218,
221
  "step": 130
222
  },
223
  {
224
- "epoch": 0.11,
225
- "learning_rate": 4.998106548810311e-07,
226
- "logits/chosen": -2.6906683444976807,
227
- "logits/rejected": -2.6913747787475586,
228
- "logps/chosen": -199.67568969726562,
229
- "logps/rejected": -253.02487182617188,
230
- "loss": 0.6171,
231
- "rewards/accuracies": 0.7124999761581421,
232
- "rewards/chosen": -0.4227059781551361,
233
- "rewards/margins": 0.27536457777023315,
234
- "rewards/rejected": -0.6980706453323364,
235
  "step": 140
236
  },
237
  {
238
- "epoch": 0.12,
239
- "learning_rate": 4.994436933879359e-07,
240
- "logits/chosen": -2.6662166118621826,
241
- "logits/rejected": -2.644784927368164,
242
- "logps/chosen": -197.07180786132812,
243
- "logps/rejected": -198.4012908935547,
244
- "loss": 0.6395,
245
- "rewards/accuracies": 0.6187499761581421,
246
- "rewards/chosen": -0.3597154915332794,
247
- "rewards/margins": 0.13716872036457062,
248
- "rewards/rejected": -0.49688419699668884,
249
  "step": 150
250
  },
251
  {
252
- "epoch": 0.13,
253
- "learning_rate": 4.988839406031596e-07,
254
- "logits/chosen": -2.647681474685669,
255
- "logits/rejected": -2.6395888328552246,
256
- "logps/chosen": -182.04420471191406,
257
- "logps/rejected": -206.59780883789062,
258
- "loss": 0.629,
259
  "rewards/accuracies": 0.6499999761581421,
260
- "rewards/chosen": -0.3573477864265442,
261
- "rewards/margins": 0.2222837507724762,
262
- "rewards/rejected": -0.579631507396698,
263
  "step": 160
264
  },
265
  {
266
- "epoch": 0.14,
267
- "learning_rate": 4.981318291512395e-07,
268
- "logits/chosen": -2.619232654571533,
269
- "logits/rejected": -2.598362684249878,
270
- "logps/chosen": -227.0933380126953,
271
- "logps/rejected": -230.9747772216797,
272
- "loss": 0.6242,
273
- "rewards/accuracies": 0.637499988079071,
274
- "rewards/chosen": -0.7434185743331909,
275
- "rewards/margins": 0.21749505400657654,
276
- "rewards/rejected": -0.9609137773513794,
277
  "step": 170
278
  },
279
  {
280
- "epoch": 0.14,
281
- "learning_rate": 4.971879403278432e-07,
282
- "logits/chosen": -2.5654754638671875,
283
- "logits/rejected": -2.5364232063293457,
284
- "logps/chosen": -241.6617431640625,
285
- "logps/rejected": -245.66268920898438,
286
- "loss": 0.6151,
287
- "rewards/accuracies": 0.6625000238418579,
288
- "rewards/chosen": -0.7342535257339478,
289
- "rewards/margins": 0.23685339093208313,
290
- "rewards/rejected": -0.9711068868637085,
291
  "step": 180
292
  },
293
  {
294
- "epoch": 0.15,
295
- "learning_rate": 4.960530036504941e-07,
296
- "logits/chosen": -2.5271048545837402,
297
- "logits/rejected": -2.486818790435791,
298
- "logps/chosen": -235.6089630126953,
299
- "logps/rejected": -251.17758178710938,
300
- "loss": 0.6215,
301
- "rewards/accuracies": 0.699999988079071,
302
- "rewards/chosen": -0.807177722454071,
303
- "rewards/margins": 0.28561535477638245,
304
- "rewards/rejected": -1.0927931070327759,
305
  "step": 190
306
  },
307
  {
308
- "epoch": 0.16,
309
- "learning_rate": 4.947278962947386e-07,
310
- "logits/chosen": -2.4217896461486816,
311
- "logits/rejected": -2.413295269012451,
312
- "logps/chosen": -251.0736083984375,
313
- "logps/rejected": -268.6098937988281,
314
- "loss": 0.6062,
315
- "rewards/accuracies": 0.6625000238418579,
316
- "rewards/chosen": -1.086307406425476,
317
- "rewards/margins": 0.24874301254749298,
318
- "rewards/rejected": -1.3350504636764526,
319
  "step": 200
320
  },
321
  {
322
- "epoch": 0.16,
323
- "eval_logits/chosen": -2.3855514526367188,
324
- "eval_logits/rejected": -2.369593858718872,
325
- "eval_logps/chosen": -246.6970672607422,
326
- "eval_logps/rejected": -289.8621826171875,
327
- "eval_loss": 0.6079375743865967,
328
- "eval_rewards/accuracies": 0.66697758436203,
329
- "eval_rewards/chosen": -1.021235704421997,
330
- "eval_rewards/margins": 0.3660426437854767,
331
- "eval_rewards/rejected": -1.3872781991958618,
332
- "eval_runtime": 184.1914,
333
- "eval_samples_per_second": 46.43,
334
- "eval_steps_per_second": 0.728,
335
  "step": 200
336
  },
337
  {
338
- "epoch": 0.17,
339
- "learning_rate": 4.932136424161899e-07,
340
- "logits/chosen": -2.3366785049438477,
341
- "logits/rejected": -2.3228511810302734,
342
- "logps/chosen": -266.292236328125,
343
- "logps/rejected": -300.22894287109375,
344
- "loss": 0.5893,
345
- "rewards/accuracies": 0.5874999761581421,
346
- "rewards/chosen": -1.2130026817321777,
347
- "rewards/margins": 0.3487839996814728,
348
- "rewards/rejected": -1.5617868900299072,
349
  "step": 210
350
  },
351
  {
352
- "epoch": 0.18,
353
- "learning_rate": 4.915114123589732e-07,
354
- "logits/chosen": -2.321228504180908,
355
- "logits/rejected": -2.3033699989318848,
356
- "logps/chosen": -336.34161376953125,
357
- "logps/rejected": -373.39935302734375,
358
- "loss": 0.612,
359
  "rewards/accuracies": 0.6812499761581421,
360
- "rewards/chosen": -1.9529145956039429,
361
- "rewards/margins": 0.2863468527793884,
362
- "rewards/rejected": -2.2392613887786865,
363
  "step": 220
364
  },
365
  {
366
- "epoch": 0.18,
367
- "learning_rate": 4.896225217511849e-07,
368
- "logits/chosen": -2.4310107231140137,
369
- "logits/rejected": -2.422048568725586,
370
- "logps/chosen": -291.1025695800781,
371
- "logps/rejected": -328.18963623046875,
372
- "loss": 0.6079,
373
- "rewards/accuracies": 0.6812499761581421,
374
- "rewards/chosen": -1.4314143657684326,
375
- "rewards/margins": 0.3364001214504242,
376
- "rewards/rejected": -1.7678143978118896,
377
  "step": 230
378
  },
379
  {
380
- "epoch": 0.19,
381
- "learning_rate": 4.875484304880629e-07,
382
- "logits/chosen": -2.3412394523620605,
383
- "logits/rejected": -2.309183120727539,
384
- "logps/chosen": -280.8785705566406,
385
- "logps/rejected": -308.54132080078125,
386
- "loss": 0.613,
387
- "rewards/accuracies": 0.612500011920929,
388
- "rewards/chosen": -1.309309482574463,
389
- "rewards/margins": 0.3731766939163208,
390
- "rewards/rejected": -1.6824861764907837,
391
  "step": 240
392
  },
393
  {
394
- "epoch": 0.2,
395
- "learning_rate": 4.852907416036558e-07,
396
- "logits/chosen": -2.415271282196045,
397
- "logits/rejected": -2.4072234630584717,
398
- "logps/chosen": -243.56332397460938,
399
- "logps/rejected": -298.7532043457031,
400
- "loss": 0.591,
401
- "rewards/accuracies": 0.737500011920929,
402
- "rewards/chosen": -0.967076301574707,
403
- "rewards/margins": 0.4581146240234375,
404
- "rewards/rejected": -1.4251911640167236,
405
  "step": 250
406
  },
407
  {
408
- "epoch": 0.21,
409
- "learning_rate": 4.828512000318616e-07,
410
- "logits/chosen": -2.3924427032470703,
411
- "logits/rejected": -2.3613152503967285,
412
- "logps/chosen": -266.86572265625,
413
- "logps/rejected": -304.2983093261719,
414
- "loss": 0.5986,
415
- "rewards/accuracies": 0.668749988079071,
416
- "rewards/chosen": -1.2214807271957397,
417
- "rewards/margins": 0.4553411602973938,
418
- "rewards/rejected": -1.6768219470977783,
419
  "step": 260
420
  },
421
  {
422
- "epoch": 0.21,
423
- "learning_rate": 4.802316912577946e-07,
424
- "logits/chosen": -2.4108529090881348,
425
- "logits/rejected": -2.3902478218078613,
426
- "logps/chosen": -252.7959442138672,
427
- "logps/rejected": -295.266357421875,
428
- "loss": 0.5917,
429
- "rewards/accuracies": 0.7250000238418579,
430
- "rewards/chosen": -1.0824626684188843,
431
- "rewards/margins": 0.39643940329551697,
432
- "rewards/rejected": -1.4789022207260132,
433
  "step": 270
434
  },
435
  {
436
- "epoch": 0.22,
437
- "learning_rate": 4.774342398605221e-07,
438
- "logits/chosen": -2.3505263328552246,
439
- "logits/rejected": -2.2942967414855957,
440
- "logps/chosen": -279.871337890625,
441
- "logps/rejected": -300.4220886230469,
442
- "loss": 0.5979,
443
- "rewards/accuracies": 0.6812499761581421,
444
- "rewards/chosen": -1.1787078380584717,
445
- "rewards/margins": 0.430286169052124,
446
- "rewards/rejected": -1.6089938879013062,
447
  "step": 280
448
  },
449
  {
450
- "epoch": 0.23,
451
- "learning_rate": 4.744610079482978e-07,
452
- "logits/chosen": -2.3269264698028564,
453
- "logits/rejected": -2.2910802364349365,
454
- "logps/chosen": -255.27706909179688,
455
- "logps/rejected": -281.60137939453125,
456
- "loss": 0.5853,
457
  "rewards/accuracies": 0.6875,
458
- "rewards/chosen": -1.1753785610198975,
459
- "rewards/margins": 0.3495523929595947,
460
- "rewards/rejected": -1.5249310731887817,
461
  "step": 290
462
  },
463
  {
464
- "epoch": 0.24,
465
- "learning_rate": 4.713142934875005e-07,
466
- "logits/chosen": -2.2868428230285645,
467
- "logits/rejected": -2.2631592750549316,
468
- "logps/chosen": -284.2200012207031,
469
- "logps/rejected": -322.45269775390625,
470
- "loss": 0.5965,
471
- "rewards/accuracies": 0.675000011920929,
472
- "rewards/chosen": -1.336501955986023,
473
- "rewards/margins": 0.3968800902366638,
474
- "rewards/rejected": -1.733382225036621,
475
  "step": 300
476
  },
477
  {
478
- "epoch": 0.24,
479
- "eval_logits/chosen": -2.265592098236084,
480
- "eval_logits/rejected": -2.244987964630127,
481
- "eval_logps/chosen": -282.3620910644531,
482
- "eval_logps/rejected": -331.2099609375,
483
- "eval_loss": 0.5907339453697205,
484
- "eval_rewards/accuracies": 0.6623134613037109,
485
- "eval_rewards/chosen": -1.3778856992721558,
486
- "eval_rewards/margins": 0.42287060618400574,
487
- "eval_rewards/rejected": -1.8007562160491943,
488
- "eval_runtime": 184.1249,
489
- "eval_samples_per_second": 46.447,
490
- "eval_steps_per_second": 0.728,
491
  "step": 300
492
  },
493
  {
494
- "epoch": 0.25,
495
- "learning_rate": 4.679965285265706e-07,
496
- "logits/chosen": -2.2354235649108887,
497
- "logits/rejected": -2.23685884475708,
498
- "logps/chosen": -277.09283447265625,
499
- "logps/rejected": -347.7145080566406,
500
- "loss": 0.5612,
501
  "rewards/accuracies": 0.6875,
502
- "rewards/chosen": -1.3514426946640015,
503
- "rewards/margins": 0.4907970428466797,
504
- "rewards/rejected": -1.8422397375106812,
505
  "step": 310
506
  },
507
- {
508
- "epoch": 0.25,
509
- "learning_rate": 4.64510277316316e-07,
510
- "logits/chosen": -2.2262344360351562,
511
- "logits/rejected": -2.226029634475708,
512
- "logps/chosen": -271.74212646484375,
513
- "logps/rejected": -332.5010986328125,
514
- "loss": 0.5903,
515
- "rewards/accuracies": 0.612500011920929,
516
- "rewards/chosen": -1.3337775468826294,
517
- "rewards/margins": 0.39512914419174194,
518
- "rewards/rejected": -1.7289068698883057,
519
- "step": 320
520
- },
521
- {
522
- "epoch": 0.26,
523
- "learning_rate": 4.6085823432804137e-07,
524
- "logits/chosen": -2.2451891899108887,
525
- "logits/rejected": -2.2502384185791016,
526
- "logps/chosen": -250.6347198486328,
527
- "logps/rejected": -333.8939208984375,
528
- "loss": 0.5722,
529
- "rewards/accuracies": 0.762499988079071,
530
- "rewards/chosen": -1.1326004266738892,
531
- "rewards/margins": 0.5066065192222595,
532
- "rewards/rejected": -1.639206886291504,
533
- "step": 330
534
- },
535
- {
536
- "epoch": 0.27,
537
- "learning_rate": 4.570432221710314e-07,
538
- "logits/chosen": -2.0656931400299072,
539
- "logits/rejected": -2.0213730335235596,
540
- "logps/chosen": -318.232177734375,
541
- "logps/rejected": -369.13311767578125,
542
- "loss": 0.5766,
543
- "rewards/accuracies": 0.7250000238418579,
544
- "rewards/chosen": -1.584176778793335,
545
- "rewards/margins": 0.5901076197624207,
546
- "rewards/rejected": -2.1742844581604004,
547
- "step": 340
548
- },
549
- {
550
- "epoch": 0.28,
551
- "learning_rate": 4.5306818941099866e-07,
552
- "logits/chosen": -1.9084612131118774,
553
- "logits/rejected": -1.8514792919158936,
554
- "logps/chosen": -316.9821472167969,
555
- "logps/rejected": -352.9412841796875,
556
- "loss": 0.5825,
557
- "rewards/accuracies": 0.65625,
558
- "rewards/chosen": -1.5627154111862183,
559
- "rewards/margins": 0.5152220726013184,
560
- "rewards/rejected": -2.077937364578247,
561
- "step": 350
562
- },
563
- {
564
- "epoch": 0.29,
565
- "learning_rate": 4.4893620829118124e-07,
566
- "logits/chosen": -1.8860156536102295,
567
- "logits/rejected": -1.8301204442977905,
568
- "logps/chosen": -309.8200378417969,
569
- "logps/rejected": -362.0408935546875,
570
- "loss": 0.5755,
571
- "rewards/accuracies": 0.6812499761581421,
572
- "rewards/chosen": -1.5869390964508057,
573
- "rewards/margins": 0.49348369240760803,
574
- "rewards/rejected": -2.080422878265381,
575
- "step": 360
576
- },
577
- {
578
- "epoch": 0.29,
579
- "learning_rate": 4.4465047235785185e-07,
580
- "logits/chosen": -1.6610889434814453,
581
- "logits/rejected": -1.585129737854004,
582
- "logps/chosen": -321.8608703613281,
583
- "logps/rejected": -380.31036376953125,
584
- "loss": 0.5697,
585
- "rewards/accuracies": 0.7250000238418579,
586
- "rewards/chosen": -1.774713158607483,
587
- "rewards/margins": 0.6593302488327026,
588
- "rewards/rejected": -2.4340434074401855,
589
- "step": 370
590
- },
591
- {
592
- "epoch": 0.3,
593
- "learning_rate": 4.40214293992074e-07,
594
- "logits/chosen": -1.385825753211975,
595
- "logits/rejected": -1.31913161277771,
596
- "logps/chosen": -377.07269287109375,
597
- "logps/rejected": -459.5557556152344,
598
- "loss": 0.5818,
599
- "rewards/accuracies": 0.706250011920929,
600
- "rewards/chosen": -2.1890993118286133,
601
- "rewards/margins": 0.7521292567253113,
602
- "rewards/rejected": -2.9412286281585693,
603
- "step": 380
604
- },
605
- {
606
- "epoch": 0.31,
607
- "learning_rate": 4.3563110184961234e-07,
608
- "logits/chosen": -1.5089499950408936,
609
- "logits/rejected": -1.4075387716293335,
610
- "logps/chosen": -338.3626708984375,
611
- "logps/rejected": -396.67578125,
612
- "loss": 0.5584,
613
- "rewards/accuracies": 0.737500011920929,
614
- "rewards/chosen": -1.9042552709579468,
615
- "rewards/margins": 0.5932050347328186,
616
- "rewards/rejected": -2.49746036529541,
617
- "step": 390
618
- },
619
- {
620
- "epoch": 0.32,
621
- "learning_rate": 4.3090443821097566e-07,
622
- "logits/chosen": -1.2587625980377197,
623
- "logits/rejected": -1.2017955780029297,
624
- "logps/chosen": -309.43377685546875,
625
- "logps/rejected": -372.00531005859375,
626
- "loss": 0.5729,
627
- "rewards/accuracies": 0.6812499761581421,
628
- "rewards/chosen": -1.9031288623809814,
629
- "rewards/margins": 0.5602144598960876,
630
- "rewards/rejected": -2.4633431434631348,
631
- "step": 400
632
- },
633
- {
634
- "epoch": 0.32,
635
- "eval_logits/chosen": -1.3760210275650024,
636
- "eval_logits/rejected": -1.2920024394989014,
637
- "eval_logps/chosen": -312.20635986328125,
638
- "eval_logps/rejected": -375.1720275878906,
639
- "eval_loss": 0.5711147785186768,
640
- "eval_rewards/accuracies": 0.6828358173370361,
641
- "eval_rewards/chosen": -1.676328182220459,
642
- "eval_rewards/margins": 0.5640482306480408,
643
- "eval_rewards/rejected": -2.2403764724731445,
644
- "eval_runtime": 184.2144,
645
- "eval_samples_per_second": 46.424,
646
- "eval_steps_per_second": 0.727,
647
- "step": 400
648
- },
649
- {
650
- "epoch": 0.33,
651
- "learning_rate": 4.2603795624364195e-07,
652
- "logits/chosen": -1.2894772291183472,
653
- "logits/rejected": -1.23129141330719,
654
- "logps/chosen": -299.457275390625,
655
- "logps/rejected": -370.8555908203125,
656
- "loss": 0.5666,
657
- "rewards/accuracies": 0.6875,
658
- "rewards/chosen": -1.6054102182388306,
659
- "rewards/margins": 0.5984233021736145,
660
- "rewards/rejected": -2.203833818435669,
661
- "step": 410
662
- },
663
- {
664
- "epoch": 0.33,
665
- "learning_rate": 4.210354171785795e-07,
666
- "logits/chosen": -1.022984266281128,
667
- "logits/rejected": -0.9285897016525269,
668
- "logps/chosen": -324.4284973144531,
669
- "logps/rejected": -385.0074157714844,
670
- "loss": 0.5596,
671
- "rewards/accuracies": 0.6875,
672
- "rewards/chosen": -1.921677589416504,
673
- "rewards/margins": 0.5404387712478638,
674
- "rewards/rejected": -2.4621164798736572,
675
- "step": 420
676
- },
677
- {
678
- "epoch": 0.34,
679
- "learning_rate": 4.15900687403248e-07,
680
- "logits/chosen": -0.8059805631637573,
681
- "logits/rejected": -0.7196700572967529,
682
- "logps/chosen": -353.788330078125,
683
- "logps/rejected": -411.4853515625,
684
- "loss": 0.5865,
685
- "rewards/accuracies": 0.643750011920929,
686
- "rewards/chosen": -2.1321234703063965,
687
- "rewards/margins": 0.463266521692276,
688
- "rewards/rejected": -2.5953898429870605,
689
- "step": 430
690
- },
691
- {
692
- "epoch": 0.35,
693
- "learning_rate": 4.1063773547332584e-07,
694
- "logits/chosen": -0.9645301699638367,
695
- "logits/rejected": -0.7601315975189209,
696
- "logps/chosen": -346.8272705078125,
697
- "logps/rejected": -392.2935791015625,
698
- "loss": 0.5591,
699
- "rewards/accuracies": 0.762499988079071,
700
- "rewards/chosen": -1.9153356552124023,
701
- "rewards/margins": 0.5891679525375366,
702
- "rewards/rejected": -2.5045037269592285,
703
- "step": 440
704
- },
705
- {
706
- "epoch": 0.36,
707
- "learning_rate": 4.0525062904547276e-07,
708
- "logits/chosen": -0.608537495136261,
709
- "logits/rejected": -0.47767123579978943,
710
- "logps/chosen": -341.55364990234375,
711
- "logps/rejected": -434.1073303222656,
712
- "loss": 0.5687,
713
- "rewards/accuracies": 0.737500011920929,
714
- "rewards/chosen": -2.105318069458008,
715
- "rewards/margins": 0.6994394659996033,
716
- "rewards/rejected": -2.8047571182250977,
717
- "step": 450
718
- },
719
- {
720
- "epoch": 0.37,
721
- "learning_rate": 3.997435317334988e-07,
722
- "logits/chosen": -0.6356207132339478,
723
- "logits/rejected": -0.25634175539016724,
724
- "logps/chosen": -384.43780517578125,
725
- "logps/rejected": -419.24176025390625,
726
- "loss": 0.5608,
727
- "rewards/accuracies": 0.731249988079071,
728
- "rewards/chosen": -2.2970900535583496,
729
- "rewards/margins": 0.6535100340843201,
730
- "rewards/rejected": -2.9506001472473145,
731
- "step": 460
732
- },
733
- {
734
- "epoch": 0.37,
735
- "learning_rate": 3.941206998903701e-07,
736
- "logits/chosen": -1.0318920612335205,
737
- "logits/rejected": -0.7451022267341614,
738
- "logps/chosen": -338.9430236816406,
739
- "logps/rejected": -384.64111328125,
740
- "loss": 0.5678,
741
- "rewards/accuracies": 0.737500011920929,
742
- "rewards/chosen": -1.9643396139144897,
743
- "rewards/margins": 0.5402536392211914,
744
- "rewards/rejected": -2.5045928955078125,
745
- "step": 470
746
- },
747
- {
748
- "epoch": 0.38,
749
- "learning_rate": 3.8838647931853684e-07,
750
- "logits/chosen": -0.6847028732299805,
751
- "logits/rejected": -0.5548251867294312,
752
- "logps/chosen": -339.61456298828125,
753
- "logps/rejected": -435.32061767578125,
754
- "loss": 0.5814,
755
- "rewards/accuracies": 0.7250000238418579,
756
- "rewards/chosen": -2.1349122524261475,
757
- "rewards/margins": 0.7573872804641724,
758
- "rewards/rejected": -2.8923001289367676,
759
- "step": 480
760
- },
761
- {
762
- "epoch": 0.39,
763
- "learning_rate": 3.825453019111281e-07,
764
- "logits/chosen": -0.5378957986831665,
765
- "logits/rejected": -0.28533270955085754,
766
- "logps/chosen": -363.78570556640625,
767
- "logps/rejected": -430.11749267578125,
768
- "loss": 0.5327,
769
- "rewards/accuracies": 0.699999988079071,
770
- "rewards/chosen": -2.134934425354004,
771
- "rewards/margins": 0.6089809536933899,
772
- "rewards/rejected": -2.743915319442749,
773
- "step": 490
774
- },
775
- {
776
- "epoch": 0.4,
777
- "learning_rate": 3.7660168222660824e-07,
778
- "logits/chosen": -0.6318235397338867,
779
- "logits/rejected": -0.5071814656257629,
780
- "logps/chosen": -350.5252380371094,
781
- "logps/rejected": -421.93353271484375,
782
- "loss": 0.5645,
783
- "rewards/accuracies": 0.6812499761581421,
784
- "rewards/chosen": -2.062009572982788,
785
- "rewards/margins": 0.5333147048950195,
786
- "rewards/rejected": -2.5953242778778076,
787
- "step": 500
788
- },
789
- {
790
- "epoch": 0.4,
791
- "eval_logits/chosen": -0.7860146760940552,
792
- "eval_logits/rejected": -0.6090859770774841,
793
- "eval_logps/chosen": -351.7882995605469,
794
- "eval_logps/rejected": -419.81939697265625,
795
- "eval_loss": 0.5639454126358032,
796
- "eval_rewards/accuracies": 0.6986940503120422,
797
- "eval_rewards/chosen": -2.0721471309661865,
798
- "eval_rewards/margins": 0.6147031188011169,
799
- "eval_rewards/rejected": -2.6868505477905273,
800
- "eval_runtime": 184.1817,
801
- "eval_samples_per_second": 46.432,
802
- "eval_steps_per_second": 0.728,
803
- "step": 500
804
- },
805
- {
806
- "epoch": 0.41,
807
- "learning_rate": 3.705602139995416e-07,
808
- "logits/chosen": -0.7258490920066833,
809
- "logits/rejected": -0.4828409254550934,
810
- "logps/chosen": -388.1371154785156,
811
- "logps/rejected": -422.11181640625,
812
- "loss": 0.574,
813
- "rewards/accuracies": 0.637499988079071,
814
- "rewards/chosen": -2.290266513824463,
815
- "rewards/margins": 0.4104091227054596,
816
- "rewards/rejected": -2.7006754875183105,
817
- "step": 510
818
- },
819
- {
820
- "epoch": 0.41,
821
- "learning_rate": 3.6442556659016475e-07,
822
- "logits/chosen": -0.5335447192192078,
823
- "logits/rejected": -0.33706527948379517,
824
- "logps/chosen": -378.86492919921875,
825
- "logps/rejected": -429.67724609375,
826
- "loss": 0.5608,
827
- "rewards/accuracies": 0.737500011920929,
828
- "rewards/chosen": -2.236337423324585,
829
- "rewards/margins": 0.556148886680603,
830
- "rewards/rejected": -2.7924864292144775,
831
- "step": 520
832
- },
833
- {
834
- "epoch": 0.42,
835
- "learning_rate": 3.582024813755076e-07,
836
- "logits/chosen": -0.39548322558403015,
837
- "logits/rejected": -0.10662730038166046,
838
- "logps/chosen": -368.8847961425781,
839
- "logps/rejected": -473.3500061035156,
840
- "loss": 0.5485,
841
- "rewards/accuracies": 0.731249988079071,
842
- "rewards/chosen": -2.3263449668884277,
843
- "rewards/margins": 0.8236624598503113,
844
- "rewards/rejected": -3.150007724761963,
845
- "step": 530
846
- },
847
- {
848
- "epoch": 0.43,
849
- "learning_rate": 3.5189576808485404e-07,
850
- "logits/chosen": 0.15742243826389313,
851
- "logits/rejected": 0.31491726636886597,
852
- "logps/chosen": -394.34930419921875,
853
- "logps/rejected": -492.82232666015625,
854
- "loss": 0.5478,
855
- "rewards/accuracies": 0.7250000238418579,
856
- "rewards/chosen": -2.6109700202941895,
857
- "rewards/margins": 0.8250136375427246,
858
- "rewards/rejected": -3.435983657836914,
859
- "step": 540
860
- },
861
- {
862
- "epoch": 0.44,
863
- "learning_rate": 3.4551030108237433e-07,
864
- "logits/chosen": -0.2550584375858307,
865
- "logits/rejected": -0.06936412304639816,
866
- "logps/chosen": -406.5508728027344,
867
- "logps/rejected": -448.47576904296875,
868
- "loss": 0.5562,
869
- "rewards/accuracies": 0.6312500238418579,
870
- "rewards/chosen": -2.5152666568756104,
871
- "rewards/margins": 0.4819938540458679,
872
- "rewards/rejected": -2.997260332107544,
873
- "step": 550
874
- },
875
- {
876
- "epoch": 0.45,
877
- "learning_rate": 3.390510155998023e-07,
878
- "logits/chosen": -0.5292027592658997,
879
- "logits/rejected": -0.2619571387767792,
880
- "logps/chosen": -371.6798095703125,
881
- "logps/rejected": -420.7915954589844,
882
- "loss": 0.5492,
883
- "rewards/accuracies": 0.7250000238418579,
884
- "rewards/chosen": -2.1147050857543945,
885
- "rewards/margins": 0.6524336338043213,
886
- "rewards/rejected": -2.7671384811401367,
887
- "step": 560
888
- },
889
- {
890
- "epoch": 0.45,
891
- "learning_rate": 3.325229039220684e-07,
892
- "logits/chosen": -0.5881962776184082,
893
- "logits/rejected": -0.4658876061439514,
894
- "logps/chosen": -343.7039794921875,
895
- "logps/rejected": -406.14178466796875,
896
- "loss": 0.57,
897
- "rewards/accuracies": 0.6875,
898
- "rewards/chosen": -2.0860273838043213,
899
- "rewards/margins": 0.498068630695343,
900
- "rewards/rejected": -2.5840957164764404,
901
- "step": 570
902
- },
903
- {
904
- "epoch": 0.46,
905
- "learning_rate": 3.2593101152883795e-07,
906
- "logits/chosen": -0.6565806269645691,
907
- "logits/rejected": -0.2549567222595215,
908
- "logps/chosen": -374.8047180175781,
909
- "logps/rejected": -430.33221435546875,
910
- "loss": 0.5512,
911
- "rewards/accuracies": 0.7437499761581421,
912
- "rewards/chosen": -2.2211391925811768,
913
- "rewards/margins": 0.6813799142837524,
914
- "rewards/rejected": -2.9025187492370605,
915
- "step": 580
916
- },
917
- {
918
- "epoch": 0.47,
919
- "learning_rate": 3.192804331949349e-07,
920
- "logits/chosen": -0.07184700667858124,
921
- "logits/rejected": 0.1699156016111374,
922
- "logps/chosen": -422.27081298828125,
923
- "logps/rejected": -490.69134521484375,
924
- "loss": 0.535,
925
- "rewards/accuracies": 0.731249988079071,
926
- "rewards/chosen": -2.730973720550537,
927
- "rewards/margins": 0.7726518511772156,
928
- "rewards/rejected": -3.5036251544952393,
929
- "step": 590
930
- },
931
- {
932
- "epoch": 0.48,
933
- "learning_rate": 3.125763090526674e-07,
934
- "logits/chosen": -0.029465889558196068,
935
- "logits/rejected": 0.15842057764530182,
936
- "logps/chosen": -417.373046875,
937
- "logps/rejected": -478.73291015625,
938
- "loss": 0.5513,
939
- "rewards/accuracies": 0.675000011920929,
940
- "rewards/chosen": -2.8006317615509033,
941
- "rewards/margins": 0.6451797485351562,
942
- "rewards/rejected": -3.4458115100860596,
943
- "step": 600
944
- },
945
- {
946
- "epoch": 0.48,
947
- "eval_logits/chosen": -0.10542195290327072,
948
- "eval_logits/rejected": 0.12242482602596283,
949
- "eval_logps/chosen": -436.9386291503906,
950
- "eval_logps/rejected": -505.02227783203125,
951
- "eval_loss": 0.5582411885261536,
952
- "eval_rewards/accuracies": 0.7108209133148193,
953
- "eval_rewards/chosen": -2.9236514568328857,
954
- "eval_rewards/margins": 0.6152271032333374,
955
- "eval_rewards/rejected": -3.5388784408569336,
956
- "eval_runtime": 184.1643,
957
- "eval_samples_per_second": 46.437,
958
- "eval_steps_per_second": 0.728,
959
- "step": 600
960
- },
961
- {
962
- "epoch": 0.49,
963
- "learning_rate": 3.0582382061909623e-07,
964
- "logits/chosen": -0.2445104569196701,
965
- "logits/rejected": -0.018268002197146416,
966
- "logps/chosen": -441.7857971191406,
967
- "logps/rejected": -502.60791015625,
968
- "loss": 0.5594,
969
- "rewards/accuracies": 0.6875,
970
- "rewards/chosen": -2.858261823654175,
971
- "rewards/margins": 0.5510683655738831,
972
- "rewards/rejected": -3.409330368041992,
973
- "step": 610
974
- },
975
- {
976
- "epoch": 0.49,
977
- "learning_rate": 2.9902818679131775e-07,
978
- "logits/chosen": -0.4190225601196289,
979
- "logits/rejected": -0.22823679447174072,
980
- "logps/chosen": -399.03924560546875,
981
- "logps/rejected": -498.6724548339844,
982
- "loss": 0.5499,
983
- "rewards/accuracies": 0.71875,
984
- "rewards/chosen": -2.650449275970459,
985
- "rewards/margins": 0.7673205137252808,
986
- "rewards/rejected": -3.4177703857421875,
987
- "step": 620
988
- },
989
- {
990
- "epoch": 0.5,
991
- "learning_rate": 2.921946598128571e-07,
992
- "logits/chosen": -0.43653860688209534,
993
- "logits/rejected": -0.20837187767028809,
994
- "logps/chosen": -402.82781982421875,
995
- "logps/rejected": -485.4117736816406,
996
- "loss": 0.5739,
997
- "rewards/accuracies": 0.7124999761581421,
998
- "rewards/chosen": -2.573000192642212,
999
- "rewards/margins": 0.7478531002998352,
1000
- "rewards/rejected": -3.3208529949188232,
1001
- "step": 630
1002
- },
1003
- {
1004
- "epoch": 0.51,
1005
- "learning_rate": 2.8532852121428733e-07,
1006
- "logits/chosen": -0.43430274724960327,
1007
- "logits/rejected": -0.13240045309066772,
1008
- "logps/chosen": -397.2491149902344,
1009
- "logps/rejected": -442.12384033203125,
1010
- "loss": 0.5462,
1011
- "rewards/accuracies": 0.7250000238418579,
1012
- "rewards/chosen": -2.4125733375549316,
1013
- "rewards/margins": 0.5821372866630554,
1014
- "rewards/rejected": -2.9947104454040527,
1015
- "step": 640
1016
- },
1017
- {
1018
- "epoch": 0.52,
1019
- "learning_rate": 2.7843507773121414e-07,
1020
- "logits/chosen": -0.4247920513153076,
1021
- "logits/rejected": -0.21372787654399872,
1022
- "logps/chosen": -389.4237976074219,
1023
- "logps/rejected": -458.3169860839844,
1024
- "loss": 0.5373,
1025
- "rewards/accuracies": 0.6812499761581421,
1026
- "rewards/chosen": -2.444688320159912,
1027
- "rewards/margins": 0.7236617207527161,
1028
- "rewards/rejected": -3.1683506965637207,
1029
- "step": 650
1030
- },
1031
- {
1032
- "epoch": 0.53,
1033
- "learning_rate": 2.715196572027789e-07,
1034
- "logits/chosen": -0.6697942614555359,
1035
- "logits/rejected": -0.4933086931705475,
1036
- "logps/chosen": -387.529296875,
1037
- "logps/rejected": -472.73944091796875,
1038
- "loss": 0.5685,
1039
- "rewards/accuracies": 0.7124999761581421,
1040
- "rewards/chosen": -2.3496451377868652,
1041
- "rewards/margins": 0.7728831171989441,
1042
- "rewards/rejected": -3.122528314590454,
1043
- "step": 660
1044
- },
1045
- {
1046
- "epoch": 0.53,
1047
- "learning_rate": 2.645876044538521e-07,
1048
- "logits/chosen": -1.0338900089263916,
1049
- "logits/rejected": -0.8813627362251282,
1050
- "logps/chosen": -372.53118896484375,
1051
- "logps/rejected": -426.54241943359375,
1052
- "loss": 0.5725,
1053
- "rewards/accuracies": 0.675000011920929,
1054
- "rewards/chosen": -2.201908588409424,
1055
- "rewards/margins": 0.5010865926742554,
1056
- "rewards/rejected": -2.7029950618743896,
1057
- "step": 670
1058
- },
1059
- {
1060
- "epoch": 0.54,
1061
- "learning_rate": 2.5764427716409815e-07,
1062
- "logits/chosen": -0.9278701543807983,
1063
- "logits/rejected": -0.7282145023345947,
1064
- "logps/chosen": -347.2828674316406,
1065
- "logps/rejected": -416.9349060058594,
1066
- "loss": 0.5479,
1067
- "rewards/accuracies": 0.7124999761581421,
1068
- "rewards/chosen": -2.0276436805725098,
1069
- "rewards/margins": 0.743033230304718,
1070
- "rewards/rejected": -2.770677089691162,
1071
- "step": 680
1072
- },
1073
- {
1074
- "epoch": 0.55,
1075
- "learning_rate": 2.5069504172710494e-07,
1076
- "logits/chosen": -0.5008482336997986,
1077
- "logits/rejected": -0.34875133633613586,
1078
- "logps/chosen": -373.7621154785156,
1079
- "logps/rejected": -485.12884521484375,
1080
- "loss": 0.5217,
1081
- "rewards/accuracies": 0.762499988079071,
1082
- "rewards/chosen": -2.347053050994873,
1083
- "rewards/margins": 0.9024646878242493,
1084
- "rewards/rejected": -3.2495174407958984,
1085
- "step": 690
1086
- },
1087
- {
1088
- "epoch": 0.56,
1089
- "learning_rate": 2.4374526910277886e-07,
1090
- "logits/chosen": 0.06850005686283112,
1091
- "logits/rejected": 0.41385045647621155,
1092
- "logps/chosen": -411.46246337890625,
1093
- "logps/rejected": -476.6162109375,
1094
- "loss": 0.5571,
1095
- "rewards/accuracies": 0.706250011920929,
1096
- "rewards/chosen": -2.6757898330688477,
1097
- "rewards/margins": 0.8085702657699585,
1098
- "rewards/rejected": -3.4843602180480957,
1099
- "step": 700
1100
- },
1101
- {
1102
- "epoch": 0.56,
1103
- "eval_logits/chosen": 0.035554468631744385,
1104
- "eval_logits/rejected": 0.2980235815048218,
1105
- "eval_logps/chosen": -424.2823486328125,
1106
- "eval_logps/rejected": -505.6960754394531,
1107
- "eval_loss": 0.5558871626853943,
1108
- "eval_rewards/accuracies": 0.704291045665741,
1109
- "eval_rewards/chosen": -2.797088146209717,
1110
- "eval_rewards/margins": 0.748529314994812,
1111
- "eval_rewards/rejected": -3.5456173419952393,
1112
- "eval_runtime": 184.1561,
1113
- "eval_samples_per_second": 46.439,
1114
- "eval_steps_per_second": 0.728,
1115
- "step": 700
1116
- },
1117
- {
1118
- "epoch": 0.57,
1119
- "learning_rate": 2.368003306662104e-07,
1120
- "logits/chosen": 0.07857178151607513,
1121
- "logits/rejected": 0.3302653729915619,
1122
- "logps/chosen": -413.8836975097656,
1123
- "logps/rejected": -535.0875244140625,
1124
- "loss": 0.5287,
1125
- "rewards/accuracies": 0.762499988079071,
1126
- "rewards/chosen": -2.7193782329559326,
1127
- "rewards/margins": 1.0089346170425415,
1128
- "rewards/rejected": -3.7283127307891846,
1129
- "step": 710
1130
- },
1131
- {
1132
- "epoch": 0.57,
1133
- "learning_rate": 2.2986559405621886e-07,
1134
- "logits/chosen": 0.2789291739463806,
1135
- "logits/rejected": 0.4242584705352783,
1136
- "logps/chosen": -422.7801818847656,
1137
- "logps/rejected": -522.7840576171875,
1138
- "loss": 0.5551,
1139
- "rewards/accuracies": 0.731249988079071,
1140
- "rewards/chosen": -2.925621509552002,
1141
- "rewards/margins": 0.8043605089187622,
1142
- "rewards/rejected": -3.729982376098633,
1143
- "step": 720
1144
- },
1145
- {
1146
- "epoch": 0.58,
1147
- "learning_rate": 2.2294641902678443e-07,
1148
- "logits/chosen": -0.19327735900878906,
1149
- "logits/rejected": 0.043265581130981445,
1150
- "logps/chosen": -363.1488342285156,
1151
- "logps/rejected": -470.94970703125,
1152
- "loss": 0.5284,
1153
- "rewards/accuracies": 0.78125,
1154
- "rewards/chosen": -2.39530873298645,
1155
- "rewards/margins": 0.908363938331604,
1156
- "rewards/rejected": -3.3036727905273438,
1157
- "step": 730
1158
- },
1159
- {
1160
- "epoch": 0.59,
1161
- "learning_rate": 2.160481533045751e-07,
1162
- "logits/chosen": -0.37412697076797485,
1163
- "logits/rejected": -0.17320053279399872,
1164
- "logps/chosen": -390.2896423339844,
1165
- "logps/rejected": -428.08099365234375,
1166
- "loss": 0.5572,
1167
- "rewards/accuracies": 0.675000011920929,
1168
- "rewards/chosen": -2.3005330562591553,
1169
- "rewards/margins": 0.48462891578674316,
1170
- "rewards/rejected": -2.7851624488830566,
1171
- "step": 740
1172
- },
1173
- {
1174
- "epoch": 0.6,
1175
- "learning_rate": 2.0917612845576882e-07,
1176
- "logits/chosen": -0.26352375745773315,
1177
- "logits/rejected": -0.0010178961092606187,
1178
- "logps/chosen": -373.3875427246094,
1179
- "logps/rejected": -440.09442138671875,
1180
- "loss": 0.5534,
1181
- "rewards/accuracies": 0.6875,
1182
- "rewards/chosen": -2.3315823078155518,
1183
- "rewards/margins": 0.6843063235282898,
1184
- "rewards/rejected": -3.0158886909484863,
1185
- "step": 750
1186
- },
1187
- {
1188
- "epoch": 0.6,
1189
- "learning_rate": 2.0233565576536564e-07,
1190
- "logits/chosen": -0.3354080021381378,
1191
- "logits/rejected": -0.006600166670978069,
1192
- "logps/chosen": -360.56463623046875,
1193
- "logps/rejected": -440.66961669921875,
1194
- "loss": 0.5328,
1195
- "rewards/accuracies": 0.7437499761581421,
1196
- "rewards/chosen": -2.1626803874969482,
1197
- "rewards/margins": 0.8473829030990601,
1198
- "rewards/rejected": -3.010063409805298,
1199
- "step": 760
1200
- },
1201
- {
1202
- "epoch": 0.61,
1203
- "learning_rate": 1.9553202213217537e-07,
1204
- "logits/chosen": -0.021420275792479515,
1205
- "logits/rejected": 0.19946305453777313,
1206
- "logps/chosen": -389.1043395996094,
1207
- "logps/rejected": -448.04998779296875,
1208
- "loss": 0.5523,
1209
- "rewards/accuracies": 0.731249988079071,
1210
- "rewards/chosen": -2.391838788986206,
1211
- "rewards/margins": 0.6678962707519531,
1212
- "rewards/rejected": -3.059735059738159,
1213
- "step": 770
1214
- },
1215
- {
1216
- "epoch": 0.62,
1217
- "learning_rate": 1.887704859826528e-07,
1218
- "logits/chosen": -0.15253478288650513,
1219
- "logits/rejected": -0.00011998042464256287,
1220
- "logps/chosen": -394.9501953125,
1221
- "logps/rejected": -462.32843017578125,
1222
- "loss": 0.5443,
1223
- "rewards/accuracies": 0.699999988079071,
1224
- "rewards/chosen": -2.476644992828369,
1225
- "rewards/margins": 0.566824734210968,
1226
- "rewards/rejected": -3.0434699058532715,
1227
- "step": 780
1228
- },
1229
- {
1230
- "epoch": 0.63,
1231
- "learning_rate": 1.8205627320673836e-07,
1232
- "logits/chosen": -0.17955633997917175,
1233
- "logits/rejected": 0.18167546391487122,
1234
- "logps/chosen": -390.32244873046875,
1235
- "logps/rejected": -444.895263671875,
1236
- "loss": 0.5566,
1237
- "rewards/accuracies": 0.699999988079071,
1238
- "rewards/chosen": -2.4376220703125,
1239
- "rewards/margins": 0.7008293271064758,
1240
- "rewards/rejected": -3.138451099395752,
1241
- "step": 790
1242
- },
1243
- {
1244
- "epoch": 0.64,
1245
- "learning_rate": 1.7539457311884675e-07,
1246
- "logits/chosen": -0.09838727861642838,
1247
- "logits/rejected": 0.11829495429992676,
1248
- "logps/chosen": -402.4017333984375,
1249
- "logps/rejected": -451.49346923828125,
1250
- "loss": 0.5609,
1251
- "rewards/accuracies": 0.699999988079071,
1252
- "rewards/chosen": -2.4441986083984375,
1253
- "rewards/margins": 0.5067789554595947,
1254
- "rewards/rejected": -2.9509775638580322,
1255
- "step": 800
1256
- },
1257
- {
1258
- "epoch": 0.64,
1259
- "eval_logits/chosen": -0.03116540051996708,
1260
- "eval_logits/rejected": 0.1922437697649002,
1261
- "eval_logps/chosen": -387.7091979980469,
1262
- "eval_logps/rejected": -459.44390869140625,
1263
- "eval_loss": 0.5468714833259583,
1264
- "eval_rewards/accuracies": 0.7108209133148193,
1265
- "eval_rewards/chosen": -2.431356430053711,
1266
- "eval_rewards/margins": 0.6517390012741089,
1267
- "eval_rewards/rejected": -3.0830955505371094,
1268
- "eval_runtime": 184.1014,
1269
- "eval_samples_per_second": 46.453,
1270
- "eval_steps_per_second": 0.728,
1271
- "step": 800
1272
- },
1273
- {
1274
- "epoch": 0.64,
1275
- "learning_rate": 1.687905344471226e-07,
1276
- "logits/chosen": 0.07735608518123627,
1277
- "logits/rejected": 0.3973601460456848,
1278
- "logps/chosen": -408.05999755859375,
1279
- "logps/rejected": -459.011474609375,
1280
- "loss": 0.5384,
1281
- "rewards/accuracies": 0.706250011920929,
1282
- "rewards/chosen": -2.5008435249328613,
1283
- "rewards/margins": 0.6535352468490601,
1284
- "rewards/rejected": -3.154379367828369,
1285
- "step": 810
1286
- },
1287
- {
1288
- "epoch": 0.65,
1289
- "learning_rate": 1.6224926135406693e-07,
1290
- "logits/chosen": 0.1125444769859314,
1291
- "logits/rejected": 0.3865428566932678,
1292
- "logps/chosen": -404.16058349609375,
1293
- "logps/rejected": -484.68621826171875,
1294
- "loss": 0.5448,
1295
- "rewards/accuracies": 0.706250011920929,
1296
- "rewards/chosen": -2.4405789375305176,
1297
- "rewards/margins": 0.718208909034729,
1298
- "rewards/rejected": -3.158787727355957,
1299
- "step": 820
1300
- },
1301
- {
1302
- "epoch": 0.66,
1303
- "learning_rate": 1.557758094916053e-07,
1304
- "logits/chosen": 0.11989516019821167,
1305
- "logits/rejected": 0.30926594138145447,
1306
- "logps/chosen": -370.29876708984375,
1307
- "logps/rejected": -452.27911376953125,
1308
- "loss": 0.5418,
1309
- "rewards/accuracies": 0.737500011920929,
1310
- "rewards/chosen": -2.3860089778900146,
1311
- "rewards/margins": 0.7260924577713013,
1312
- "rewards/rejected": -3.1121015548706055,
1313
- "step": 830
1314
- },
1315
  {
1316
  "epoch": 0.67,
1317
- "learning_rate": 1.4937518209365108e-07,
1318
- "logits/chosen": -0.14239154756069183,
1319
- "logits/rejected": 0.14250756800174713,
1320
- "logps/chosen": -395.55755615234375,
1321
- "logps/rejected": -447.6368713378906,
1322
- "loss": 0.5573,
1323
- "rewards/accuracies": 0.7250000238418579,
1324
- "rewards/chosen": -2.378154754638672,
1325
- "rewards/margins": 0.6160937547683716,
1326
- "rewards/rejected": -2.994248390197754,
1327
- "step": 840
1328
- },
1329
- {
1330
- "epoch": 0.68,
1331
- "learning_rate": 1.4305232610918045e-07,
1332
- "logits/chosen": -0.16526366770267487,
1333
- "logits/rejected": 0.16432161629199982,
1334
- "logps/chosen": -373.45330810546875,
1335
- "logps/rejected": -436.6773376464844,
1336
- "loss": 0.5415,
1337
- "rewards/accuracies": 0.737500011920929,
1338
- "rewards/chosen": -2.3134028911590576,
1339
- "rewards/margins": 0.774810791015625,
1340
- "rewards/rejected": -3.0882136821746826,
1341
- "step": 850
1342
- },
1343
- {
1344
- "epoch": 0.68,
1345
- "learning_rate": 1.3681212837880977e-07,
1346
- "logits/chosen": -0.1321481615304947,
1347
- "logits/rejected": 0.23287932574748993,
1348
- "logps/chosen": -364.96990966796875,
1349
- "logps/rejected": -447.7923278808594,
1350
- "loss": 0.5396,
1351
- "rewards/accuracies": 0.7875000238418579,
1352
- "rewards/chosen": -2.176964282989502,
1353
- "rewards/margins": 0.8955341577529907,
1354
- "rewards/rejected": -3.0724985599517822,
1355
- "step": 860
1356
  },
1357
  {
1358
  "epoch": 0.69,
1359
- "learning_rate": 1.3065941185782977e-07,
1360
- "logits/chosen": 0.05437428876757622,
1361
- "logits/rejected": 0.2819867432117462,
1362
- "logps/chosen": -383.08599853515625,
1363
- "logps/rejected": -439.3629455566406,
1364
- "loss": 0.5505,
1365
- "rewards/accuracies": 0.637499988079071,
1366
- "rewards/chosen": -2.405247211456299,
1367
- "rewards/margins": 0.5403125882148743,
1368
- "rewards/rejected": -2.9455599784851074,
1369
- "step": 870
1370
- },
1371
- {
1372
- "epoch": 0.7,
1373
- "learning_rate": 1.2459893188861613e-07,
1374
- "logits/chosen": -0.12052659690380096,
1375
- "logits/rejected": 0.12284734100103378,
1376
- "logps/chosen": -367.1181640625,
1377
- "logps/rejected": -468.1044921875,
1378
- "loss": 0.5185,
1379
- "rewards/accuracies": 0.75,
1380
- "rewards/chosen": -2.39152455329895,
1381
- "rewards/margins": 0.9137696027755737,
1382
- "rewards/rejected": -3.3052947521209717,
1383
- "step": 880
1384
  },
1385
  {
1386
  "epoch": 0.71,
1387
- "learning_rate": 1.1863537252529548e-07,
1388
- "logits/chosen": 0.14598000049591064,
1389
- "logits/rejected": 0.38815659284591675,
1390
- "logps/chosen": -397.891357421875,
1391
- "logps/rejected": -472.38677978515625,
1392
- "loss": 0.5323,
1393
- "rewards/accuracies": 0.731249988079071,
1394
- "rewards/chosen": -2.512676239013672,
1395
- "rewards/margins": 0.7713057994842529,
1396
- "rewards/rejected": -3.2839818000793457,
1397
- "step": 890
1398
- },
1399
- {
1400
- "epoch": 0.72,
1401
- "learning_rate": 1.1277334291351145e-07,
1402
- "logits/chosen": 0.15319526195526123,
1403
- "logits/rejected": 0.35974830389022827,
1404
- "logps/chosen": -380.77783203125,
1405
- "logps/rejected": -449.54315185546875,
1406
- "loss": 0.5514,
1407
- "rewards/accuracies": 0.6812499761581421,
1408
- "rewards/chosen": -2.3706305027008057,
1409
- "rewards/margins": 0.6724039912223816,
1410
- "rewards/rejected": -3.043034076690674,
1411
- "step": 900
1412
- },
1413
- {
1414
- "epoch": 0.72,
1415
- "eval_logits/chosen": 0.28598034381866455,
1416
- "eval_logits/rejected": 0.5382024645805359,
1417
- "eval_logps/chosen": -392.3096008300781,
1418
- "eval_logps/rejected": -471.95330810546875,
1419
- "eval_loss": 0.5473664402961731,
1420
- "eval_rewards/accuracies": 0.6996268630027771,
1421
- "eval_rewards/chosen": -2.4773612022399902,
1422
- "eval_rewards/margins": 0.7308279275894165,
1423
- "eval_rewards/rejected": -3.2081892490386963,
1424
- "eval_runtime": 184.2204,
1425
- "eval_samples_per_second": 46.423,
1426
- "eval_steps_per_second": 0.727,
1427
- "step": 900
1428
- },
1429
- {
1430
- "epoch": 0.72,
1431
- "learning_rate": 1.0701737372808431e-07,
1432
- "logits/chosen": 0.15951867401599884,
1433
- "logits/rejected": 0.46630391478538513,
1434
- "logps/chosen": -383.52850341796875,
1435
- "logps/rejected": -467.2303771972656,
1436
- "loss": 0.5362,
1437
  "rewards/accuracies": 0.7124999761581421,
1438
- "rewards/chosen": -2.2782187461853027,
1439
- "rewards/margins": 0.8473943471908569,
1440
- "rewards/rejected": -3.125612735748291,
1441
- "step": 910
1442
  },
1443
  {
1444
  "epoch": 0.73,
1445
- "learning_rate": 1.0137191367132078e-07,
1446
- "logits/chosen": 0.2791319191455841,
1447
- "logits/rejected": 0.45174160599708557,
1448
- "logps/chosen": -372.1945495605469,
1449
- "logps/rejected": -446.6507263183594,
1450
- "loss": 0.5458,
1451
- "rewards/accuracies": 0.706250011920929,
1452
- "rewards/chosen": -2.3160648345947266,
1453
- "rewards/margins": 0.68004310131073,
1454
- "rewards/rejected": -2.996107816696167,
1455
- "step": 920
1456
- },
1457
- {
1458
- "epoch": 0.74,
1459
- "learning_rate": 9.584132603467827e-08,
1460
- "logits/chosen": -0.12192128598690033,
1461
- "logits/rejected": 0.1477951854467392,
1462
- "logps/chosen": -366.48321533203125,
1463
- "logps/rejected": -453.130126953125,
1464
- "loss": 0.5467,
1465
- "rewards/accuracies": 0.75,
1466
- "rewards/chosen": -2.200005531311035,
1467
- "rewards/margins": 0.7978888750076294,
1468
- "rewards/rejected": -2.997894287109375,
1469
- "step": 930
1470
  },
1471
  {
1472
  "epoch": 0.75,
1473
- "learning_rate": 9.042988532644249e-08,
1474
- "logits/chosen": -0.03106372058391571,
1475
- "logits/rejected": 0.07721444219350815,
1476
- "logps/chosen": -344.21270751953125,
1477
- "logps/rejected": -438.11077880859375,
1478
- "loss": 0.5161,
1479
- "rewards/accuracies": 0.71875,
1480
- "rewards/chosen": -2.149094581604004,
1481
- "rewards/margins": 0.7353444695472717,
1482
- "rewards/rejected": -2.884438991546631,
1483
- "step": 940
1484
- },
1485
- {
1486
- "epoch": 0.76,
1487
- "learning_rate": 8.514177396802428e-08,
1488
- "logits/chosen": 0.006801058538258076,
1489
- "logits/rejected": 0.20282092690467834,
1490
- "logps/chosen": -358.15167236328125,
1491
- "logps/rejected": -436.4964294433594,
1492
- "loss": 0.5385,
1493
- "rewards/accuracies": 0.7562500238418579,
1494
- "rewards/chosen": -2.2220425605773926,
1495
- "rewards/margins": 0.7004804611206055,
1496
- "rewards/rejected": -2.922523260116577,
1497
- "step": 950
1498
- },
1499
- {
1500
- "epoch": 0.76,
1501
- "learning_rate": 7.998107906142839e-08,
1502
- "logits/chosen": 0.41448846459388733,
1503
- "logits/rejected": 0.705254852771759,
1504
- "logps/chosen": -371.27801513671875,
1505
- "logps/rejected": -434.56866455078125,
1506
- "loss": 0.5236,
1507
- "rewards/accuracies": 0.7437499761581421,
1508
- "rewards/chosen": -2.2166616916656494,
1509
- "rewards/margins": 0.6714047193527222,
1510
- "rewards/rejected": -2.888066530227661,
1511
- "step": 960
1512
  },
1513
  {
1514
  "epoch": 0.77,
1515
- "learning_rate": 7.495178923039396e-08,
1516
- "logits/chosen": 0.23847150802612305,
1517
- "logits/rejected": 0.48661884665489197,
1518
- "logps/chosen": -366.28179931640625,
1519
- "logps/rejected": -462.679443359375,
1520
- "loss": 0.5459,
1521
- "rewards/accuracies": 0.7437499761581421,
1522
- "rewards/chosen": -2.1916985511779785,
1523
- "rewards/margins": 0.8472123146057129,
1524
- "rewards/rejected": -3.038910388946533,
1525
- "step": 970
1526
- },
1527
- {
1528
- "epoch": 0.78,
1529
- "learning_rate": 7.005779153764682e-08,
1530
- "logits/chosen": 0.41438961029052734,
1531
- "logits/rejected": 0.6912784576416016,
1532
- "logps/chosen": -382.70123291015625,
1533
- "logps/rejected": -461.8614807128906,
1534
- "loss": 0.5453,
1535
- "rewards/accuracies": 0.737500011920929,
1536
- "rewards/chosen": -2.4304287433624268,
1537
- "rewards/margins": 0.7116767764091492,
1538
- "rewards/rejected": -3.1421055793762207,
1539
- "step": 980
1540
  },
1541
  {
1542
  "epoch": 0.79,
1543
- "learning_rate": 6.530286848064698e-08,
1544
- "logits/chosen": 0.36573725938796997,
1545
- "logits/rejected": 0.5834362506866455,
1546
- "logps/chosen": -384.49749755859375,
1547
- "logps/rejected": -466.30096435546875,
1548
- "loss": 0.5528,
1549
- "rewards/accuracies": 0.71875,
1550
- "rewards/chosen": -2.5111565589904785,
1551
- "rewards/margins": 0.7234699130058289,
1552
- "rewards/rejected": -3.234626054763794,
1553
- "step": 990
1554
- },
1555
- {
1556
- "epoch": 0.8,
1557
- "learning_rate": 6.069069506815325e-08,
1558
- "logits/chosen": 0.45530566573143005,
1559
- "logits/rejected": 0.5909157991409302,
1560
- "logps/chosen": -379.1433410644531,
1561
- "logps/rejected": -468.88458251953125,
1562
- "loss": 0.527,
1563
- "rewards/accuracies": 0.762499988079071,
1564
- "rewards/chosen": -2.5361268520355225,
1565
- "rewards/margins": 0.7407721281051636,
1566
- "rewards/rejected": -3.2768986225128174,
1567
- "step": 1000
1568
- },
1569
- {
1570
- "epoch": 0.8,
1571
- "eval_logits/chosen": 0.3871051073074341,
1572
- "eval_logits/rejected": 0.6372014284133911,
1573
- "eval_logps/chosen": -394.97113037109375,
1574
- "eval_logps/rejected": -471.8453674316406,
1575
- "eval_loss": 0.5453863739967346,
1576
- "eval_rewards/accuracies": 0.70802241563797,
1577
- "eval_rewards/chosen": -2.503976345062256,
1578
- "eval_rewards/margins": 0.7031334638595581,
1579
- "eval_rewards/rejected": -3.2071101665496826,
1580
- "eval_runtime": 184.1335,
1581
- "eval_samples_per_second": 46.445,
1582
- "eval_steps_per_second": 0.728,
1583
- "step": 1000
1584
- },
1585
- {
1586
- "epoch": 0.8,
1587
- "learning_rate": 5.6224835979863714e-08,
1588
- "logits/chosen": 0.31174296140670776,
1589
- "logits/rejected": 0.6193565130233765,
1590
- "logps/chosen": -390.387451171875,
1591
- "logps/rejected": -468.4959411621094,
1592
- "loss": 0.5568,
1593
- "rewards/accuracies": 0.71875,
1594
- "rewards/chosen": -2.32766056060791,
1595
- "rewards/margins": 0.68747878074646,
1596
- "rewards/rejected": -3.015139102935791,
1597
- "step": 1010
1598
- },
1599
- {
1600
- "epoch": 0.81,
1601
- "learning_rate": 5.190874281132851e-08,
1602
- "logits/chosen": 0.22277125716209412,
1603
- "logits/rejected": 0.6487134099006653,
1604
- "logps/chosen": -402.0958557128906,
1605
- "logps/rejected": -448.5992736816406,
1606
- "loss": 0.5408,
1607
- "rewards/accuracies": 0.71875,
1608
- "rewards/chosen": -2.359062671661377,
1609
- "rewards/margins": 0.6533006429672241,
1610
- "rewards/rejected": -3.0123631954193115,
1611
- "step": 1020
1612
- },
1613
- {
1614
- "epoch": 0.82,
1615
- "learning_rate": 4.774575140626316e-08,
1616
- "logits/chosen": 0.23170511424541473,
1617
- "logits/rejected": 0.47184085845947266,
1618
- "logps/chosen": -363.46917724609375,
1619
- "logps/rejected": -442.47918701171875,
1620
- "loss": 0.5309,
1621
- "rewards/accuracies": 0.7437499761581421,
1622
- "rewards/chosen": -2.142770290374756,
1623
- "rewards/margins": 0.7513145208358765,
1624
- "rewards/rejected": -2.894084930419922,
1625
- "step": 1030
1626
- },
1627
- {
1628
- "epoch": 0.83,
1629
- "learning_rate": 4.373907927832513e-08,
1630
- "logits/chosen": 0.07573021948337555,
1631
- "logits/rejected": 0.32997313141822815,
1632
- "logps/chosen": -381.45599365234375,
1633
- "logps/rejected": -443.0684509277344,
1634
- "loss": 0.5407,
1635
  "rewards/accuracies": 0.7250000238418579,
1636
- "rewards/chosen": -2.2033116817474365,
1637
- "rewards/margins": 0.710732638835907,
1638
- "rewards/rejected": -2.914044141769409,
1639
- "step": 1040
1640
  },
1641
  {
1642
- "epoch": 0.84,
1643
- "learning_rate": 3.9891823124345665e-08,
1644
- "logits/chosen": 0.23884686827659607,
1645
- "logits/rejected": 0.6128005385398865,
1646
- "logps/chosen": -364.00567626953125,
1647
- "logps/rejected": -433.3273010253906,
1648
- "loss": 0.5471,
1649
  "rewards/accuracies": 0.731249988079071,
1650
- "rewards/chosen": -2.2081527709960938,
1651
- "rewards/margins": 0.7639263868331909,
1652
- "rewards/rejected": -2.972079038619995,
1653
- "step": 1050
1654
  },
1655
  {
1656
  "epoch": 0.84,
1657
- "learning_rate": 3.620695643093924e-08,
1658
- "logits/chosen": 0.21963253617286682,
1659
- "logits/rejected": 0.6894062757492065,
1660
- "logps/chosen": -399.5767517089844,
1661
- "logps/rejected": -452.88909912109375,
1662
- "loss": 0.5154,
1663
- "rewards/accuracies": 0.71875,
1664
- "rewards/chosen": -2.3384335041046143,
1665
- "rewards/margins": 0.7010769844055176,
1666
- "rewards/rejected": -3.0395102500915527,
1667
- "step": 1060
1668
  },
1669
  {
1670
- "epoch": 0.85,
1671
- "learning_rate": 3.268732717634032e-08,
1672
- "logits/chosen": 0.3474286198616028,
1673
- "logits/rejected": 0.695271372795105,
1674
- "logps/chosen": -368.0654602050781,
1675
- "logps/rejected": -431.47222900390625,
1676
- "loss": 0.5499,
1677
- "rewards/accuracies": 0.731249988079071,
1678
- "rewards/chosen": -2.1910276412963867,
1679
- "rewards/margins": 0.7267633080482483,
1680
- "rewards/rejected": -2.9177908897399902,
1681
- "step": 1070
 
 
1682
  },
1683
  {
1684
  "epoch": 0.86,
1685
- "learning_rate": 2.9335655629243645e-08,
1686
- "logits/chosen": 0.2347393035888672,
1687
- "logits/rejected": 0.5894696712493896,
1688
- "logps/chosen": -388.94757080078125,
1689
- "logps/rejected": -447.3855895996094,
1690
- "loss": 0.525,
1691
- "rewards/accuracies": 0.78125,
1692
- "rewards/chosen": -2.254683017730713,
1693
- "rewards/margins": 0.7334609031677246,
1694
- "rewards/rejected": -2.9881439208984375,
1695
- "step": 1080
1696
- },
1697
- {
1698
- "epoch": 0.87,
1699
- "learning_rate": 2.6154532246349476e-08,
1700
- "logits/chosen": 0.25378522276878357,
1701
- "logits/rejected": 0.5771256685256958,
1702
- "logps/chosen": -358.50640869140625,
1703
- "logps/rejected": -431.145751953125,
1704
- "loss": 0.5462,
1705
- "rewards/accuracies": 0.7250000238418579,
1706
- "rewards/chosen": -2.1666626930236816,
1707
- "rewards/margins": 0.756801426410675,
1708
- "rewards/rejected": -2.923464059829712,
1709
- "step": 1090
1710
- },
1711
- {
1712
- "epoch": 0.88,
1713
- "learning_rate": 2.31464156702382e-08,
1714
- "logits/chosen": 0.35370689630508423,
1715
- "logits/rejected": 0.5671936273574829,
1716
- "logps/chosen": -363.0,
1717
- "logps/rejected": -438.209228515625,
1718
- "loss": 0.5487,
1719
- "rewards/accuracies": 0.7250000238418579,
1720
- "rewards/chosen": -2.2130119800567627,
1721
- "rewards/margins": 0.7499077916145325,
1722
- "rewards/rejected": -2.9629194736480713,
1723
- "step": 1100
1724
- },
1725
- {
1726
- "epoch": 0.88,
1727
- "eval_logits/chosen": 0.1857856959104538,
1728
- "eval_logits/rejected": 0.43363669514656067,
1729
- "eval_logps/chosen": -373.08306884765625,
1730
- "eval_logps/rejected": -450.7598876953125,
1731
- "eval_loss": 0.5444055199623108,
1732
- "eval_rewards/accuracies": 0.7089552283287048,
1733
- "eval_rewards/chosen": -2.285095453262329,
1734
- "eval_rewards/margins": 0.711159884929657,
1735
- "eval_rewards/rejected": -2.996255397796631,
1736
- "eval_runtime": 184.1088,
1737
- "eval_samples_per_second": 46.451,
1738
- "eval_steps_per_second": 0.728,
1739
- "step": 1100
1740
  },
1741
  {
1742
  "epoch": 0.88,
1743
- "learning_rate": 2.031363082912252e-08,
1744
- "logits/chosen": 0.070524200797081,
1745
- "logits/rejected": 0.4635602533817291,
1746
- "logps/chosen": -373.29327392578125,
1747
- "logps/rejected": -426.85552978515625,
1748
- "loss": 0.5513,
1749
- "rewards/accuracies": 0.6937500238418579,
1750
- "rewards/chosen": -2.2541089057922363,
1751
- "rewards/margins": 0.6198171973228455,
1752
- "rewards/rejected": -2.8739261627197266,
1753
- "step": 1110
1754
- },
1755
- {
1756
- "epoch": 0.89,
1757
- "learning_rate": 1.7658367139945228e-08,
1758
- "logits/chosen": 0.2600646913051605,
1759
- "logits/rejected": 0.5517584681510925,
1760
- "logps/chosen": -390.8568115234375,
1761
- "logps/rejected": -462.80828857421875,
1762
- "loss": 0.5471,
1763
- "rewards/accuracies": 0.6875,
1764
- "rewards/chosen": -2.377396821975708,
1765
- "rewards/margins": 0.6719989776611328,
1766
- "rewards/rejected": -3.049395799636841,
1767
- "step": 1120
1768
  },
1769
  {
1770
  "epoch": 0.9,
1771
- "learning_rate": 1.5182676816211632e-08,
1772
- "logits/chosen": 0.04413030296564102,
1773
- "logits/rejected": 0.30151715874671936,
1774
- "logps/chosen": -382.0662536621094,
1775
- "logps/rejected": -447.08673095703125,
1776
- "loss": 0.5431,
1777
- "rewards/accuracies": 0.7124999761581421,
1778
- "rewards/chosen": -2.228654384613037,
1779
- "rewards/margins": 0.6926024556159973,
1780
- "rewards/rejected": -2.9212570190429688,
1781
- "step": 1130
1782
- },
1783
- {
1784
- "epoch": 0.91,
1785
- "learning_rate": 1.2888473281864597e-08,
1786
- "logits/chosen": 0.14212054014205933,
1787
- "logits/rejected": 0.47429710626602173,
1788
- "logps/chosen": -367.8409729003906,
1789
- "logps/rejected": -435.02764892578125,
1790
- "loss": 0.5369,
1791
- "rewards/accuracies": 0.768750011920929,
1792
- "rewards/chosen": -2.2534115314483643,
1793
- "rewards/margins": 0.7273036241531372,
1794
- "rewards/rejected": -2.980715274810791,
1795
- "step": 1140
1796
- },
1797
- {
1798
- "epoch": 0.92,
1799
- "learning_rate": 1.0777529692427679e-08,
1800
- "logits/chosen": 0.04115242511034012,
1801
- "logits/rejected": 0.28970104455947876,
1802
- "logps/chosen": -372.7949523925781,
1803
- "logps/rejected": -456.10675048828125,
1804
- "loss": 0.5265,
1805
- "rewards/accuracies": 0.731249988079071,
1806
- "rewards/chosen": -2.300356388092041,
1807
- "rewards/margins": 0.8059718012809753,
1808
- "rewards/rejected": -3.106328248977661,
1809
- "step": 1150
1810
  },
1811
  {
1812
  "epoch": 0.92,
1813
- "learning_rate": 8.851477564560061e-09,
1814
- "logits/chosen": 0.0867738351225853,
1815
- "logits/rejected": 0.4068300127983093,
1816
- "logps/chosen": -372.08636474609375,
1817
- "logps/rejected": -426.42388916015625,
1818
- "loss": 0.5342,
1819
- "rewards/accuracies": 0.6937500238418579,
1820
- "rewards/chosen": -2.331385850906372,
1821
- "rewards/margins": 0.6490964293479919,
1822
- "rewards/rejected": -2.9804821014404297,
1823
- "step": 1160
1824
- },
1825
- {
1826
- "epoch": 0.93,
1827
- "learning_rate": 7.111805515081531e-09,
1828
- "logits/chosen": 0.02022993005812168,
1829
- "logits/rejected": 0.41968393325805664,
1830
- "logps/chosen": -363.818603515625,
1831
- "logps/rejected": -447.7919006347656,
1832
- "loss": 0.5312,
1833
- "rewards/accuracies": 0.7562500238418579,
1834
- "rewards/chosen": -2.2372307777404785,
1835
- "rewards/margins": 0.8540315628051758,
1836
- "rewards/rejected": -3.0912623405456543,
1837
- "step": 1170
1838
  },
1839
  {
1840
  "epoch": 0.94,
1841
- "learning_rate": 5.559858110443016e-09,
1842
- "logits/chosen": 0.29695388674736023,
1843
- "logits/rejected": 0.714096188545227,
1844
- "logps/chosen": -372.5519714355469,
1845
- "logps/rejected": -442.5354919433594,
1846
- "loss": 0.5383,
1847
- "rewards/accuracies": 0.71875,
1848
- "rewards/chosen": -2.3107995986938477,
1849
- "rewards/margins": 0.8070123791694641,
1850
- "rewards/rejected": -3.117811918258667,
1851
- "step": 1180
1852
- },
1853
- {
1854
- "epoch": 0.95,
1855
- "learning_rate": 4.196834827531276e-09,
1856
- "logits/chosen": 0.140055850148201,
1857
- "logits/rejected": 0.3409932255744934,
1858
- "logps/chosen": -355.64324951171875,
1859
- "logps/rejected": -447.585693359375,
1860
- "loss": 0.5152,
1861
- "rewards/accuracies": 0.737500011920929,
1862
- "rewards/chosen": -2.149151563644409,
1863
- "rewards/margins": 0.7904965877532959,
1864
- "rewards/rejected": -2.939648151397705,
1865
- "step": 1190
1866
- },
1867
- {
1868
- "epoch": 0.96,
1869
- "learning_rate": 3.023789126611137e-09,
1870
- "logits/chosen": 0.03294936567544937,
1871
- "logits/rejected": 0.2933207154273987,
1872
- "logps/chosen": -363.29290771484375,
1873
- "logps/rejected": -435.640380859375,
1874
- "loss": 0.5483,
1875
- "rewards/accuracies": 0.699999988079071,
1876
- "rewards/chosen": -2.281057357788086,
1877
- "rewards/margins": 0.7091296911239624,
1878
- "rewards/rejected": -2.990186929702759,
1879
- "step": 1200
1880
- },
1881
- {
1882
- "epoch": 0.96,
1883
- "eval_logits/chosen": 0.07418080419301987,
1884
- "eval_logits/rejected": 0.32435521483421326,
1885
- "eval_logps/chosen": -373.978515625,
1886
- "eval_logps/rejected": -451.6764831542969,
1887
- "eval_loss": 0.5440130829811096,
1888
- "eval_rewards/accuracies": 0.7089552283287048,
1889
- "eval_rewards/chosen": -2.2940499782562256,
1890
- "eval_rewards/margins": 0.7113713622093201,
1891
- "eval_rewards/rejected": -3.0054211616516113,
1892
- "eval_runtime": 184.1178,
1893
- "eval_samples_per_second": 46.449,
1894
- "eval_steps_per_second": 0.728,
1895
- "step": 1200
1896
  },
1897
  {
1898
  "epoch": 0.96,
1899
- "learning_rate": 2.041627637121929e-09,
1900
- "logits/chosen": 0.10010697692632675,
1901
- "logits/rejected": 0.3795483410358429,
1902
- "logps/chosen": -348.8675231933594,
1903
- "logps/rejected": -437.20361328125,
1904
- "loss": 0.5398,
1905
- "rewards/accuracies": 0.7749999761581421,
1906
- "rewards/chosen": -2.087364673614502,
1907
- "rewards/margins": 0.828387439250946,
1908
- "rewards/rejected": -2.9157521724700928,
1909
- "step": 1210
1910
- },
1911
- {
1912
- "epoch": 0.97,
1913
- "learning_rate": 1.2511094569571668e-09,
1914
- "logits/chosen": 0.09991980344057083,
1915
- "logits/rejected": 0.4467397630214691,
1916
- "logps/chosen": -380.14520263671875,
1917
- "logps/rejected": -440.24658203125,
1918
- "loss": 0.5345,
1919
- "rewards/accuracies": 0.737500011920929,
1920
- "rewards/chosen": -2.253425121307373,
1921
- "rewards/margins": 0.702509880065918,
1922
- "rewards/rejected": -2.955935001373291,
1923
- "step": 1220
1924
  },
1925
  {
1926
  "epoch": 0.98,
1927
- "learning_rate": 6.528455657691112e-10,
1928
- "logits/chosen": 0.11626466363668442,
1929
- "logits/rejected": 0.41348797082901,
1930
- "logps/chosen": -372.7298889160156,
1931
- "logps/rejected": -427.22576904296875,
1932
- "loss": 0.549,
1933
- "rewards/accuracies": 0.762499988079071,
1934
- "rewards/chosen": -2.2800345420837402,
1935
- "rewards/margins": 0.6291176080703735,
1936
- "rewards/rejected": -2.909151792526245,
1937
- "step": 1230
1938
- },
1939
- {
1940
- "epoch": 0.99,
1941
- "learning_rate": 2.4729835275189016e-10,
1942
- "logits/chosen": 0.06715863198041916,
1943
- "logits/rejected": 0.29241910576820374,
1944
- "logps/chosen": -393.8903503417969,
1945
- "logps/rejected": -477.9420471191406,
1946
- "loss": 0.5462,
1947
- "rewards/accuracies": 0.737500011920929,
1948
- "rewards/chosen": -2.323488473892212,
1949
- "rewards/margins": 0.8067766427993774,
1950
- "rewards/rejected": -3.1302647590637207,
1951
- "step": 1240
1952
- },
1953
- {
1954
- "epoch": 0.99,
1955
- "learning_rate": 3.478125926756337e-11,
1956
- "logits/chosen": 0.25983649492263794,
1957
- "logits/rejected": 0.4905417561531067,
1958
- "logps/chosen": -364.73431396484375,
1959
- "logps/rejected": -443.79296875,
1960
- "loss": 0.5474,
1961
- "rewards/accuracies": 0.7250000238418579,
1962
- "rewards/chosen": -2.2353272438049316,
1963
- "rewards/margins": 0.771331787109375,
1964
- "rewards/rejected": -3.0066590309143066,
1965
- "step": 1250
1966
  },
1967
  {
1968
  "epoch": 1.0,
1969
- "step": 1256,
1970
  "total_flos": 0.0,
1971
- "train_loss": 0.5712926928784438,
1972
- "train_runtime": 11451.0976,
1973
- "train_samples_per_second": 14.042,
1974
- "train_steps_per_second": 0.11
1975
  }
1976
  ],
1977
  "logging_steps": 10,
1978
- "max_steps": 1256,
1979
  "num_train_epochs": 1,
1980
  "save_steps": 100,
1981
  "total_flos": 0.0,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
  "eval_steps": 100,
6
+ "global_step": 478,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.0,
13
+ "learning_rate": 1.0416666666666666e-08,
14
+ "logits/chosen": -2.8386030197143555,
15
+ "logits/rejected": -2.823939323425293,
16
+ "logps/chosen": -324.3727722167969,
17
+ "logps/rejected": -231.64634704589844,
18
+ "loss": 0.1918,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
23
  "step": 1
24
  },
25
  {
26
+ "epoch": 0.02,
27
+ "learning_rate": 1.0416666666666667e-07,
28
+ "logits/chosen": -2.824544668197632,
29
+ "logits/rejected": -2.7506117820739746,
30
+ "logps/chosen": -275.7240295410156,
31
+ "logps/rejected": -253.47032165527344,
32
+ "loss": 0.1927,
33
+ "rewards/accuracies": 0.5069444179534912,
34
+ "rewards/chosen": 0.0003705685958266258,
35
+ "rewards/margins": 0.0005802758387289941,
36
+ "rewards/rejected": -0.00020970719924662262,
37
  "step": 10
38
  },
39
  {
40
+ "epoch": 0.04,
41
+ "learning_rate": 2.0833333333333333e-07,
42
+ "logits/chosen": -2.7971882820129395,
43
+ "logits/rejected": -2.779716968536377,
44
+ "logps/chosen": -261.8994445800781,
45
+ "logps/rejected": -257.0425720214844,
46
+ "loss": 0.1928,
47
+ "rewards/accuracies": 0.574999988079071,
48
+ "rewards/chosen": 0.0005474781501106918,
49
+ "rewards/margins": 0.0016361282905563712,
50
+ "rewards/rejected": -0.001088650431483984,
51
  "step": 20
52
  },
53
  {
54
+ "epoch": 0.06,
55
+ "learning_rate": 3.1249999999999997e-07,
56
+ "logits/chosen": -2.7837774753570557,
57
+ "logits/rejected": -2.777256488800049,
58
+ "logps/chosen": -294.86572265625,
59
+ "logps/rejected": -259.10992431640625,
60
+ "loss": 0.1924,
61
+ "rewards/accuracies": 0.6625000238418579,
62
+ "rewards/chosen": 0.0020633183885365725,
63
+ "rewards/margins": 0.010893032886087894,
64
+ "rewards/rejected": -0.008829714730381966,
65
  "step": 30
66
  },
67
  {
68
+ "epoch": 0.08,
69
+ "learning_rate": 4.1666666666666667e-07,
70
+ "logits/chosen": -2.8023006916046143,
71
+ "logits/rejected": -2.7713534832000732,
72
+ "logps/chosen": -284.7164611816406,
73
+ "logps/rejected": -264.82354736328125,
74
+ "loss": 0.1868,
75
+ "rewards/accuracies": 0.6875,
76
+ "rewards/chosen": -0.008054533042013645,
77
+ "rewards/margins": 0.02058727666735649,
78
+ "rewards/rejected": -0.02864181064069271,
79
  "step": 40
80
  },
81
  {
82
+ "epoch": 0.1,
83
+ "learning_rate": 4.999733114418725e-07,
84
+ "logits/chosen": -2.7800755500793457,
85
+ "logits/rejected": -2.7177579402923584,
86
+ "logps/chosen": -284.4308166503906,
87
+ "logps/rejected": -254.9806365966797,
88
+ "loss": 0.1743,
89
+ "rewards/accuracies": 0.737500011920929,
90
+ "rewards/chosen": -0.02050088346004486,
91
+ "rewards/margins": 0.06260053813457489,
92
+ "rewards/rejected": -0.08310142904520035,
93
  "step": 50
94
  },
95
  {
96
+ "epoch": 0.13,
97
+ "learning_rate": 4.990398100856366e-07,
98
+ "logits/chosen": -2.807746410369873,
99
+ "logits/rejected": -2.7858967781066895,
100
+ "logps/chosen": -296.1127014160156,
101
+ "logps/rejected": -264.55828857421875,
102
+ "loss": 0.1597,
103
+ "rewards/accuracies": 0.612500011920929,
104
+ "rewards/chosen": -0.09463275969028473,
105
+ "rewards/margins": 0.07932767271995544,
106
+ "rewards/rejected": -0.17396041750907898,
107
  "step": 60
108
  },
109
  {
110
+ "epoch": 0.15,
111
+ "learning_rate": 4.967775735898179e-07,
112
+ "logits/chosen": -2.70173978805542,
113
+ "logits/rejected": -2.665942430496216,
114
+ "logps/chosen": -275.4799499511719,
115
+ "logps/rejected": -269.7330627441406,
116
+ "loss": 0.1463,
117
+ "rewards/accuracies": 0.71875,
118
+ "rewards/chosen": -0.1543087661266327,
119
+ "rewards/margins": 0.1567164957523346,
120
+ "rewards/rejected": -0.3110252916812897,
121
  "step": 70
122
  },
123
  {
124
+ "epoch": 0.17,
125
+ "learning_rate": 4.931986719649298e-07,
126
+ "logits/chosen": -2.7185983657836914,
127
+ "logits/rejected": -2.701369047164917,
128
+ "logps/chosen": -297.1450500488281,
129
+ "logps/rejected": -291.1013488769531,
130
+ "loss": 0.1291,
131
+ "rewards/accuracies": 0.65625,
132
+ "rewards/chosen": -0.293169230222702,
133
+ "rewards/margins": 0.154391810297966,
134
+ "rewards/rejected": -0.4475610852241516,
135
  "step": 80
136
  },
137
  {
138
+ "epoch": 0.19,
139
+ "learning_rate": 4.883222001996351e-07,
140
+ "logits/chosen": -2.751041889190674,
141
+ "logits/rejected": -2.7259504795074463,
142
+ "logps/chosen": -330.4771728515625,
143
+ "logps/rejected": -344.82342529296875,
144
+ "loss": 0.0953,
145
+ "rewards/accuracies": 0.675000011920929,
146
+ "rewards/chosen": -0.45313897728919983,
147
+ "rewards/margins": 0.3178847134113312,
148
+ "rewards/rejected": -0.771023690700531,
149
  "step": 90
150
  },
151
  {
152
+ "epoch": 0.21,
153
+ "learning_rate": 4.821741763807186e-07,
154
+ "logits/chosen": -2.6622979640960693,
155
+ "logits/rejected": -2.6552236080169678,
156
+ "logps/chosen": -340.332275390625,
157
+ "logps/rejected": -333.6282043457031,
158
+ "loss": 0.1041,
159
+ "rewards/accuracies": 0.643750011920929,
160
+ "rewards/chosen": -0.4600113034248352,
161
+ "rewards/margins": 0.23454304039478302,
162
+ "rewards/rejected": -0.6945544481277466,
163
  "step": 100
164
  },
165
  {
166
+ "epoch": 0.21,
167
+ "eval_logits/chosen": -2.787762403488159,
168
+ "eval_logits/rejected": -2.767549753189087,
169
+ "eval_logps/chosen": -297.7511291503906,
170
+ "eval_logps/rejected": -336.2553405761719,
171
+ "eval_loss": 0.10276182740926743,
172
+ "eval_rewards/accuracies": 0.73828125,
173
+ "eval_rewards/chosen": -0.4071148931980133,
174
+ "eval_rewards/margins": 0.3819059133529663,
175
+ "eval_rewards/rejected": -0.7890207767486572,
176
+ "eval_runtime": 53.3695,
177
+ "eval_samples_per_second": 37.475,
178
+ "eval_steps_per_second": 0.6,
179
  "step": 100
180
  },
181
  {
182
+ "epoch": 0.23,
183
+ "learning_rate": 4.747874028753375e-07,
184
+ "logits/chosen": -2.7773475646972656,
185
+ "logits/rejected": -2.723445415496826,
186
+ "logps/chosen": -323.4300842285156,
187
+ "logps/rejected": -320.19635009765625,
188
+ "loss": 0.0872,
189
+ "rewards/accuracies": 0.706250011920929,
190
+ "rewards/chosen": -0.48955243825912476,
191
+ "rewards/margins": 0.42650121450424194,
192
+ "rewards/rejected": -0.9160536527633667,
193
  "step": 110
194
  },
195
  {
196
+ "epoch": 0.25,
197
+ "learning_rate": 4.662012913161997e-07,
198
+ "logits/chosen": -2.7073817253112793,
199
+ "logits/rejected": -2.6860263347625732,
200
+ "logps/chosen": -342.6280212402344,
201
+ "logps/rejected": -382.22393798828125,
202
+ "loss": 0.0765,
203
+ "rewards/accuracies": 0.6875,
204
+ "rewards/chosen": -0.7917813062667847,
205
+ "rewards/margins": 0.421453058719635,
206
+ "rewards/rejected": -1.213234305381775,
207
  "step": 120
208
  },
209
  {
210
+ "epoch": 0.27,
211
+ "learning_rate": 4.5646165232345103e-07,
212
+ "logits/chosen": -2.7049379348754883,
213
+ "logits/rejected": -2.6863231658935547,
214
+ "logps/chosen": -354.7309265136719,
215
+ "logps/rejected": -386.9947204589844,
216
+ "loss": 0.0634,
217
+ "rewards/accuracies": 0.6875,
218
+ "rewards/chosen": -0.9187179803848267,
219
+ "rewards/margins": 0.4910743832588196,
220
+ "rewards/rejected": -1.409792184829712,
221
  "step": 130
222
  },
223
  {
224
+ "epoch": 0.29,
225
+ "learning_rate": 4.456204510851956e-07,
226
+ "logits/chosen": -2.647672653198242,
227
+ "logits/rejected": -2.6301255226135254,
228
+ "logps/chosen": -345.8076477050781,
229
+ "logps/rejected": -359.37530517578125,
230
+ "loss": 0.0671,
231
+ "rewards/accuracies": 0.6875,
232
+ "rewards/chosen": -0.91509610414505,
233
+ "rewards/margins": 0.3521556854248047,
234
+ "rewards/rejected": -1.26725172996521,
235
  "step": 140
236
  },
237
  {
238
+ "epoch": 0.31,
239
+ "learning_rate": 4.337355301007335e-07,
240
+ "logits/chosen": -2.5693020820617676,
241
+ "logits/rejected": -2.545379877090454,
242
+ "logps/chosen": -357.9366455078125,
243
+ "logps/rejected": -380.52459716796875,
244
+ "loss": 0.0651,
245
+ "rewards/accuracies": 0.6875,
246
+ "rewards/chosen": -0.8748415112495422,
247
+ "rewards/margins": 0.3778706192970276,
248
+ "rewards/rejected": -1.2527120113372803,
249
  "step": 150
250
  },
251
  {
252
+ "epoch": 0.33,
253
+ "learning_rate": 4.2087030056579986e-07,
254
+ "logits/chosen": -2.55464506149292,
255
+ "logits/rejected": -2.546234607696533,
256
+ "logps/chosen": -376.0520935058594,
257
+ "logps/rejected": -412.2450256347656,
258
+ "loss": 0.0576,
259
  "rewards/accuracies": 0.6499999761581421,
260
+ "rewards/chosen": -1.1176189184188843,
261
+ "rewards/margins": 0.3281309902667999,
262
+ "rewards/rejected": -1.4457499980926514,
263
  "step": 160
264
  },
265
  {
266
+ "epoch": 0.36,
267
+ "learning_rate": 4.070934040463998e-07,
268
+ "logits/chosen": -2.532057285308838,
269
+ "logits/rejected": -2.4876461029052734,
270
+ "logps/chosen": -376.68865966796875,
271
+ "logps/rejected": -378.6046142578125,
272
+ "loss": 0.0616,
273
+ "rewards/accuracies": 0.6187499761581421,
274
+ "rewards/chosen": -1.052880048751831,
275
+ "rewards/margins": 0.3885950744152069,
276
+ "rewards/rejected": -1.4414750337600708,
277
  "step": 170
278
  },
279
  {
280
+ "epoch": 0.38,
281
+ "learning_rate": 3.9247834624635404e-07,
282
+ "logits/chosen": -2.54783296585083,
283
+ "logits/rejected": -2.5421698093414307,
284
+ "logps/chosen": -329.34271240234375,
285
+ "logps/rejected": -376.6134338378906,
286
+ "loss": 0.0672,
287
+ "rewards/accuracies": 0.6499999761581421,
288
+ "rewards/chosen": -0.9524626731872559,
289
+ "rewards/margins": 0.43748727440834045,
290
+ "rewards/rejected": -1.389950156211853,
291
  "step": 180
292
  },
293
  {
294
+ "epoch": 0.4,
295
+ "learning_rate": 3.7710310482256523e-07,
296
+ "logits/chosen": -2.575817584991455,
297
+ "logits/rejected": -2.5711379051208496,
298
+ "logps/chosen": -369.71002197265625,
299
+ "logps/rejected": -397.71551513671875,
300
+ "loss": 0.0522,
301
+ "rewards/accuracies": 0.675000011920929,
302
+ "rewards/chosen": -1.144073724746704,
303
+ "rewards/margins": 0.3777850270271301,
304
+ "rewards/rejected": -1.5218586921691895,
305
  "step": 190
306
  },
307
  {
308
+ "epoch": 0.42,
309
+ "learning_rate": 3.610497133404795e-07,
310
+ "logits/chosen": -2.5250983238220215,
311
+ "logits/rejected": -2.5394480228424072,
312
+ "logps/chosen": -409.21258544921875,
313
+ "logps/rejected": -433.49365234375,
314
+ "loss": 0.0503,
315
+ "rewards/accuracies": 0.5874999761581421,
316
+ "rewards/chosen": -1.3282392024993896,
317
+ "rewards/margins": 0.30823636054992676,
318
+ "rewards/rejected": -1.6364758014678955,
319
  "step": 200
320
  },
321
  {
322
+ "epoch": 0.42,
323
+ "eval_logits/chosen": -2.556122303009033,
324
+ "eval_logits/rejected": -2.5374484062194824,
325
+ "eval_logps/chosen": -385.5240478515625,
326
+ "eval_logps/rejected": -441.7351989746094,
327
+ "eval_loss": 0.050499577075242996,
328
+ "eval_rewards/accuracies": 0.70703125,
329
+ "eval_rewards/chosen": -1.2848442792892456,
330
+ "eval_rewards/margins": 0.558975100517273,
331
+ "eval_rewards/rejected": -1.843819499015808,
332
+ "eval_runtime": 53.3391,
333
+ "eval_samples_per_second": 37.496,
334
+ "eval_steps_per_second": 0.6,
335
  "step": 200
336
  },
337
  {
338
+ "epoch": 0.44,
339
+ "learning_rate": 3.4440382358952115e-07,
340
+ "logits/chosen": -2.479553699493408,
341
+ "logits/rejected": -2.4708049297332764,
342
+ "logps/chosen": -385.39190673828125,
343
+ "logps/rejected": -417.0789489746094,
344
+ "loss": 0.0546,
345
+ "rewards/accuracies": 0.6187499761581421,
346
+ "rewards/chosen": -1.278079867362976,
347
+ "rewards/margins": 0.4998950958251953,
348
+ "rewards/rejected": -1.7779748439788818,
349
  "step": 210
350
  },
351
  {
352
+ "epoch": 0.46,
353
+ "learning_rate": 3.272542485937368e-07,
354
+ "logits/chosen": -2.581570863723755,
355
+ "logits/rejected": -2.5581154823303223,
356
+ "logps/chosen": -366.68414306640625,
357
+ "logps/rejected": -386.2957763671875,
358
+ "loss": 0.0632,
359
  "rewards/accuracies": 0.6812499761581421,
360
+ "rewards/chosen": -0.9979816675186157,
361
+ "rewards/margins": 0.4385088086128235,
362
+ "rewards/rejected": -1.436490535736084,
363
  "step": 220
364
  },
365
  {
366
+ "epoch": 0.48,
367
+ "learning_rate": 3.096924887558854e-07,
368
+ "logits/chosen": -2.5714306831359863,
369
+ "logits/rejected": -2.528597593307495,
370
+ "logps/chosen": -413.03973388671875,
371
+ "logps/rejected": -402.9679260253906,
372
+ "loss": 0.0603,
373
+ "rewards/accuracies": 0.699999988079071,
374
+ "rewards/chosen": -0.9944814443588257,
375
+ "rewards/margins": 0.43937864899635315,
376
+ "rewards/rejected": -1.4338600635528564,
377
  "step": 230
378
  },
379
  {
380
+ "epoch": 0.5,
381
+ "learning_rate": 2.9181224366319943e-07,
382
+ "logits/chosen": -2.4398398399353027,
383
+ "logits/rejected": -2.401116371154785,
384
+ "logps/chosen": -392.19744873046875,
385
+ "logps/rejected": -401.7131652832031,
386
+ "loss": 0.0552,
387
+ "rewards/accuracies": 0.699999988079071,
388
+ "rewards/chosen": -1.2129409313201904,
389
+ "rewards/margins": 0.45234808325767517,
390
+ "rewards/rejected": -1.6652891635894775,
391
  "step": 240
392
  },
393
  {
394
+ "epoch": 0.52,
395
+ "learning_rate": 2.7370891215954565e-07,
396
+ "logits/chosen": -2.4040493965148926,
397
+ "logits/rejected": -2.3864128589630127,
398
+ "logps/chosen": -413.65924072265625,
399
+ "logps/rejected": -434.7686462402344,
400
+ "loss": 0.0467,
401
+ "rewards/accuracies": 0.699999988079071,
402
+ "rewards/chosen": -1.365596055984497,
403
+ "rewards/margins": 0.563629150390625,
404
+ "rewards/rejected": -1.9292253255844116,
405
  "step": 250
406
  },
407
  {
408
+ "epoch": 0.54,
409
+ "learning_rate": 2.55479083351317e-07,
410
+ "logits/chosen": -2.4468510150909424,
411
+ "logits/rejected": -2.4228568077087402,
412
+ "logps/chosen": -423.4908752441406,
413
+ "logps/rejected": -464.5816955566406,
414
+ "loss": 0.0505,
415
+ "rewards/accuracies": 0.75,
416
+ "rewards/chosen": -1.3393833637237549,
417
+ "rewards/margins": 0.726915180683136,
418
+ "rewards/rejected": -2.066298484802246,
419
  "step": 260
420
  },
421
  {
422
+ "epoch": 0.56,
423
+ "learning_rate": 2.3722002126275822e-07,
424
+ "logits/chosen": -2.3682193756103516,
425
+ "logits/rejected": -2.3572616577148438,
426
+ "logps/chosen": -397.89862060546875,
427
+ "logps/rejected": -425.05035400390625,
428
+ "loss": 0.0506,
429
+ "rewards/accuracies": 0.675000011920929,
430
+ "rewards/chosen": -1.2890106439590454,
431
+ "rewards/margins": 0.541452169418335,
432
+ "rewards/rejected": -1.8304630517959595,
433
  "step": 270
434
  },
435
  {
436
+ "epoch": 0.59,
437
+ "learning_rate": 2.19029145890313e-07,
438
+ "logits/chosen": -2.3844916820526123,
439
+ "logits/rejected": -2.3584766387939453,
440
+ "logps/chosen": -406.8147888183594,
441
+ "logps/rejected": -452.04541015625,
442
+ "loss": 0.0447,
443
+ "rewards/accuracies": 0.731249988079071,
444
+ "rewards/chosen": -1.516322374343872,
445
+ "rewards/margins": 0.6245102882385254,
446
+ "rewards/rejected": -2.1408326625823975,
447
  "step": 280
448
  },
449
  {
450
+ "epoch": 0.61,
451
+ "learning_rate": 2.0100351342479216e-07,
452
+ "logits/chosen": -2.389385461807251,
453
+ "logits/rejected": -2.3362462520599365,
454
+ "logps/chosen": -440.2704162597656,
455
+ "logps/rejected": -452.8388671875,
456
+ "loss": 0.0452,
457
  "rewards/accuracies": 0.6875,
458
+ "rewards/chosen": -1.549391746520996,
459
+ "rewards/margins": 0.5749727487564087,
460
+ "rewards/rejected": -2.1243643760681152,
461
  "step": 290
462
  },
463
  {
464
+ "epoch": 0.63,
465
+ "learning_rate": 1.8323929841460178e-07,
466
+ "logits/chosen": -2.4288883209228516,
467
+ "logits/rejected": -2.40447735786438,
468
+ "logps/chosen": -411.8253479003906,
469
+ "logps/rejected": -468.8287048339844,
470
+ "loss": 0.0486,
471
+ "rewards/accuracies": 0.731249988079071,
472
+ "rewards/chosen": -1.2203395366668701,
473
+ "rewards/margins": 0.628702700138092,
474
+ "rewards/rejected": -1.849042534828186,
475
  "step": 300
476
  },
477
  {
478
+ "epoch": 0.63,
479
+ "eval_logits/chosen": -2.4070372581481934,
480
+ "eval_logits/rejected": -2.389354944229126,
481
+ "eval_logps/chosen": -377.24566650390625,
482
+ "eval_logps/rejected": -445.87652587890625,
483
+ "eval_loss": 0.05445738509297371,
484
+ "eval_rewards/accuracies": 0.7265625,
485
+ "eval_rewards/chosen": -1.2020601034164429,
486
+ "eval_rewards/margins": 0.6831726431846619,
487
+ "eval_rewards/rejected": -1.88523268699646,
488
+ "eval_runtime": 53.3474,
489
+ "eval_samples_per_second": 37.49,
490
+ "eval_steps_per_second": 0.6,
491
  "step": 300
492
  },
493
  {
494
+ "epoch": 0.65,
495
+ "learning_rate": 1.6583128063291573e-07,
496
+ "logits/chosen": -2.397949695587158,
497
+ "logits/rejected": -2.3695032596588135,
498
+ "logps/chosen": -391.604736328125,
499
+ "logps/rejected": -428.22198486328125,
500
+ "loss": 0.0504,
501
  "rewards/accuracies": 0.6875,
502
+ "rewards/chosen": -1.2964208126068115,
503
+ "rewards/margins": 0.5123800039291382,
504
+ "rewards/rejected": -1.8088006973266602,
505
  "step": 310
506
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
507
  {
508
  "epoch": 0.67,
509
+ "learning_rate": 1.488723393865766e-07,
510
+ "logits/chosen": -2.315542221069336,
511
+ "logits/rejected": -2.306222915649414,
512
+ "logps/chosen": -423.572265625,
513
+ "logps/rejected": -455.32305908203125,
514
+ "loss": 0.0443,
515
+ "rewards/accuracies": 0.6937500238418579,
516
+ "rewards/chosen": -1.4080761671066284,
517
+ "rewards/margins": 0.48968425393104553,
518
+ "rewards/rejected": -1.8977606296539307,
519
+ "step": 320
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
520
  },
521
  {
522
  "epoch": 0.69,
523
+ "learning_rate": 1.3245295796480788e-07,
524
+ "logits/chosen": -2.3131089210510254,
525
+ "logits/rejected": -2.307924747467041,
526
+ "logps/chosen": -387.47515869140625,
527
+ "logps/rejected": -460.91705322265625,
528
+ "loss": 0.0449,
529
+ "rewards/accuracies": 0.7250000238418579,
530
+ "rewards/chosen": -1.3790541887283325,
531
+ "rewards/margins": 0.6737614274024963,
532
+ "rewards/rejected": -2.0528156757354736,
533
+ "step": 330
 
 
 
 
 
 
 
 
 
 
 
 
 
 
534
  },
535
  {
536
  "epoch": 0.71,
537
+ "learning_rate": 1.1666074087171627e-07,
538
+ "logits/chosen": -2.325040578842163,
539
+ "logits/rejected": -2.325911283493042,
540
+ "logps/chosen": -433.72857666015625,
541
+ "logps/rejected": -475.69390869140625,
542
+ "loss": 0.0502,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
543
  "rewards/accuracies": 0.7124999761581421,
544
+ "rewards/chosen": -1.4316760301589966,
545
+ "rewards/margins": 0.6238799691200256,
546
+ "rewards/rejected": -2.055556058883667,
547
+ "step": 340
548
  },
549
  {
550
  "epoch": 0.73,
551
+ "learning_rate": 1.0157994641835734e-07,
552
+ "logits/chosen": -2.361835479736328,
553
+ "logits/rejected": -2.322991371154785,
554
+ "logps/chosen": -382.35736083984375,
555
+ "logps/rejected": -421.2584533691406,
556
+ "loss": 0.0493,
557
+ "rewards/accuracies": 0.6937500238418579,
558
+ "rewards/chosen": -1.3148142099380493,
559
+ "rewards/margins": 0.5485247373580933,
560
+ "rewards/rejected": -1.8633390665054321,
561
+ "step": 350
 
 
 
 
 
 
 
 
 
 
 
 
 
 
562
  },
563
  {
564
  "epoch": 0.75,
565
+ "learning_rate": 8.729103716819111e-08,
566
+ "logits/chosen": -2.356658935546875,
567
+ "logits/rejected": -2.315314769744873,
568
+ "logps/chosen": -399.7149963378906,
569
+ "logps/rejected": -423.53326416015625,
570
+ "loss": 0.0538,
571
+ "rewards/accuracies": 0.675000011920929,
572
+ "rewards/chosen": -1.354021668434143,
573
+ "rewards/margins": 0.5520496964454651,
574
+ "rewards/rejected": -1.9060713052749634,
575
+ "step": 360
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
576
  },
577
  {
578
  "epoch": 0.77,
579
+ "learning_rate": 7.387025063449081e-08,
580
+ "logits/chosen": -2.3429181575775146,
581
+ "logits/rejected": -2.30153489112854,
582
+ "logps/chosen": -419.21160888671875,
583
+ "logps/rejected": -458.06884765625,
584
+ "loss": 0.0432,
585
+ "rewards/accuracies": 0.7124999761581421,
586
+ "rewards/chosen": -1.3274778127670288,
587
+ "rewards/margins": 0.6080622673034668,
588
+ "rewards/rejected": -1.935539960861206,
589
+ "step": 370
 
 
 
 
 
 
 
 
 
 
 
 
 
 
590
  },
591
  {
592
  "epoch": 0.79,
593
+ "learning_rate": 6.138919252022435e-08,
594
+ "logits/chosen": -2.3714020252227783,
595
+ "logits/rejected": -2.361142158508301,
596
+ "logps/chosen": -419.2672424316406,
597
+ "logps/rejected": -452.9471740722656,
598
+ "loss": 0.0483,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
599
  "rewards/accuracies": 0.7250000238418579,
600
+ "rewards/chosen": -1.336124300956726,
601
+ "rewards/margins": 0.5412160754203796,
602
+ "rewards/rejected": -1.87734055519104,
603
+ "step": 380
604
  },
605
  {
606
+ "epoch": 0.82,
607
+ "learning_rate": 4.991445467064689e-08,
608
+ "logits/chosen": -2.275606155395508,
609
+ "logits/rejected": -2.2492728233337402,
610
+ "logps/chosen": -406.21112060546875,
611
+ "logps/rejected": -463.1912536621094,
612
+ "loss": 0.0462,
613
  "rewards/accuracies": 0.731249988079071,
614
+ "rewards/chosen": -1.3669636249542236,
615
+ "rewards/margins": 0.6890057325363159,
616
+ "rewards/rejected": -2.055969715118408,
617
+ "step": 390
618
  },
619
  {
620
  "epoch": 0.84,
621
+ "learning_rate": 3.9507259776993954e-08,
622
+ "logits/chosen": -2.364297389984131,
623
+ "logits/rejected": -2.3375916481018066,
624
+ "logps/chosen": -408.903076171875,
625
+ "logps/rejected": -438.1351013183594,
626
+ "loss": 0.0435,
627
+ "rewards/accuracies": 0.6937500238418579,
628
+ "rewards/chosen": -1.332507848739624,
629
+ "rewards/margins": 0.5097385048866272,
630
+ "rewards/rejected": -1.8422462940216064,
631
+ "step": 400
632
  },
633
  {
634
+ "epoch": 0.84,
635
+ "eval_logits/chosen": -2.372504234313965,
636
+ "eval_logits/rejected": -2.3534233570098877,
637
+ "eval_logps/chosen": -384.0027160644531,
638
+ "eval_logps/rejected": -455.4245910644531,
639
+ "eval_loss": 0.05103687942028046,
640
+ "eval_rewards/accuracies": 0.72265625,
641
+ "eval_rewards/chosen": -1.2696306705474854,
642
+ "eval_rewards/margins": 0.7110825777053833,
643
+ "eval_rewards/rejected": -1.9807134866714478,
644
+ "eval_runtime": 53.2911,
645
+ "eval_samples_per_second": 37.53,
646
+ "eval_steps_per_second": 0.6,
647
+ "step": 400
648
  },
649
  {
650
  "epoch": 0.86,
651
+ "learning_rate": 3.022313472693447e-08,
652
+ "logits/chosen": -2.3608033657073975,
653
+ "logits/rejected": -2.3090267181396484,
654
+ "logps/chosen": -396.9889831542969,
655
+ "logps/rejected": -402.0162658691406,
656
+ "loss": 0.0472,
657
+ "rewards/accuracies": 0.699999988079071,
658
+ "rewards/chosen": -1.2232677936553955,
659
+ "rewards/margins": 0.5818590521812439,
660
+ "rewards/rejected": -1.8051269054412842,
661
+ "step": 410
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
662
  },
663
  {
664
  "epoch": 0.88,
665
+ "learning_rate": 2.2111614344599684e-08,
666
+ "logits/chosen": -2.3692984580993652,
667
+ "logits/rejected": -2.327254056930542,
668
+ "logps/chosen": -428.22698974609375,
669
+ "logps/rejected": -461.43798828125,
670
+ "loss": 0.048,
671
+ "rewards/accuracies": 0.699999988079071,
672
+ "rewards/chosen": -1.373509168624878,
673
+ "rewards/margins": 0.740686297416687,
674
+ "rewards/rejected": -2.1141953468322754,
675
+ "step": 420
 
 
 
 
 
 
 
 
 
 
 
 
 
 
676
  },
677
  {
678
  "epoch": 0.9,
679
+ "learning_rate": 1.521597710086439e-08,
680
+ "logits/chosen": -2.3224453926086426,
681
+ "logits/rejected": -2.3250882625579834,
682
+ "logps/chosen": -418.9920349121094,
683
+ "logps/rejected": -470.03857421875,
684
+ "loss": 0.0457,
685
+ "rewards/accuracies": 0.706250011920929,
686
+ "rewards/chosen": -1.4404687881469727,
687
+ "rewards/margins": 0.5571845769882202,
688
+ "rewards/rejected": -1.9976533651351929,
689
+ "step": 430
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
690
  },
691
  {
692
  "epoch": 0.92,
693
+ "learning_rate": 9.57301420397924e-09,
694
+ "logits/chosen": -2.3068556785583496,
695
+ "logits/rejected": -2.3014369010925293,
696
+ "logps/chosen": -404.3832702636719,
697
+ "logps/rejected": -442.23089599609375,
698
+ "loss": 0.0478,
699
+ "rewards/accuracies": 0.6187499761581421,
700
+ "rewards/chosen": -1.4350427389144897,
701
+ "rewards/margins": 0.5088328123092651,
702
+ "rewards/rejected": -1.9438753128051758,
703
+ "step": 440
 
 
 
 
 
 
 
 
 
 
 
 
 
 
704
  },
705
  {
706
  "epoch": 0.94,
707
+ "learning_rate": 5.212833302556258e-09,
708
+ "logits/chosen": -2.3136403560638428,
709
+ "logits/rejected": -2.3258073329925537,
710
+ "logps/chosen": -397.1946716308594,
711
+ "logps/rejected": -443.69830322265625,
712
+ "loss": 0.0467,
713
+ "rewards/accuracies": 0.6875,
714
+ "rewards/chosen": -1.4375298023223877,
715
+ "rewards/margins": 0.5240463018417358,
716
+ "rewards/rejected": -1.9615761041641235,
717
+ "step": 450
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
718
  },
719
  {
720
  "epoch": 0.96,
721
+ "learning_rate": 2.158697848236607e-09,
722
+ "logits/chosen": -2.3062024116516113,
723
+ "logits/rejected": -2.2698886394500732,
724
+ "logps/chosen": -397.9000244140625,
725
+ "logps/rejected": -424.3349609375,
726
+ "loss": 0.0462,
727
+ "rewards/accuracies": 0.6499999761581421,
728
+ "rewards/chosen": -1.4474247694015503,
729
+ "rewards/margins": 0.4248392581939697,
730
+ "rewards/rejected": -1.8722641468048096,
731
+ "step": 460
 
 
 
 
 
 
 
 
 
 
 
 
 
 
732
  },
733
  {
734
  "epoch": 0.98,
735
+ "learning_rate": 4.269029751107489e-10,
736
+ "logits/chosen": -2.290329933166504,
737
+ "logits/rejected": -2.2513458728790283,
738
+ "logps/chosen": -393.18310546875,
739
+ "logps/rejected": -458.83209228515625,
740
+ "loss": 0.0474,
741
+ "rewards/accuracies": 0.731249988079071,
742
+ "rewards/chosen": -1.3453119993209839,
743
+ "rewards/margins": 0.7067785859107971,
744
+ "rewards/rejected": -2.0520904064178467,
745
+ "step": 470
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
746
  },
747
  {
748
  "epoch": 1.0,
749
+ "step": 478,
750
  "total_flos": 0.0,
751
+ "train_loss": 0.07493206359851311,
752
+ "train_runtime": 4367.2364,
753
+ "train_samples_per_second": 13.998,
754
+ "train_steps_per_second": 0.109
755
  }
756
  ],
757
  "logging_steps": 10,
758
+ "max_steps": 478,
759
  "num_train_epochs": 1,
760
  "save_steps": 100,
761
  "total_flos": 0.0,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ef3f3bcb1d637ffd73632ad00af47d3006ac1e6c1f0c109c90bd802bdaba6dcd
3
  size 5944
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b563d8ba7853a3ff70d20b889cac2dc1b6f510a9e1fcf017bbae2c1ee40b07d
3
  size 5944