Hugging Face's logo

DUAL-GPO
/
7b-kto-0-20-i1

PEFT
TensorBoard
Safetensors
mistral
alignment-handbook
Generated from Trainer
trl
dpo
Model card Files Metrics Community
BraylonDash commited on
Commit
aa68aa7
·
verified ·
1 Parent(s): 24309e6

Model save

Browse files
Files changed (7) hide show
  1. README.md +61 -0
  2. adapter_model.safetensors +1 -1
  3. all_results.json +8 -0
  4. emissions.csv +2 -0
  5. runs/Sep30_02-06-02_gpu4-119-5/events.out.tfevents.1727627485.gpu4-119-5.1443073.0 +2 -2
  6. train_results.json +8 -0
  7. trainer_state.json +478 -0
README.md ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ tags:
4
+ - trl
5
+ - dpo
6
+ - generated_from_trainer
7
+ base_model: DUAL-GPO/7b-kto-i0-merged
8
+ model-index:
9
+ - name: 7b-kto-0-20-i1
10
+ results: []
11
+ ---
12
+
13
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
14
+ should probably proofread and complete it, then remove this comment. -->
15
+
16
+ # 7b-kto-0-20-i1
17
+
18
+ This model is a fine-tuned version of [DUAL-GPO/7b-kto-i0-merged](https://huggingface.co/DUAL-GPO/7b-kto-i0-merged) on the None dataset.
19
+
20
+ ## Model description
21
+
22
+ More information needed
23
+
24
+ ## Intended uses & limitations
25
+
26
+ More information needed
27
+
28
+ ## Training and evaluation data
29
+
30
+ More information needed
31
+
32
+ ## Training procedure
33
+
34
+ ### Training hyperparameters
35
+
36
+ The following hyperparameters were used during training:
37
+ - learning_rate: 5e-06
38
+ - train_batch_size: 4
39
+ - eval_batch_size: 4
40
+ - seed: 42
41
+ - distributed_type: multi-GPU
42
+ - num_devices: 4
43
+ - gradient_accumulation_steps: 4
44
+ - total_train_batch_size: 64
45
+ - total_eval_batch_size: 16
46
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
47
+ - lr_scheduler_type: cosine
48
+ - lr_scheduler_warmup_ratio: 0.1
49
+ - num_epochs: 1
50
+
51
+ ### Training results
52
+
53
+
54
+
55
+ ### Framework versions
56
+
57
+ - PEFT 0.7.1
58
+ - Transformers 4.36.2
59
+ - Pytorch 2.1.2
60
+ - Datasets 2.14.6
61
+ - Tokenizers 0.15.2
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d43afbbbee9643e71988ea9aca4af949a79641589986598f3bf10a1d964dd471
3
  size 151021328
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d76243dc6ad450136541dd564079420749365c7604062225b0b02f7eb344b3b
3
  size 151021328
all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.47231930035811204,
4
+ "train_runtime": 7349.4695,
5
+ "train_samples": 20000,
6
+ "train_samples_per_second": 2.721,
7
+ "train_steps_per_second": 0.042
8
+ }
emissions.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ timestamp,project_name,run_id,duration,emissions,emissions_rate,cpu_power,gpu_power,ram_power,cpu_energy,gpu_energy,ram_energy,energy_consumed,country_name,country_iso_code,region,cloud_provider,cloud_region,os,python_version,codecarbon_version,cpu_count,cpu_model,gpu_count,gpu_model,longitude,latitude,ram_total_size,tracking_mode,on_cloud,pue
2
+ 2024-09-30T04:33:55,codecarbon,e949b6b3-51c9-4041-8e56-fb048ed8a689,7349.48065161705,0.006793945301443062,9.244116181118515e-07,42.5,1252.361,188.74309015274048,0.08676339168879714,2.3874307339818395,0.3837446331469837,2.8579387588176237,Canada,CAN,quebec,,,Linux-5.15.0-84-generic-x86_64-with-glibc2.35,3.10.14,2.2.3,32,Intel(R) Xeon(R) W-3335 CPU @ 3.40GHz,4,4 x NVIDIA GeForce RTX 4090,-71.2,46.8,503.3149070739746,machine,N,1.0
runs/Sep30_02-06-02_gpu4-119-5/events.out.tfevents.1727627485.gpu4-119-5.1443073.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0f53ed89e75a50125167417e3a069774067af7daef2ae692affe973085e1090d
3
- size 24066
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d23d6fe470360c883bf966f31fcbca5d7bf06b7634762325736ef80409438761
3
+ size 25054
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.47231930035811204,
4
+ "train_runtime": 7349.4695,
5
+ "train_samples": 20000,
6
+ "train_samples_per_second": 2.721,
7
+ "train_steps_per_second": 0.042
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,478 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9984,
5
+ "eval_steps": 500,
6
+ "global_step": 312,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "learning_rate": 1.5625e-07,
14
+ "logits/chosen": -2.001559019088745,
15
+ "logits/rejected": -2.052518367767334,
16
+ "logps/chosen": -2030.77099609375,
17
+ "logps/rejected": -2033.691650390625,
18
+ "loss": 0.5,
19
+ "rewards/accuracies": 0.0,
20
+ "rewards/chosen": 0.0,
21
+ "rewards/margins": 0.0,
22
+ "rewards/rejected": 0.0,
23
+ "step": 1
24
+ },
25
+ {
26
+ "epoch": 0.03,
27
+ "learning_rate": 1.5625e-06,
28
+ "logits/chosen": -2.146657705307007,
29
+ "logits/rejected": -2.073806047439575,
30
+ "logps/chosen": -1147.4302978515625,
31
+ "logps/rejected": -2748.2314453125,
32
+ "loss": 0.4979,
33
+ "rewards/accuracies": 0.4236111044883728,
34
+ "rewards/chosen": -0.028285933658480644,
35
+ "rewards/margins": 0.02654173970222473,
36
+ "rewards/rejected": -0.054827671498060226,
37
+ "step": 10
38
+ },
39
+ {
40
+ "epoch": 0.06,
41
+ "learning_rate": 3.125e-06,
42
+ "logits/chosen": -2.2760894298553467,
43
+ "logits/rejected": -2.1618006229400635,
44
+ "logps/chosen": -1438.9384765625,
45
+ "logps/rejected": -3313.58837890625,
46
+ "loss": 0.4847,
47
+ "rewards/accuracies": 0.5687500238418579,
48
+ "rewards/chosen": -0.2823066711425781,
49
+ "rewards/margins": 0.31389349699020386,
50
+ "rewards/rejected": -0.596200168132782,
51
+ "step": 20
52
+ },
53
+ {
54
+ "epoch": 0.1,
55
+ "learning_rate": 4.6875000000000004e-06,
56
+ "logits/chosen": -2.164494752883911,
57
+ "logits/rejected": -2.0667026042938232,
58
+ "logps/chosen": -2890.416015625,
59
+ "logps/rejected": -4221.6728515625,
60
+ "loss": 0.4841,
61
+ "rewards/accuracies": 0.5,
62
+ "rewards/chosen": -1.3330113887786865,
63
+ "rewards/margins": 0.03289594128727913,
64
+ "rewards/rejected": -1.3659074306488037,
65
+ "step": 30
66
+ },
67
+ {
68
+ "epoch": 0.13,
69
+ "learning_rate": 4.989935734988098e-06,
70
+ "logits/chosen": -2.1548264026641846,
71
+ "logits/rejected": -2.0702128410339355,
72
+ "logps/chosen": -3225.27685546875,
73
+ "logps/rejected": -4589.10107421875,
74
+ "loss": 0.4908,
75
+ "rewards/accuracies": 0.4749999940395355,
76
+ "rewards/chosen": -1.890707015991211,
77
+ "rewards/margins": 0.1443764716386795,
78
+ "rewards/rejected": -2.035083293914795,
79
+ "step": 40
80
+ },
81
+ {
82
+ "epoch": 0.16,
83
+ "learning_rate": 4.949188496058089e-06,
84
+ "logits/chosen": -2.221364974975586,
85
+ "logits/rejected": -2.0859131813049316,
86
+ "logps/chosen": -2741.037353515625,
87
+ "logps/rejected": -4698.10302734375,
88
+ "loss": 0.4781,
89
+ "rewards/accuracies": 0.53125,
90
+ "rewards/chosen": -1.1827226877212524,
91
+ "rewards/margins": 0.47015246748924255,
92
+ "rewards/rejected": -1.6528751850128174,
93
+ "step": 50
94
+ },
95
+ {
96
+ "epoch": 0.19,
97
+ "learning_rate": 4.8776412907378845e-06,
98
+ "logits/chosen": -2.3111491203308105,
99
+ "logits/rejected": -2.244699001312256,
100
+ "logps/chosen": -2379.827392578125,
101
+ "logps/rejected": -4096.80712890625,
102
+ "loss": 0.4761,
103
+ "rewards/accuracies": 0.550000011920929,
104
+ "rewards/chosen": -0.8030478358268738,
105
+ "rewards/margins": 0.2678872048854828,
106
+ "rewards/rejected": -1.0709350109100342,
107
+ "step": 60
108
+ },
109
+ {
110
+ "epoch": 0.22,
111
+ "learning_rate": 4.7761938666470405e-06,
112
+ "logits/chosen": -2.369278907775879,
113
+ "logits/rejected": -2.2874133586883545,
114
+ "logps/chosen": -2155.563720703125,
115
+ "logps/rejected": -3842.12109375,
116
+ "loss": 0.4766,
117
+ "rewards/accuracies": 0.543749988079071,
118
+ "rewards/chosen": -0.7144564986228943,
119
+ "rewards/margins": 0.4151995778083801,
120
+ "rewards/rejected": -1.1296560764312744,
121
+ "step": 70
122
+ },
123
+ {
124
+ "epoch": 0.26,
125
+ "learning_rate": 4.646121984004666e-06,
126
+ "logits/chosen": -2.3896870613098145,
127
+ "logits/rejected": -2.330310583114624,
128
+ "logps/chosen": -2313.40380859375,
129
+ "logps/rejected": -4401.6201171875,
130
+ "loss": 0.4816,
131
+ "rewards/accuracies": 0.512499988079071,
132
+ "rewards/chosen": -0.8072439432144165,
133
+ "rewards/margins": 0.2879006266593933,
134
+ "rewards/rejected": -1.0951446294784546,
135
+ "step": 80
136
+ },
137
+ {
138
+ "epoch": 0.29,
139
+ "learning_rate": 4.4890613722044526e-06,
140
+ "logits/chosen": -2.4341788291931152,
141
+ "logits/rejected": -2.4094738960266113,
142
+ "logps/chosen": -2331.335205078125,
143
+ "logps/rejected": -3481.241455078125,
144
+ "loss": 0.4816,
145
+ "rewards/accuracies": 0.48750001192092896,
146
+ "rewards/chosen": -0.9047203063964844,
147
+ "rewards/margins": 0.22198553383350372,
148
+ "rewards/rejected": -1.1267058849334717,
149
+ "step": 90
150
+ },
151
+ {
152
+ "epoch": 0.32,
153
+ "learning_rate": 4.3069871595684795e-06,
154
+ "logits/chosen": -2.4191980361938477,
155
+ "logits/rejected": -2.3792288303375244,
156
+ "logps/chosen": -2510.74267578125,
157
+ "logps/rejected": -3801.65771484375,
158
+ "loss": 0.4795,
159
+ "rewards/accuracies": 0.53125,
160
+ "rewards/chosen": -0.9170597791671753,
161
+ "rewards/margins": 0.13030724227428436,
162
+ "rewards/rejected": -1.0473670959472656,
163
+ "step": 100
164
+ },
165
+ {
166
+ "epoch": 0.35,
167
+ "learning_rate": 4.102189034962561e-06,
168
+ "logits/chosen": -2.4569015502929688,
169
+ "logits/rejected": -2.4183101654052734,
170
+ "logps/chosen": -2428.78369140625,
171
+ "logps/rejected": -5115.49267578125,
172
+ "loss": 0.4677,
173
+ "rewards/accuracies": 0.5562499761581421,
174
+ "rewards/chosen": -0.9490839242935181,
175
+ "rewards/margins": 0.8531773686408997,
176
+ "rewards/rejected": -1.8022613525390625,
177
+ "step": 110
178
+ },
179
+ {
180
+ "epoch": 0.38,
181
+ "learning_rate": 3.8772424536302565e-06,
182
+ "logits/chosen": -2.5435500144958496,
183
+ "logits/rejected": -2.540524482727051,
184
+ "logps/chosen": -2736.87158203125,
185
+ "logps/rejected": -4036.8984375,
186
+ "loss": 0.4729,
187
+ "rewards/accuracies": 0.5062500238418579,
188
+ "rewards/chosen": -1.0938019752502441,
189
+ "rewards/margins": 0.23258857429027557,
190
+ "rewards/rejected": -1.3263905048370361,
191
+ "step": 120
192
+ },
193
+ {
194
+ "epoch": 0.42,
195
+ "learning_rate": 3.634976249348867e-06,
196
+ "logits/chosen": -2.5451807975769043,
197
+ "logits/rejected": -2.5353500843048096,
198
+ "logps/chosen": -2400.534423828125,
199
+ "logps/rejected": -3994.819091796875,
200
+ "loss": 0.4725,
201
+ "rewards/accuracies": 0.5,
202
+ "rewards/chosen": -1.0581142902374268,
203
+ "rewards/margins": 0.3071446716785431,
204
+ "rewards/rejected": -1.3652589321136475,
205
+ "step": 130
206
+ },
207
+ {
208
+ "epoch": 0.45,
209
+ "learning_rate": 3.3784370602033572e-06,
210
+ "logits/chosen": -2.502204179763794,
211
+ "logits/rejected": -2.5256900787353516,
212
+ "logps/chosen": -2380.7509765625,
213
+ "logps/rejected": -4508.63818359375,
214
+ "loss": 0.4723,
215
+ "rewards/accuracies": 0.5062500238418579,
216
+ "rewards/chosen": -0.9423351287841797,
217
+ "rewards/margins": 0.6326543092727661,
218
+ "rewards/rejected": -1.5749894380569458,
219
+ "step": 140
220
+ },
221
+ {
222
+ "epoch": 0.48,
223
+ "learning_rate": 3.1108510153447352e-06,
224
+ "logits/chosen": -2.583056688308716,
225
+ "logits/rejected": -2.599966049194336,
226
+ "logps/chosen": -1865.611328125,
227
+ "logps/rejected": -4491.51806640625,
228
+ "loss": 0.4669,
229
+ "rewards/accuracies": 0.5874999761581421,
230
+ "rewards/chosen": -0.6541633009910583,
231
+ "rewards/margins": 0.6614173054695129,
232
+ "rewards/rejected": -1.3155806064605713,
233
+ "step": 150
234
+ },
235
+ {
236
+ "epoch": 0.51,
237
+ "learning_rate": 2.835583164544139e-06,
238
+ "logits/chosen": -2.5848755836486816,
239
+ "logits/rejected": -2.6103827953338623,
240
+ "logps/chosen": -2029.941162109375,
241
+ "logps/rejected": -3397.024169921875,
242
+ "loss": 0.4691,
243
+ "rewards/accuracies": 0.543749988079071,
244
+ "rewards/chosen": -0.725816011428833,
245
+ "rewards/margins": 0.292774498462677,
246
+ "rewards/rejected": -1.0185905694961548,
247
+ "step": 160
248
+ },
249
+ {
250
+ "epoch": 0.54,
251
+ "learning_rate": 2.556095160739513e-06,
252
+ "logits/chosen": -2.5292410850524902,
253
+ "logits/rejected": -2.5528335571289062,
254
+ "logps/chosen": -1871.8792724609375,
255
+ "logps/rejected": -3524.819091796875,
256
+ "loss": 0.4714,
257
+ "rewards/accuracies": 0.5625,
258
+ "rewards/chosen": -0.6304308176040649,
259
+ "rewards/margins": 0.3158620595932007,
260
+ "rewards/rejected": -0.9462926983833313,
261
+ "step": 170
262
+ },
263
+ {
264
+ "epoch": 0.58,
265
+ "learning_rate": 2.2759017277414165e-06,
266
+ "logits/chosen": -2.618015766143799,
267
+ "logits/rejected": -2.658297061920166,
268
+ "logps/chosen": -2715.969970703125,
269
+ "logps/rejected": -4858.7861328125,
270
+ "loss": 0.4621,
271
+ "rewards/accuracies": 0.5062500238418579,
272
+ "rewards/chosen": -1.088996171951294,
273
+ "rewards/margins": 0.6818682551383972,
274
+ "rewards/rejected": -1.770864486694336,
275
+ "step": 180
276
+ },
277
+ {
278
+ "epoch": 0.61,
279
+ "learning_rate": 1.9985264605418185e-06,
280
+ "logits/chosen": -2.663578748703003,
281
+ "logits/rejected": -2.7316274642944336,
282
+ "logps/chosen": -2506.532958984375,
283
+ "logps/rejected": -5105.27001953125,
284
+ "loss": 0.4641,
285
+ "rewards/accuracies": 0.5687500238418579,
286
+ "rewards/chosen": -1.2244542837142944,
287
+ "rewards/margins": 0.8673825263977051,
288
+ "rewards/rejected": -2.091836929321289,
289
+ "step": 190
290
+ },
291
+ {
292
+ "epoch": 0.64,
293
+ "learning_rate": 1.7274575140626318e-06,
294
+ "logits/chosen": -2.6198081970214844,
295
+ "logits/rejected": -2.7005550861358643,
296
+ "logps/chosen": -2381.856201171875,
297
+ "logps/rejected": -4935.6943359375,
298
+ "loss": 0.4612,
299
+ "rewards/accuracies": 0.581250011920929,
300
+ "rewards/chosen": -1.1094557046890259,
301
+ "rewards/margins": 1.003557562828064,
302
+ "rewards/rejected": -2.113013505935669,
303
+ "step": 200
304
+ },
305
+ {
306
+ "epoch": 0.67,
307
+ "learning_rate": 1.466103737583699e-06,
308
+ "logits/chosen": -2.535006046295166,
309
+ "logits/rejected": -2.603625774383545,
310
+ "logps/chosen": -1250.8673095703125,
311
+ "logps/rejected": -3654.595703125,
312
+ "loss": 0.4603,
313
+ "rewards/accuracies": 0.59375,
314
+ "rewards/chosen": -0.42699164152145386,
315
+ "rewards/margins": 0.7609654664993286,
316
+ "rewards/rejected": -1.1879570484161377,
317
+ "step": 210
318
+ },
319
+ {
320
+ "epoch": 0.7,
321
+ "learning_rate": 1.217751806485235e-06,
322
+ "logits/chosen": -2.5781049728393555,
323
+ "logits/rejected": -2.6412928104400635,
324
+ "logps/chosen": -1635.0478515625,
325
+ "logps/rejected": -4128.06884765625,
326
+ "loss": 0.4629,
327
+ "rewards/accuracies": 0.5249999761581421,
328
+ "rewards/chosen": -0.655454695224762,
329
+ "rewards/margins": 0.8185914754867554,
330
+ "rewards/rejected": -1.4740461111068726,
331
+ "step": 220
332
+ },
333
+ {
334
+ "epoch": 0.74,
335
+ "learning_rate": 9.855248903979505e-07,
336
+ "logits/chosen": -2.583189010620117,
337
+ "logits/rejected": -2.722602605819702,
338
+ "logps/chosen": -2002.4896240234375,
339
+ "logps/rejected": -4570.92041015625,
340
+ "loss": 0.4644,
341
+ "rewards/accuracies": 0.612500011920929,
342
+ "rewards/chosen": -0.8033896684646606,
343
+ "rewards/margins": 0.8625272512435913,
344
+ "rewards/rejected": -1.6659168004989624,
345
+ "step": 230
346
+ },
347
+ {
348
+ "epoch": 0.77,
349
+ "learning_rate": 7.723433775328385e-07,
350
+ "logits/chosen": -2.6383585929870605,
351
+ "logits/rejected": -2.7192063331604004,
352
+ "logps/chosen": -2607.29638671875,
353
+ "logps/rejected": -4494.58349609375,
354
+ "loss": 0.4638,
355
+ "rewards/accuracies": 0.4937500059604645,
356
+ "rewards/chosen": -0.9701069593429565,
357
+ "rewards/margins": 0.6917791962623596,
358
+ "rewards/rejected": -1.661886215209961,
359
+ "step": 240
360
+ },
361
+ {
362
+ "epoch": 0.8,
363
+ "learning_rate": 5.808881491049723e-07,
364
+ "logits/chosen": -2.616407871246338,
365
+ "logits/rejected": -2.639394760131836,
366
+ "logps/chosen": -2655.154296875,
367
+ "logps/rejected": -3849.689453125,
368
+ "loss": 0.4711,
369
+ "rewards/accuracies": 0.48750001192092896,
370
+ "rewards/chosen": -0.885340690612793,
371
+ "rewards/margins": 0.4874494969844818,
372
+ "rewards/rejected": -1.3727902173995972,
373
+ "step": 250
374
+ },
375
+ {
376
+ "epoch": 0.83,
377
+ "learning_rate": 4.1356686569674344e-07,
378
+ "logits/chosen": -2.6079611778259277,
379
+ "logits/rejected": -2.7340540885925293,
380
+ "logps/chosen": -2142.09814453125,
381
+ "logps/rejected": -4976.4326171875,
382
+ "loss": 0.4604,
383
+ "rewards/accuracies": 0.612500011920929,
384
+ "rewards/chosen": -0.790375828742981,
385
+ "rewards/margins": 0.7060346603393555,
386
+ "rewards/rejected": -1.496410608291626,
387
+ "step": 260
388
+ },
389
+ {
390
+ "epoch": 0.86,
391
+ "learning_rate": 2.7248368952908055e-07,
392
+ "logits/chosen": -2.6824259757995605,
393
+ "logits/rejected": -2.7371983528137207,
394
+ "logps/chosen": -2655.15673828125,
395
+ "logps/rejected": -4562.94091796875,
396
+ "loss": 0.4754,
397
+ "rewards/accuracies": 0.518750011920929,
398
+ "rewards/chosen": -1.0050941705703735,
399
+ "rewards/margins": 0.3014536201953888,
400
+ "rewards/rejected": -1.3065478801727295,
401
+ "step": 270
402
+ },
403
+ {
404
+ "epoch": 0.9,
405
+ "learning_rate": 1.59412823400657e-07,
406
+ "logits/chosen": -2.636775493621826,
407
+ "logits/rejected": -2.734402656555176,
408
+ "logps/chosen": -2063.6806640625,
409
+ "logps/rejected": -4341.87109375,
410
+ "loss": 0.4669,
411
+ "rewards/accuracies": 0.606249988079071,
412
+ "rewards/chosen": -0.7057104706764221,
413
+ "rewards/margins": 0.690462589263916,
414
+ "rewards/rejected": -1.3961730003356934,
415
+ "step": 280
416
+ },
417
+ {
418
+ "epoch": 0.93,
419
+ "learning_rate": 7.577619905828281e-08,
420
+ "logits/chosen": -2.631303310394287,
421
+ "logits/rejected": -2.7183616161346436,
422
+ "logps/chosen": -2191.196044921875,
423
+ "logps/rejected": -4163.2451171875,
424
+ "loss": 0.4691,
425
+ "rewards/accuracies": 0.5249999761581421,
426
+ "rewards/chosen": -0.7605796456336975,
427
+ "rewards/margins": 0.5482227206230164,
428
+ "rewards/rejected": -1.3088023662567139,
429
+ "step": 290
430
+ },
431
+ {
432
+ "epoch": 0.96,
433
+ "learning_rate": 2.262559558016325e-08,
434
+ "logits/chosen": -2.61507511138916,
435
+ "logits/rejected": -2.646120548248291,
436
+ "logps/chosen": -1883.6669921875,
437
+ "logps/rejected": -3444.8046875,
438
+ "loss": 0.466,
439
+ "rewards/accuracies": 0.53125,
440
+ "rewards/chosen": -0.6400176286697388,
441
+ "rewards/margins": 0.3954530656337738,
442
+ "rewards/rejected": -1.035470724105835,
443
+ "step": 300
444
+ },
445
+ {
446
+ "epoch": 0.99,
447
+ "learning_rate": 6.294126437336734e-10,
448
+ "logits/chosen": -2.6119272708892822,
449
+ "logits/rejected": -2.7076706886291504,
450
+ "logps/chosen": -2384.146240234375,
451
+ "logps/rejected": -4828.73095703125,
452
+ "loss": 0.4701,
453
+ "rewards/accuracies": 0.5687500238418579,
454
+ "rewards/chosen": -0.8787592053413391,
455
+ "rewards/margins": 0.646748960018158,
456
+ "rewards/rejected": -1.525508165359497,
457
+ "step": 310
458
+ },
459
+ {
460
+ "epoch": 1.0,
461
+ "step": 312,
462
+ "total_flos": 0.0,
463
+ "train_loss": 0.47231930035811204,
464
+ "train_runtime": 7349.4695,
465
+ "train_samples_per_second": 2.721,
466
+ "train_steps_per_second": 0.042
467
+ }
468
+ ],
469
+ "logging_steps": 10,
470
+ "max_steps": 312,
471
+ "num_input_tokens_seen": 0,
472
+ "num_train_epochs": 1,
473
+ "save_steps": 20,
474
+ "total_flos": 0.0,
475
+ "train_batch_size": 4,
476
+ "trial_name": null,
477
+ "trial_params": null
478
+ }