Hugging Face's logo

DUAL-GPO-2
/
phi-2-gpo-v27-i1

PEFT
TensorBoard
Safetensors
phi
alignment-handbook
Generated from Trainer
trl
dpo
custom_code
Model card Files Metrics Community
lole25 commited on
Commit
eb482d6
•
1 Parent(s): 39e1e35

Model save

Browse files
Files changed (6) hide show
  1. README.md +62 -0
  2. adapter_model.safetensors +1 -1
  3. all_results.json +8 -0
  4. runs/May15_20-58-28_gpu4-119-5/events.out.tfevents.1715770793.gpu4-119-5.3263155.0 +2 -2
  5. train_results.json +8 -0
  6. trainer_state.json +324 -0
README.md ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: mit
3
+ library_name: peft
4
+ tags:
5
+ - trl
6
+ - dpo
7
+ - generated_from_trainer
8
+ base_model: DUAL-GPO/phi-2-gpo-new-i0
9
+ model-index:
10
+ - name: phi-2-gpo-v27-i1
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # phi-2-gpo-v27-i1
18
+
19
+ This model is a fine-tuned version of [DUAL-GPO/phi-2-gpo-new-i0](https://huggingface.co/DUAL-GPO/phi-2-gpo-new-i0) on the None dataset.
20
+
21
+ ## Model description
22
+
23
+ More information needed
24
+
25
+ ## Intended uses & limitations
26
+
27
+ More information needed
28
+
29
+ ## Training and evaluation data
30
+
31
+ More information needed
32
+
33
+ ## Training procedure
34
+
35
+ ### Training hyperparameters
36
+
37
+ The following hyperparameters were used during training:
38
+ - learning_rate: 5e-06
39
+ - train_batch_size: 4
40
+ - eval_batch_size: 4
41
+ - seed: 42
42
+ - distributed_type: multi-GPU
43
+ - num_devices: 3
44
+ - gradient_accumulation_steps: 4
45
+ - total_train_batch_size: 48
46
+ - total_eval_batch_size: 12
47
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
48
+ - lr_scheduler_type: cosine
49
+ - lr_scheduler_warmup_ratio: 0.1
50
+ - num_epochs: 1
51
+
52
+ ### Training results
53
+
54
+
55
+
56
+ ### Framework versions
57
+
58
+ - PEFT 0.7.1
59
+ - Transformers 4.36.2
60
+ - Pytorch 2.1.2+cu121
61
+ - Datasets 2.14.6
62
+ - Tokenizers 0.15.2
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0161014b58ccb916065aad24d3bc6d63521f282159f8753feef5557c2232a80b
3
  size 167807296
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81a43958dd8227a9e33361307a484896835ea7ab009c5c31d64a985498cc391d
3
  size 167807296
all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.29713730256144816,
4
+ "train_runtime": 2891.8659,
5
+ "train_samples": 10000,
6
+ "train_samples_per_second": 3.458,
7
+ "train_steps_per_second": 0.072
8
+ }
runs/May15_20-58-28_gpu4-119-5/events.out.tfevents.1715770793.gpu4-119-5.3263155.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a61ac5a0e36febf093f7de1d34029a5ce4b5181e1cfb1e6a03a46fb57650cade
3
- size 18004
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a69f9bd0c51282e7622ee4017af1702d4db6d2e909d68763e51cc94a053d904e
3
+ size 18358
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.29713730256144816,
4
+ "train_runtime": 2891.8659,
5
+ "train_samples": 10000,
6
+ "train_samples_per_second": 3.458,
7
+ "train_steps_per_second": 0.072
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9976019184652278,
5
+ "eval_steps": 500,
6
+ "global_step": 208,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "learning_rate": 2.3809523809523811e-07,
14
+ "logits/chosen": 0.15803536772727966,
15
+ "logits/rejected": 0.08697354793548584,
16
+ "logps/chosen": -431.6365661621094,
17
+ "logps/rejected": -312.2266845703125,
18
+ "loss": 0.3497,
19
+ "rewards/accuracies": 0.0,
20
+ "rewards/chosen": 0.0,
21
+ "rewards/margins": 0.0,
22
+ "rewards/rejected": 0.0,
23
+ "step": 1
24
+ },
25
+ {
26
+ "epoch": 0.05,
27
+ "learning_rate": 2.380952380952381e-06,
28
+ "logits/chosen": 0.08879230171442032,
29
+ "logits/rejected": 0.23703241348266602,
30
+ "logps/chosen": -334.3096008300781,
31
+ "logps/rejected": -325.03387451171875,
32
+ "loss": 0.3916,
33
+ "rewards/accuracies": 0.4513888955116272,
34
+ "rewards/chosen": 0.011717023327946663,
35
+ "rewards/margins": 0.0023966077715158463,
36
+ "rewards/rejected": 0.009320415556430817,
37
+ "step": 10
38
+ },
39
+ {
40
+ "epoch": 0.1,
41
+ "learning_rate": 4.761904761904762e-06,
42
+ "logits/chosen": 0.10182257741689682,
43
+ "logits/rejected": 0.21816711127758026,
44
+ "logps/chosen": -337.0960388183594,
45
+ "logps/rejected": -311.6546936035156,
46
+ "loss": 0.3814,
47
+ "rewards/accuracies": 0.48750001192092896,
48
+ "rewards/chosen": -0.020739946514368057,
49
+ "rewards/margins": 0.02319016307592392,
50
+ "rewards/rejected": -0.04393010586500168,
51
+ "step": 20
52
+ },
53
+ {
54
+ "epoch": 0.14,
55
+ "learning_rate": 4.97147773390341e-06,
56
+ "logits/chosen": 0.10252387821674347,
57
+ "logits/rejected": 0.20911017060279846,
58
+ "logps/chosen": -333.58074951171875,
59
+ "logps/rejected": -314.22686767578125,
60
+ "loss": 0.3406,
61
+ "rewards/accuracies": 0.5562499761581421,
62
+ "rewards/chosen": 0.03605792671442032,
63
+ "rewards/margins": 0.06152229756116867,
64
+ "rewards/rejected": -0.0254643764346838,
65
+ "step": 30
66
+ },
67
+ {
68
+ "epoch": 0.19,
69
+ "learning_rate": 4.873717504456219e-06,
70
+ "logits/chosen": 0.1256047487258911,
71
+ "logits/rejected": 0.18428723514080048,
72
+ "logps/chosen": -361.2447509765625,
73
+ "logps/rejected": -337.5652770996094,
74
+ "loss": 0.3075,
75
+ "rewards/accuracies": 0.637499988079071,
76
+ "rewards/chosen": 0.1919463872909546,
77
+ "rewards/margins": 0.11055928468704224,
78
+ "rewards/rejected": 0.08138711750507355,
79
+ "step": 40
80
+ },
81
+ {
82
+ "epoch": 0.24,
83
+ "learning_rate": 4.709119209978242e-06,
84
+ "logits/chosen": 0.13542751967906952,
85
+ "logits/rejected": 0.17227646708488464,
86
+ "logps/chosen": -339.5179138183594,
87
+ "logps/rejected": -339.02984619140625,
88
+ "loss": 0.3253,
89
+ "rewards/accuracies": 0.6499999761581421,
90
+ "rewards/chosen": 0.29387253522872925,
91
+ "rewards/margins": 0.18539837002754211,
92
+ "rewards/rejected": 0.10847418010234833,
93
+ "step": 50
94
+ },
95
+ {
96
+ "epoch": 0.29,
97
+ "learning_rate": 4.482317534878901e-06,
98
+ "logits/chosen": 0.14710070192813873,
99
+ "logits/rejected": 0.1668437272310257,
100
+ "logps/chosen": -338.6156311035156,
101
+ "logps/rejected": -317.2750549316406,
102
+ "loss": 0.3199,
103
+ "rewards/accuracies": 0.637499988079071,
104
+ "rewards/chosen": 0.2612247169017792,
105
+ "rewards/margins": 0.14398948848247528,
106
+ "rewards/rejected": 0.1172352284193039,
107
+ "step": 60
108
+ },
109
+ {
110
+ "epoch": 0.34,
111
+ "learning_rate": 4.199698658255298e-06,
112
+ "logits/chosen": 0.10598815977573395,
113
+ "logits/rejected": 0.15820710361003876,
114
+ "logps/chosen": -339.61456298828125,
115
+ "logps/rejected": -329.31536865234375,
116
+ "loss": 0.2793,
117
+ "rewards/accuracies": 0.699999988079071,
118
+ "rewards/chosen": 0.3697130084037781,
119
+ "rewards/margins": 0.20087119936943054,
120
+ "rewards/rejected": 0.16884183883666992,
121
+ "step": 70
122
+ },
123
+ {
124
+ "epoch": 0.38,
125
+ "learning_rate": 3.869220434746509e-06,
126
+ "logits/chosen": 0.07073510438203812,
127
+ "logits/rejected": 0.15016858279705048,
128
+ "logps/chosen": -314.06280517578125,
129
+ "logps/rejected": -323.94268798828125,
130
+ "loss": 0.2928,
131
+ "rewards/accuracies": 0.6625000238418579,
132
+ "rewards/chosen": 0.3701319396495819,
133
+ "rewards/margins": 0.16388371586799622,
134
+ "rewards/rejected": 0.2062481939792633,
135
+ "step": 80
136
+ },
137
+ {
138
+ "epoch": 0.43,
139
+ "learning_rate": 3.5001883208580668e-06,
140
+ "logits/chosen": 0.12372653186321259,
141
+ "logits/rejected": 0.20820951461791992,
142
+ "logps/chosen": -373.20831298828125,
143
+ "logps/rejected": -339.1376037597656,
144
+ "loss": 0.2895,
145
+ "rewards/accuracies": 0.637499988079071,
146
+ "rewards/chosen": 0.33848610520362854,
147
+ "rewards/margins": 0.1839863359928131,
148
+ "rewards/rejected": 0.15449976921081543,
149
+ "step": 90
150
+ },
151
+ {
152
+ "epoch": 0.48,
153
+ "learning_rate": 3.102993356121938e-06,
154
+ "logits/chosen": 0.11043532192707062,
155
+ "logits/rejected": 0.21166983246803284,
156
+ "logps/chosen": -332.3284606933594,
157
+ "logps/rejected": -328.7100524902344,
158
+ "loss": 0.3023,
159
+ "rewards/accuracies": 0.643750011920929,
160
+ "rewards/chosen": 0.28233280777931213,
161
+ "rewards/margins": 0.1811174899339676,
162
+ "rewards/rejected": 0.10121532529592514,
163
+ "step": 100
164
+ },
165
+ {
166
+ "epoch": 0.53,
167
+ "learning_rate": 2.6888195769001147e-06,
168
+ "logits/chosen": 0.12863442301750183,
169
+ "logits/rejected": 0.2186942994594574,
170
+ "logps/chosen": -322.6536560058594,
171
+ "logps/rejected": -326.0906066894531,
172
+ "loss": 0.2747,
173
+ "rewards/accuracies": 0.75,
174
+ "rewards/chosen": 0.3540286421775818,
175
+ "rewards/margins": 0.2065799981355667,
176
+ "rewards/rejected": 0.14744864404201508,
177
+ "step": 110
178
+ },
179
+ {
180
+ "epoch": 0.58,
181
+ "learning_rate": 2.269329101341745e-06,
182
+ "logits/chosen": 0.11101800203323364,
183
+ "logits/rejected": 0.21909146010875702,
184
+ "logps/chosen": -386.37506103515625,
185
+ "logps/rejected": -338.628662109375,
186
+ "loss": 0.2594,
187
+ "rewards/accuracies": 0.7250000238418579,
188
+ "rewards/chosen": 0.32755765318870544,
189
+ "rewards/margins": 0.19837772846221924,
190
+ "rewards/rejected": 0.1291799247264862,
191
+ "step": 120
192
+ },
193
+ {
194
+ "epoch": 0.62,
195
+ "learning_rate": 1.856333752729311e-06,
196
+ "logits/chosen": 0.108365498483181,
197
+ "logits/rejected": 0.21159549057483673,
198
+ "logps/chosen": -354.3942565917969,
199
+ "logps/rejected": -350.58026123046875,
200
+ "loss": 0.2845,
201
+ "rewards/accuracies": 0.6875,
202
+ "rewards/chosen": 0.32795828580856323,
203
+ "rewards/margins": 0.22649607062339783,
204
+ "rewards/rejected": 0.10146218538284302,
205
+ "step": 130
206
+ },
207
+ {
208
+ "epoch": 0.67,
209
+ "learning_rate": 1.4614624674952843e-06,
210
+ "logits/chosen": 0.13152627646923065,
211
+ "logits/rejected": 0.1252177655696869,
212
+ "logps/chosen": -310.82440185546875,
213
+ "logps/rejected": -306.65057373046875,
214
+ "loss": 0.3034,
215
+ "rewards/accuracies": 0.625,
216
+ "rewards/chosen": 0.28935328125953674,
217
+ "rewards/margins": 0.17421108484268188,
218
+ "rewards/rejected": 0.11514218151569366,
219
+ "step": 140
220
+ },
221
+ {
222
+ "epoch": 0.72,
223
+ "learning_rate": 1.0958338528840893e-06,
224
+ "logits/chosen": 0.14218227565288544,
225
+ "logits/rejected": 0.17058388888835907,
226
+ "logps/chosen": -365.53497314453125,
227
+ "logps/rejected": -336.30670166015625,
228
+ "loss": 0.279,
229
+ "rewards/accuracies": 0.7124999761581421,
230
+ "rewards/chosen": 0.34200993180274963,
231
+ "rewards/margins": 0.18471720814704895,
232
+ "rewards/rejected": 0.1572926938533783,
233
+ "step": 150
234
+ },
235
+ {
236
+ "epoch": 0.77,
237
+ "learning_rate": 7.697431142327633e-07,
238
+ "logits/chosen": 0.1493266373872757,
239
+ "logits/rejected": 0.1491091400384903,
240
+ "logps/chosen": -363.55841064453125,
241
+ "logps/rejected": -332.8086853027344,
242
+ "loss": 0.2657,
243
+ "rewards/accuracies": 0.6499999761581421,
244
+ "rewards/chosen": 0.3300887942314148,
245
+ "rewards/margins": 0.20138521492481232,
246
+ "rewards/rejected": 0.12870360910892487,
247
+ "step": 160
248
+ },
249
+ {
250
+ "epoch": 0.82,
251
+ "learning_rate": 4.923721672305148e-07,
252
+ "logits/chosen": 0.09402619302272797,
253
+ "logits/rejected": 0.1920977234840393,
254
+ "logps/chosen": -334.378662109375,
255
+ "logps/rejected": -331.62322998046875,
256
+ "loss": 0.2685,
257
+ "rewards/accuracies": 0.65625,
258
+ "rewards/chosen": 0.3077259361743927,
259
+ "rewards/margins": 0.17248141765594482,
260
+ "rewards/rejected": 0.13524451851844788,
261
+ "step": 170
262
+ },
263
+ {
264
+ "epoch": 0.86,
265
+ "learning_rate": 2.7153109768518926e-07,
266
+ "logits/chosen": 0.11278879642486572,
267
+ "logits/rejected": 0.18854503333568573,
268
+ "logps/chosen": -388.7989501953125,
269
+ "logps/rejected": -336.02105712890625,
270
+ "loss": 0.2612,
271
+ "rewards/accuracies": 0.800000011920929,
272
+ "rewards/chosen": 0.34268996119499207,
273
+ "rewards/margins": 0.21422357857227325,
274
+ "rewards/rejected": 0.12846639752388,
275
+ "step": 180
276
+ },
277
+ {
278
+ "epoch": 0.91,
279
+ "learning_rate": 1.1343824865573422e-07,
280
+ "logits/chosen": 0.09929057955741882,
281
+ "logits/rejected": 0.15045389533042908,
282
+ "logps/chosen": -343.9807434082031,
283
+ "logps/rejected": -318.48028564453125,
284
+ "loss": 0.2662,
285
+ "rewards/accuracies": 0.6937500238418579,
286
+ "rewards/chosen": 0.29964178800582886,
287
+ "rewards/margins": 0.17701460421085358,
288
+ "rewards/rejected": 0.1226271539926529,
289
+ "step": 190
290
+ },
291
+ {
292
+ "epoch": 0.96,
293
+ "learning_rate": 2.2545127157831416e-08,
294
+ "logits/chosen": 0.07950419932603836,
295
+ "logits/rejected": 0.178897887468338,
296
+ "logps/chosen": -297.21661376953125,
297
+ "logps/rejected": -294.276611328125,
298
+ "loss": 0.2553,
299
+ "rewards/accuracies": 0.6812499761581421,
300
+ "rewards/chosen": 0.3100071847438812,
301
+ "rewards/margins": 0.16973480582237244,
302
+ "rewards/rejected": 0.14027239382266998,
303
+ "step": 200
304
+ },
305
+ {
306
+ "epoch": 1.0,
307
+ "step": 208,
308
+ "total_flos": 0.0,
309
+ "train_loss": 0.29713730256144816,
310
+ "train_runtime": 2891.8659,
311
+ "train_samples_per_second": 3.458,
312
+ "train_steps_per_second": 0.072
313
+ }
314
+ ],
315
+ "logging_steps": 10,
316
+ "max_steps": 208,
317
+ "num_input_tokens_seen": 0,
318
+ "num_train_epochs": 1,
319
+ "save_steps": 100,
320
+ "total_flos": 0.0,
321
+ "train_batch_size": 4,
322
+ "trial_name": null,
323
+ "trial_params": null
324
+ }