Hugging Face's logo

DUAL-GPO
/
phi-2-gpo-v25-i1

PEFT
TensorBoard
Safetensors
phi
alignment-handbook
Generated from Trainer
trl
dpo
custom_code
Model card Files Metrics Community
lole25 commited on
Commit
25a262a
•
1 Parent(s): 3f08b69

Model save

Browse files
Files changed (9) hide show
  1. README.md +62 -0
  2. adapter_config.json +3 -3
  3. adapter_model.safetensors +1 -1
  4. all_results.json +8 -0
  5. runs/May15_10-26-23_gpu4-119-5/events.out.tfevents.1715732883.gpu4-119-5.2950679.0 +2 -2
  6. runs/May15_11-47-43_gpu4-119-5/events.out.tfevents.1715737777.gpu4-119-5.3002159.0 +3 -0
  7. train_results.json +8 -0
  8. trainer_state.json +688 -0
  9. training_args.bin +1 -1
README.md ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: mit
3
+ library_name: peft
4
+ tags:
5
+ - trl
6
+ - dpo
7
+ - generated_from_trainer
8
+ base_model: DUAL-GPO/phi-2-gpo-new-i0
9
+ model-index:
10
+ - name: phi-2-gpo-v25-i1
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # phi-2-gpo-v25-i1
18
+
19
+ This model is a fine-tuned version of [DUAL-GPO/phi-2-gpo-new-i0](https://huggingface.co/DUAL-GPO/phi-2-gpo-new-i0) on the None dataset.
20
+
21
+ ## Model description
22
+
23
+ More information needed
24
+
25
+ ## Intended uses & limitations
26
+
27
+ More information needed
28
+
29
+ ## Training and evaluation data
30
+
31
+ More information needed
32
+
33
+ ## Training procedure
34
+
35
+ ### Training hyperparameters
36
+
37
+ The following hyperparameters were used during training:
38
+ - learning_rate: 5e-06
39
+ - train_batch_size: 4
40
+ - eval_batch_size: 4
41
+ - seed: 42
42
+ - distributed_type: multi-GPU
43
+ - num_devices: 2
44
+ - gradient_accumulation_steps: 4
45
+ - total_train_batch_size: 32
46
+ - total_eval_batch_size: 8
47
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
48
+ - lr_scheduler_type: cosine
49
+ - lr_scheduler_warmup_ratio: 0.1
50
+ - num_epochs: 1
51
+
52
+ ### Training results
53
+
54
+
55
+
56
+ ### Framework versions
57
+
58
+ - PEFT 0.7.1
59
+ - Transformers 4.36.2
60
+ - Pytorch 2.1.2+cu121
61
+ - Datasets 2.14.6
62
+ - Tokenizers 0.15.2
adapter_config.json CHANGED
@@ -19,10 +19,10 @@
19
  "rank_pattern": {},
20
  "revision": null,
21
  "target_modules": [
22
- "q_proj",
23
- "v_proj",
24
  "dense",
25
- "k_proj"
 
 
26
  ],
27
  "task_type": "CAUSAL_LM"
28
  }
 
19
  "rank_pattern": {},
20
  "revision": null,
21
  "target_modules": [
 
 
22
  "dense",
23
+ "k_proj",
24
+ "q_proj",
25
+ "v_proj"
26
  ],
27
  "task_type": "CAUSAL_LM"
28
  }
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:19e51ed2d91041e6f40e3543d1b82f8c7a68d227d4a24098e0d3d9a003f5f51d
3
  size 167807296
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af8569cd28cf0c0b22e2167fd7ee12fd496e94b449a692343cd32dccd75c5beb
3
  size 167807296
all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.105732237171923,
4
+ "train_runtime": 2052.0868,
5
+ "train_samples": 15000,
6
+ "train_samples_per_second": 7.31,
7
+ "train_steps_per_second": 0.228
8
+ }
runs/May15_10-26-23_gpu4-119-5/events.out.tfevents.1715732883.gpu4-119-5.2950679.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6f75a04de6cdaa0297ae70a71de2c4d0b571afba081d466dec4f4b90888279fd
3
- size 24338
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:caa19c796054bce6921fb4bd4b091210445e9f3521149665c74890352dea6f22
3
+ size 28776
runs/May15_11-47-43_gpu4-119-5/events.out.tfevents.1715737777.gpu4-119-5.3002159.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f26a10f96129f77d21076694f82a2df386c98e4e4ff52dd4971c1a780c270b4
3
+ size 15325
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.105732237171923,
4
+ "train_runtime": 2052.0868,
5
+ "train_samples": 15000,
6
+ "train_samples_per_second": 7.31,
7
+ "train_steps_per_second": 0.228
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,688 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9984,
5
+ "eval_steps": 500,
6
+ "global_step": 468,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "learning_rate": 1.0638297872340426e-07,
14
+ "logits/chosen": 0.1422731578350067,
15
+ "logits/rejected": 0.3313992917537689,
16
+ "logps/chosen": -382.77789306640625,
17
+ "logps/rejected": -331.4576416015625,
18
+ "loss": 0.4045,
19
+ "rewards/accuracies": 0.0,
20
+ "rewards/chosen": 0.0,
21
+ "rewards/margins": 0.0,
22
+ "rewards/rejected": 0.0,
23
+ "step": 1
24
+ },
25
+ {
26
+ "epoch": 0.02,
27
+ "learning_rate": 1.0638297872340427e-06,
28
+ "logits/chosen": 0.11944054067134857,
29
+ "logits/rejected": 0.18962062895298004,
30
+ "logps/chosen": -326.4944763183594,
31
+ "logps/rejected": -291.0369873046875,
32
+ "loss": 0.3779,
33
+ "rewards/accuracies": 0.4236111044883728,
34
+ "rewards/chosen": 5.325177335180342e-05,
35
+ "rewards/margins": 5.4291256674332544e-05,
36
+ "rewards/rejected": -1.039480139297666e-06,
37
+ "step": 10
38
+ },
39
+ {
40
+ "epoch": 0.04,
41
+ "learning_rate": 2.1276595744680853e-06,
42
+ "logits/chosen": 0.12437532097101212,
43
+ "logits/rejected": 0.17575038969516754,
44
+ "logps/chosen": -355.1729736328125,
45
+ "logps/rejected": -326.6689453125,
46
+ "loss": 0.379,
47
+ "rewards/accuracies": 0.4749999940395355,
48
+ "rewards/chosen": 0.0001221647544298321,
49
+ "rewards/margins": 0.00025352081866003573,
50
+ "rewards/rejected": -0.0001313560496782884,
51
+ "step": 20
52
+ },
53
+ {
54
+ "epoch": 0.06,
55
+ "learning_rate": 3.191489361702128e-06,
56
+ "logits/chosen": 0.14455276727676392,
57
+ "logits/rejected": 0.17237640917301178,
58
+ "logps/chosen": -338.4366455078125,
59
+ "logps/rejected": -344.5356140136719,
60
+ "loss": 0.3714,
61
+ "rewards/accuracies": 0.512499988079071,
62
+ "rewards/chosen": -8.463774429401383e-05,
63
+ "rewards/margins": 0.00026104756398126483,
64
+ "rewards/rejected": -0.00034568531555123627,
65
+ "step": 30
66
+ },
67
+ {
68
+ "epoch": 0.09,
69
+ "learning_rate": 4.255319148936171e-06,
70
+ "logits/chosen": 0.09771748632192612,
71
+ "logits/rejected": 0.20755767822265625,
72
+ "logps/chosen": -372.2322692871094,
73
+ "logps/rejected": -351.06793212890625,
74
+ "loss": 0.3867,
75
+ "rewards/accuracies": 0.643750011920929,
76
+ "rewards/chosen": -4.432376954355277e-05,
77
+ "rewards/margins": 0.0008093343349173665,
78
+ "rewards/rejected": -0.00085365807171911,
79
+ "step": 40
80
+ },
81
+ {
82
+ "epoch": 0.11,
83
+ "learning_rate": 4.999373573764188e-06,
84
+ "logits/chosen": 0.07760350406169891,
85
+ "logits/rejected": 0.154631569981575,
86
+ "logps/chosen": -344.04986572265625,
87
+ "logps/rejected": -310.27630615234375,
88
+ "loss": 0.3668,
89
+ "rewards/accuracies": 0.643750011920929,
90
+ "rewards/chosen": 0.0006088110385462642,
91
+ "rewards/margins": 0.0018962211906909943,
92
+ "rewards/rejected": -0.0012874103849753737,
93
+ "step": 50
94
+ },
95
+ {
96
+ "epoch": 0.13,
97
+ "learning_rate": 4.988245838331339e-06,
98
+ "logits/chosen": 0.14376303553581238,
99
+ "logits/rejected": 0.1840187907218933,
100
+ "logps/chosen": -348.7585754394531,
101
+ "logps/rejected": -347.42633056640625,
102
+ "loss": 0.3701,
103
+ "rewards/accuracies": 0.606249988079071,
104
+ "rewards/chosen": 0.0007757244748063385,
105
+ "rewards/margins": 0.0029361725319176912,
106
+ "rewards/rejected": -0.00216044788248837,
107
+ "step": 60
108
+ },
109
+ {
110
+ "epoch": 0.15,
111
+ "learning_rate": 4.963268819535228e-06,
112
+ "logits/chosen": 0.10374955832958221,
113
+ "logits/rejected": 0.20326051115989685,
114
+ "logps/chosen": -330.6069641113281,
115
+ "logps/rejected": -329.6276550292969,
116
+ "loss": 0.3494,
117
+ "rewards/accuracies": 0.6187499761581421,
118
+ "rewards/chosen": 0.0009276155615225434,
119
+ "rewards/margins": 0.005808635149151087,
120
+ "rewards/rejected": -0.004881019704043865,
121
+ "step": 70
122
+ },
123
+ {
124
+ "epoch": 0.17,
125
+ "learning_rate": 4.9245815365216115e-06,
126
+ "logits/chosen": 0.11984933912754059,
127
+ "logits/rejected": 0.19475053250789642,
128
+ "logps/chosen": -341.2612609863281,
129
+ "logps/rejected": -341.21063232421875,
130
+ "loss": 0.3772,
131
+ "rewards/accuracies": 0.71875,
132
+ "rewards/chosen": 0.0007902610814198852,
133
+ "rewards/margins": 0.014523616060614586,
134
+ "rewards/rejected": -0.013733354397118092,
135
+ "step": 80
136
+ },
137
+ {
138
+ "epoch": 0.19,
139
+ "learning_rate": 4.872399318152594e-06,
140
+ "logits/chosen": 0.09717285633087158,
141
+ "logits/rejected": 0.19757065176963806,
142
+ "logps/chosen": -383.49658203125,
143
+ "logps/rejected": -375.48980712890625,
144
+ "loss": 0.3443,
145
+ "rewards/accuracies": 0.643750011920929,
146
+ "rewards/chosen": -0.013060875236988068,
147
+ "rewards/margins": 0.018941307440400124,
148
+ "rewards/rejected": -0.03200218454003334,
149
+ "step": 90
150
+ },
151
+ {
152
+ "epoch": 0.21,
153
+ "learning_rate": 4.807012604511542e-06,
154
+ "logits/chosen": 0.09459100663661957,
155
+ "logits/rejected": 0.172474667429924,
156
+ "logps/chosen": -414.6573181152344,
157
+ "logps/rejected": -423.6068420410156,
158
+ "loss": 0.3207,
159
+ "rewards/accuracies": 0.699999988079071,
160
+ "rewards/chosen": -0.0480206236243248,
161
+ "rewards/margins": 0.05255536362528801,
162
+ "rewards/rejected": -0.10057598352432251,
163
+ "step": 100
164
+ },
165
+ {
166
+ "epoch": 0.23,
167
+ "learning_rate": 4.728785330347771e-06,
168
+ "logits/chosen": 0.17286503314971924,
169
+ "logits/rejected": 0.2384227067232132,
170
+ "logps/chosen": -535.0108032226562,
171
+ "logps/rejected": -592.4139404296875,
172
+ "loss": 0.3274,
173
+ "rewards/accuracies": 0.6312500238418579,
174
+ "rewards/chosen": -0.14901921153068542,
175
+ "rewards/margins": 0.07979042828083038,
176
+ "rewards/rejected": -0.22880962491035461,
177
+ "step": 110
178
+ },
179
+ {
180
+ "epoch": 0.26,
181
+ "learning_rate": 4.63815289945858e-06,
182
+ "logits/chosen": 0.17661482095718384,
183
+ "logits/rejected": 0.20918302237987518,
184
+ "logps/chosen": -542.838134765625,
185
+ "logps/rejected": -599.6415405273438,
186
+ "loss": 0.3159,
187
+ "rewards/accuracies": 0.6875,
188
+ "rewards/chosen": -0.1861373484134674,
189
+ "rewards/margins": 0.09151609241962433,
190
+ "rewards/rejected": -0.27765345573425293,
191
+ "step": 120
192
+ },
193
+ {
194
+ "epoch": 0.28,
195
+ "learning_rate": 4.535619761282989e-06,
196
+ "logits/chosen": 0.13282707333564758,
197
+ "logits/rejected": 0.18375340104103088,
198
+ "logps/chosen": -484.95953369140625,
199
+ "logps/rejected": -634.0150146484375,
200
+ "loss": 0.2988,
201
+ "rewards/accuracies": 0.7124999761581421,
202
+ "rewards/chosen": -0.15236204862594604,
203
+ "rewards/margins": 0.14143863320350647,
204
+ "rewards/rejected": -0.2938006520271301,
205
+ "step": 130
206
+ },
207
+ {
208
+ "epoch": 0.3,
209
+ "learning_rate": 4.42175660319555e-06,
210
+ "logits/chosen": 0.18953588604927063,
211
+ "logits/rejected": 0.19973225891590118,
212
+ "logps/chosen": -518.3944091796875,
213
+ "logps/rejected": -572.0684814453125,
214
+ "loss": 0.3261,
215
+ "rewards/accuracies": 0.6499999761581421,
216
+ "rewards/chosen": -0.17030271887779236,
217
+ "rewards/margins": 0.08826728165149689,
218
+ "rewards/rejected": -0.25857001543045044,
219
+ "step": 140
220
+ },
221
+ {
222
+ "epoch": 0.32,
223
+ "learning_rate": 4.297197174127619e-06,
224
+ "logits/chosen": 0.14177818596363068,
225
+ "logits/rejected": 0.19645562767982483,
226
+ "logps/chosen": -483.99383544921875,
227
+ "logps/rejected": -587.1974487304688,
228
+ "loss": 0.3197,
229
+ "rewards/accuracies": 0.6312500238418579,
230
+ "rewards/chosen": -0.1454397737979889,
231
+ "rewards/margins": 0.09491118788719177,
232
+ "rewards/rejected": -0.24035099148750305,
233
+ "step": 150
234
+ },
235
+ {
236
+ "epoch": 0.34,
237
+ "learning_rate": 4.162634757195418e-06,
238
+ "logits/chosen": 0.1273745745420456,
239
+ "logits/rejected": 0.21790139377117157,
240
+ "logps/chosen": -491.3045959472656,
241
+ "logps/rejected": -556.7537841796875,
242
+ "loss": 0.3097,
243
+ "rewards/accuracies": 0.637499988079071,
244
+ "rewards/chosen": -0.15721774101257324,
245
+ "rewards/margins": 0.08056008815765381,
246
+ "rewards/rejected": -0.23777782917022705,
247
+ "step": 160
248
+ },
249
+ {
250
+ "epoch": 0.36,
251
+ "learning_rate": 4.018818310967843e-06,
252
+ "logits/chosen": 0.16942057013511658,
253
+ "logits/rejected": 0.25549784302711487,
254
+ "logps/chosen": -509.8619079589844,
255
+ "logps/rejected": -641.41748046875,
256
+ "loss": 0.3034,
257
+ "rewards/accuracies": 0.6812499761581421,
258
+ "rewards/chosen": -0.1676618903875351,
259
+ "rewards/margins": 0.12202789634466171,
260
+ "rewards/rejected": -0.2896897792816162,
261
+ "step": 170
262
+ },
263
+ {
264
+ "epoch": 0.38,
265
+ "learning_rate": 3.866548300851254e-06,
266
+ "logits/chosen": 0.15910522639751434,
267
+ "logits/rejected": 0.1666603833436966,
268
+ "logps/chosen": -499.43865966796875,
269
+ "logps/rejected": -589.9558715820312,
270
+ "loss": 0.3048,
271
+ "rewards/accuracies": 0.637499988079071,
272
+ "rewards/chosen": -0.17921218276023865,
273
+ "rewards/margins": 0.09624715149402618,
274
+ "rewards/rejected": -0.275459349155426,
275
+ "step": 180
276
+ },
277
+ {
278
+ "epoch": 0.41,
279
+ "learning_rate": 3.706672243793271e-06,
280
+ "logits/chosen": 0.11776936054229736,
281
+ "logits/rejected": 0.15903571248054504,
282
+ "logps/chosen": -492.0912170410156,
283
+ "logps/rejected": -582.832275390625,
284
+ "loss": 0.3032,
285
+ "rewards/accuracies": 0.637499988079071,
286
+ "rewards/chosen": -0.18043668568134308,
287
+ "rewards/margins": 0.09369800984859467,
288
+ "rewards/rejected": -0.27413469552993774,
289
+ "step": 190
290
+ },
291
+ {
292
+ "epoch": 0.43,
293
+ "learning_rate": 3.5400799911032357e-06,
294
+ "logits/chosen": 0.1436721682548523,
295
+ "logits/rejected": 0.1706967055797577,
296
+ "logps/chosen": -506.0414123535156,
297
+ "logps/rejected": -571.4830322265625,
298
+ "loss": 0.3173,
299
+ "rewards/accuracies": 0.581250011920929,
300
+ "rewards/chosen": -0.15588083863258362,
301
+ "rewards/margins": 0.07620591670274734,
302
+ "rewards/rejected": -0.23208674788475037,
303
+ "step": 200
304
+ },
305
+ {
306
+ "epoch": 0.45,
307
+ "learning_rate": 3.3676987756445894e-06,
308
+ "logits/chosen": 0.1567540317773819,
309
+ "logits/rejected": 0.21821483969688416,
310
+ "logps/chosen": -516.2056884765625,
311
+ "logps/rejected": -605.7388916015625,
312
+ "loss": 0.2942,
313
+ "rewards/accuracies": 0.699999988079071,
314
+ "rewards/chosen": -0.16281944513320923,
315
+ "rewards/margins": 0.11313279718160629,
316
+ "rewards/rejected": -0.2759522497653961,
317
+ "step": 210
318
+ },
319
+ {
320
+ "epoch": 0.47,
321
+ "learning_rate": 3.1904880509659397e-06,
322
+ "logits/chosen": 0.09412068128585815,
323
+ "logits/rejected": 0.26981237530708313,
324
+ "logps/chosen": -573.1748046875,
325
+ "logps/rejected": -600.4771728515625,
326
+ "loss": 0.3023,
327
+ "rewards/accuracies": 0.637499988079071,
328
+ "rewards/chosen": -0.1811535358428955,
329
+ "rewards/margins": 0.09422539919614792,
330
+ "rewards/rejected": -0.275378942489624,
331
+ "step": 220
332
+ },
333
+ {
334
+ "epoch": 0.49,
335
+ "learning_rate": 3.0094341510955697e-06,
336
+ "logits/chosen": 0.11837532371282578,
337
+ "logits/rejected": 0.19376713037490845,
338
+ "logps/chosen": -569.38037109375,
339
+ "logps/rejected": -616.151123046875,
340
+ "loss": 0.3027,
341
+ "rewards/accuracies": 0.5687500238418579,
342
+ "rewards/chosen": -0.21316485106945038,
343
+ "rewards/margins": 0.07764045149087906,
344
+ "rewards/rejected": -0.29080528020858765,
345
+ "step": 230
346
+ },
347
+ {
348
+ "epoch": 0.51,
349
+ "learning_rate": 2.825544800722376e-06,
350
+ "logits/chosen": 0.1659005582332611,
351
+ "logits/rejected": 0.2277708798646927,
352
+ "logps/chosen": -571.8699951171875,
353
+ "logps/rejected": -628.5322875976562,
354
+ "loss": 0.2852,
355
+ "rewards/accuracies": 0.637499988079071,
356
+ "rewards/chosen": -0.20772342383861542,
357
+ "rewards/margins": 0.10460670292377472,
358
+ "rewards/rejected": -0.31233012676239014,
359
+ "step": 240
360
+ },
361
+ {
362
+ "epoch": 0.53,
363
+ "learning_rate": 2.639843506318899e-06,
364
+ "logits/chosen": 0.1882917284965515,
365
+ "logits/rejected": 0.2411785125732422,
366
+ "logps/chosen": -557.4056396484375,
367
+ "logps/rejected": -673.3493041992188,
368
+ "loss": 0.2959,
369
+ "rewards/accuracies": 0.6937500238418579,
370
+ "rewards/chosen": -0.20086392760276794,
371
+ "rewards/margins": 0.11475809663534164,
372
+ "rewards/rejected": -0.3156220316886902,
373
+ "step": 250
374
+ },
375
+ {
376
+ "epoch": 0.55,
377
+ "learning_rate": 2.4533638594248094e-06,
378
+ "logits/chosen": 0.13379403948783875,
379
+ "logits/rejected": 0.19422146677970886,
380
+ "logps/chosen": -565.0435180664062,
381
+ "logps/rejected": -694.2730712890625,
382
+ "loss": 0.2895,
383
+ "rewards/accuracies": 0.6812499761581421,
384
+ "rewards/chosen": -0.23057802021503448,
385
+ "rewards/margins": 0.11758030951023102,
386
+ "rewards/rejected": -0.3481583297252655,
387
+ "step": 260
388
+ },
389
+ {
390
+ "epoch": 0.58,
391
+ "learning_rate": 2.2671437837980943e-06,
392
+ "logits/chosen": 0.1472124606370926,
393
+ "logits/rejected": 0.21264033019542694,
394
+ "logps/chosen": -553.3011474609375,
395
+ "logps/rejected": -670.6328125,
396
+ "loss": 0.285,
397
+ "rewards/accuracies": 0.643750011920929,
398
+ "rewards/chosen": -0.21868601441383362,
399
+ "rewards/margins": 0.12228033691644669,
400
+ "rewards/rejected": -0.3409663736820221,
401
+ "step": 270
402
+ },
403
+ {
404
+ "epoch": 0.6,
405
+ "learning_rate": 2.082219758453629e-06,
406
+ "logits/chosen": 0.15410706400871277,
407
+ "logits/rejected": 0.17184455692768097,
408
+ "logps/chosen": -515.2255859375,
409
+ "logps/rejected": -623.633544921875,
410
+ "loss": 0.296,
411
+ "rewards/accuracies": 0.606249988079071,
412
+ "rewards/chosen": -0.21407786011695862,
413
+ "rewards/margins": 0.105091392993927,
414
+ "rewards/rejected": -0.3191692531108856,
415
+ "step": 280
416
+ },
417
+ {
418
+ "epoch": 0.62,
419
+ "learning_rate": 1.899621048743019e-06,
420
+ "logits/chosen": 0.08964169770479202,
421
+ "logits/rejected": 0.16084159910678864,
422
+ "logps/chosen": -526.2240600585938,
423
+ "logps/rejected": -663.767822265625,
424
+ "loss": 0.3056,
425
+ "rewards/accuracies": 0.7124999761581421,
426
+ "rewards/chosen": -0.21246090531349182,
427
+ "rewards/margins": 0.12431824207305908,
428
+ "rewards/rejected": -0.3367791175842285,
429
+ "step": 290
430
+ },
431
+ {
432
+ "epoch": 0.64,
433
+ "learning_rate": 1.7203639775848423e-06,
434
+ "logits/chosen": 0.06412859261035919,
435
+ "logits/rejected": 0.1526188850402832,
436
+ "logps/chosen": -508.3192443847656,
437
+ "logps/rejected": -625.1074829101562,
438
+ "loss": 0.2862,
439
+ "rewards/accuracies": 0.699999988079071,
440
+ "rewards/chosen": -0.18599674105644226,
441
+ "rewards/margins": 0.11884822696447372,
442
+ "rewards/rejected": -0.3048449456691742,
443
+ "step": 300
444
+ },
445
+ {
446
+ "epoch": 0.66,
447
+ "learning_rate": 1.5454462687309445e-06,
448
+ "logits/chosen": 0.12014339864253998,
449
+ "logits/rejected": 0.17034398019313812,
450
+ "logps/chosen": -528.6392822265625,
451
+ "logps/rejected": -658.5401611328125,
452
+ "loss": 0.3008,
453
+ "rewards/accuracies": 0.6499999761581421,
454
+ "rewards/chosen": -0.2073144167661667,
455
+ "rewards/margins": 0.1150335818529129,
456
+ "rewards/rejected": -0.3223479390144348,
457
+ "step": 310
458
+ },
459
+ {
460
+ "epoch": 0.68,
461
+ "learning_rate": 1.3758414935535147e-06,
462
+ "logits/chosen": 0.1499626636505127,
463
+ "logits/rejected": 0.1570507287979126,
464
+ "logps/chosen": -577.6300659179688,
465
+ "logps/rejected": -674.1231079101562,
466
+ "loss": 0.2964,
467
+ "rewards/accuracies": 0.6812499761581421,
468
+ "rewards/chosen": -0.2138519585132599,
469
+ "rewards/margins": 0.10956285148859024,
470
+ "rewards/rejected": -0.32341480255126953,
471
+ "step": 320
472
+ },
473
+ {
474
+ "epoch": 0.7,
475
+ "learning_rate": 1.2124936522614622e-06,
476
+ "logits/chosen": 0.08589409291744232,
477
+ "logits/rejected": 0.18073198199272156,
478
+ "logps/chosen": -520.1068115234375,
479
+ "logps/rejected": -660.3935546875,
480
+ "loss": 0.284,
481
+ "rewards/accuracies": 0.6812499761581421,
482
+ "rewards/chosen": -0.17694085836410522,
483
+ "rewards/margins": 0.15071728825569153,
484
+ "rewards/rejected": -0.32765814661979675,
485
+ "step": 330
486
+ },
487
+ {
488
+ "epoch": 0.73,
489
+ "learning_rate": 1.0563119197063934e-06,
490
+ "logits/chosen": 0.09931635111570358,
491
+ "logits/rejected": 0.19644983112812042,
492
+ "logps/chosen": -560.4113159179688,
493
+ "logps/rejected": -655.904296875,
494
+ "loss": 0.2886,
495
+ "rewards/accuracies": 0.6499999761581421,
496
+ "rewards/chosen": -0.21122217178344727,
497
+ "rewards/margins": 0.09574859589338303,
498
+ "rewards/rejected": -0.3069707751274109,
499
+ "step": 340
500
+ },
501
+ {
502
+ "epoch": 0.75,
503
+ "learning_rate": 9.081655850224449e-07,
504
+ "logits/chosen": 0.10707017034292221,
505
+ "logits/rejected": 0.1817469596862793,
506
+ "logps/chosen": -537.7971801757812,
507
+ "logps/rejected": -607.695068359375,
508
+ "loss": 0.3062,
509
+ "rewards/accuracies": 0.59375,
510
+ "rewards/chosen": -0.21373042464256287,
511
+ "rewards/margins": 0.07750532776117325,
512
+ "rewards/rejected": -0.2912357449531555,
513
+ "step": 350
514
+ },
515
+ {
516
+ "epoch": 0.77,
517
+ "learning_rate": 7.688792132653111e-07,
518
+ "logits/chosen": 0.09279724210500717,
519
+ "logits/rejected": 0.14600899815559387,
520
+ "logps/chosen": -518.6719970703125,
521
+ "logps/rejected": -613.5050048828125,
522
+ "loss": 0.3102,
523
+ "rewards/accuracies": 0.6625000238418579,
524
+ "rewards/chosen": -0.20901672542095184,
525
+ "rewards/margins": 0.10807285457849503,
526
+ "rewards/rejected": -0.31708958745002747,
527
+ "step": 360
528
+ },
529
+ {
530
+ "epoch": 0.79,
531
+ "learning_rate": 6.392280559802341e-07,
532
+ "logits/chosen": 0.09189110994338989,
533
+ "logits/rejected": 0.14240947365760803,
534
+ "logps/chosen": -499.902587890625,
535
+ "logps/rejected": -645.5587158203125,
536
+ "loss": 0.2901,
537
+ "rewards/accuracies": 0.706250011920929,
538
+ "rewards/chosen": -0.19295868277549744,
539
+ "rewards/margins": 0.1267748475074768,
540
+ "rewards/rejected": -0.31973353028297424,
541
+ "step": 370
542
+ },
543
+ {
544
+ "epoch": 0.81,
545
+ "learning_rate": 5.199337362431792e-07,
546
+ "logits/chosen": 0.10996093600988388,
547
+ "logits/rejected": 0.17150555551052094,
548
+ "logps/chosen": -541.5203857421875,
549
+ "logps/rejected": -660.8944091796875,
550
+ "loss": 0.2869,
551
+ "rewards/accuracies": 0.71875,
552
+ "rewards/chosen": -0.19876644015312195,
553
+ "rewards/margins": 0.13654473423957825,
554
+ "rewards/rejected": -0.3353111743927002,
555
+ "step": 380
556
+ },
557
+ {
558
+ "epoch": 0.83,
559
+ "learning_rate": 4.1166023219176176e-07,
560
+ "logits/chosen": 0.08110027015209198,
561
+ "logits/rejected": 0.1370091438293457,
562
+ "logps/chosen": -551.21923828125,
563
+ "logps/rejected": -639.4774780273438,
564
+ "loss": 0.2989,
565
+ "rewards/accuracies": 0.6499999761581421,
566
+ "rewards/chosen": -0.2069006711244583,
567
+ "rewards/margins": 0.1049899011850357,
568
+ "rewards/rejected": -0.31189054250717163,
569
+ "step": 390
570
+ },
571
+ {
572
+ "epoch": 0.85,
573
+ "learning_rate": 3.150101814011136e-07,
574
+ "logits/chosen": 0.09940965473651886,
575
+ "logits/rejected": 0.14982450008392334,
576
+ "logps/chosen": -575.6122436523438,
577
+ "logps/rejected": -670.1174926757812,
578
+ "loss": 0.3034,
579
+ "rewards/accuracies": 0.625,
580
+ "rewards/chosen": -0.2270297110080719,
581
+ "rewards/margins": 0.09964431822299957,
582
+ "rewards/rejected": -0.3266740143299103,
583
+ "step": 400
584
+ },
585
+ {
586
+ "epoch": 0.87,
587
+ "learning_rate": 2.3052152667409289e-07,
588
+ "logits/chosen": 0.12830981612205505,
589
+ "logits/rejected": 0.12991704046726227,
590
+ "logps/chosen": -512.4361572265625,
591
+ "logps/rejected": -703.7268676757812,
592
+ "loss": 0.2663,
593
+ "rewards/accuracies": 0.731249988079071,
594
+ "rewards/chosen": -0.19513638317584991,
595
+ "rewards/margins": 0.15631332993507385,
596
+ "rewards/rejected": -0.3514496684074402,
597
+ "step": 410
598
+ },
599
+ {
600
+ "epoch": 0.9,
601
+ "learning_rate": 1.5866452191498488e-07,
602
+ "logits/chosen": 0.12449488788843155,
603
+ "logits/rejected": 0.12310103327035904,
604
+ "logps/chosen": -507.05352783203125,
605
+ "logps/rejected": -690.96337890625,
606
+ "loss": 0.3093,
607
+ "rewards/accuracies": 0.706250011920929,
608
+ "rewards/chosen": -0.19581075012683868,
609
+ "rewards/margins": 0.1489998996257782,
610
+ "rewards/rejected": -0.3448106646537781,
611
+ "step": 420
612
+ },
613
+ {
614
+ "epoch": 0.92,
615
+ "learning_rate": 9.983911475163727e-08,
616
+ "logits/chosen": 0.12271595001220703,
617
+ "logits/rejected": 0.11245056241750717,
618
+ "logps/chosen": -534.17724609375,
619
+ "logps/rejected": -601.7576904296875,
620
+ "loss": 0.2942,
621
+ "rewards/accuracies": 0.6187499761581421,
622
+ "rewards/chosen": -0.20346157252788544,
623
+ "rewards/margins": 0.10961834341287613,
624
+ "rewards/rejected": -0.3130798935890198,
625
+ "step": 430
626
+ },
627
+ {
628
+ "epoch": 0.94,
629
+ "learning_rate": 5.437272047405712e-08,
630
+ "logits/chosen": 0.10363437235355377,
631
+ "logits/rejected": 0.19413235783576965,
632
+ "logps/chosen": -529.4188232421875,
633
+ "logps/rejected": -675.1129150390625,
634
+ "loss": 0.2833,
635
+ "rewards/accuracies": 0.6812499761581421,
636
+ "rewards/chosen": -0.20683610439300537,
637
+ "rewards/margins": 0.13403114676475525,
638
+ "rewards/rejected": -0.3408672511577606,
639
+ "step": 440
640
+ },
641
+ {
642
+ "epoch": 0.96,
643
+ "learning_rate": 2.251839967945535e-08,
644
+ "logits/chosen": 0.08495406061410904,
645
+ "logits/rejected": 0.15138773620128632,
646
+ "logps/chosen": -553.8279418945312,
647
+ "logps/rejected": -693.7943725585938,
648
+ "loss": 0.3031,
649
+ "rewards/accuracies": 0.6937500238418579,
650
+ "rewards/chosen": -0.22638258337974548,
651
+ "rewards/margins": 0.11836589872837067,
652
+ "rewards/rejected": -0.34474846720695496,
653
+ "step": 450
654
+ },
655
+ {
656
+ "epoch": 0.98,
657
+ "learning_rate": 4.453449766758933e-09,
658
+ "logits/chosen": 0.11232316493988037,
659
+ "logits/rejected": 0.1733480840921402,
660
+ "logps/chosen": -540.5206298828125,
661
+ "logps/rejected": -626.1546630859375,
662
+ "loss": 0.2912,
663
+ "rewards/accuracies": 0.675000011920929,
664
+ "rewards/chosen": -0.18953736126422882,
665
+ "rewards/margins": 0.12424160540103912,
666
+ "rewards/rejected": -0.31377896666526794,
667
+ "step": 460
668
+ },
669
+ {
670
+ "epoch": 1.0,
671
+ "step": 468,
672
+ "total_flos": 0.0,
673
+ "train_loss": 0.105732237171923,
674
+ "train_runtime": 2052.0868,
675
+ "train_samples_per_second": 7.31,
676
+ "train_steps_per_second": 0.228
677
+ }
678
+ ],
679
+ "logging_steps": 10,
680
+ "max_steps": 468,
681
+ "num_input_tokens_seen": 0,
682
+ "num_train_epochs": 1,
683
+ "save_steps": 100,
684
+ "total_flos": 0.0,
685
+ "train_batch_size": 4,
686
+ "trial_name": null,
687
+ "trial_params": null
688
+ }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1f90b85a2e647c1076830f6ed25b6e1e96b08d57450e08d39506f500636bc3a0
3
  size 5816
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:424d34e46ecb42c6eba463445015b9492f6b36a676c92be44b67b60a695a5d0c
3
  size 5816