BraylonDash commited on
Commit
79d9d19
1 Parent(s): de0b05c

Model save

Browse files
README.md CHANGED
@@ -2,14 +2,10 @@
2
  license: mit
3
  library_name: peft
4
  tags:
5
- - alignment-handbook
6
- - generated_from_trainer
7
  - trl
8
  - dpo
9
  - generated_from_trainer
10
  base_model: microsoft/phi-2
11
- datasets:
12
- - HuggingFaceH4/ultrafeedback_binarized
13
  model-index:
14
  - name: phi-2-gpo-test-longest-iter-0
15
  results: []
@@ -20,7 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
20
 
21
  # phi-2-gpo-test-longest-iter-0
22
 
23
- This model is a fine-tuned version of [lole25/phi-2-sft-ultrachat-lora](https://huggingface.co/lole25/phi-2-sft-ultrachat-lora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 
 
 
 
 
 
 
 
 
 
24
 
25
  ## Model description
26
 
@@ -53,6 +59,9 @@ The following hyperparameters were used during training:
53
 
54
  ### Training results
55
 
 
 
 
56
 
57
 
58
  ### Framework versions
 
2
  license: mit
3
  library_name: peft
4
  tags:
 
 
5
  - trl
6
  - dpo
7
  - generated_from_trainer
8
  base_model: microsoft/phi-2
 
 
9
  model-index:
10
  - name: phi-2-gpo-test-longest-iter-0
11
  results: []
 
16
 
17
  # phi-2-gpo-test-longest-iter-0
18
 
19
+ This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the None dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 0.0103
22
+ - Rewards/chosen: 0.0007
23
+ - Rewards/rejected: 0.0005
24
+ - Rewards/accuracies: 0.5035
25
+ - Rewards/margins: 0.0001
26
+ - Logps/rejected: -233.4872
27
+ - Logps/chosen: -256.5486
28
+ - Logits/rejected: 0.8934
29
+ - Logits/chosen: 0.8359
30
 
31
  ## Model description
32
 
 
59
 
60
  ### Training results
61
 
62
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
63
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
64
+ | 0.0101 | 1.6 | 100 | 0.0102 | 0.0004 | 0.0001 | 0.5020 | 0.0003 | -233.5265 | -256.5746 | 0.8922 | 0.8348 |
65
 
66
 
67
  ### Framework versions
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b950c475743272d17772f4131c649574db6ee1573ed2f4de766fd10c7a5000cf
3
  size 41977616
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65ab005bbdbaaed810abd6c2e2f8124e0bd2ae749447f487dda995411fd117b5
3
  size 41977616
all_results.json CHANGED
@@ -1,8 +1,21 @@
1
  {
2
- "epoch": 1.33,
3
- "train_loss": 0.0022073863074183464,
4
- "train_runtime": 8.0706,
5
- "train_samples": 30567,
6
- "train_samples_per_second": 2.478,
7
- "train_steps_per_second": 0.248
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  }
 
1
  {
2
+ "epoch": 1.98,
3
+ "eval_logits/chosen": 0.8359472155570984,
4
+ "eval_logits/rejected": 0.893373429775238,
5
+ "eval_logps/chosen": -256.5486145019531,
6
+ "eval_logps/rejected": -233.4872283935547,
7
+ "eval_loss": 0.010263882577419281,
8
+ "eval_rewards/accuracies": 0.5034999847412109,
9
+ "eval_rewards/chosen": 0.00065155781339854,
10
+ "eval_rewards/margins": 0.00013779415166936815,
11
+ "eval_rewards/rejected": 0.0005137636326253414,
12
+ "eval_runtime": 535.2197,
13
+ "eval_samples": 2000,
14
+ "eval_samples_per_second": 3.737,
15
+ "eval_steps_per_second": 0.934,
16
+ "train_loss": 0.01022457688926689,
17
+ "train_runtime": 1632.1621,
18
+ "train_samples": 61135,
19
+ "train_samples_per_second": 1.225,
20
+ "train_steps_per_second": 0.076
21
  }
eval_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.98,
3
+ "eval_logits/chosen": 0.8359472155570984,
4
+ "eval_logits/rejected": 0.893373429775238,
5
+ "eval_logps/chosen": -256.5486145019531,
6
+ "eval_logps/rejected": -233.4872283935547,
7
+ "eval_loss": 0.010263882577419281,
8
+ "eval_rewards/accuracies": 0.5034999847412109,
9
+ "eval_rewards/chosen": 0.00065155781339854,
10
+ "eval_rewards/margins": 0.00013779415166936815,
11
+ "eval_rewards/rejected": 0.0005137636326253414,
12
+ "eval_runtime": 535.2197,
13
+ "eval_samples": 2000,
14
+ "eval_samples_per_second": 3.737,
15
+ "eval_steps_per_second": 0.934
16
+ }
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
- "epoch": 1.33,
3
- "train_loss": 0.0022073863074183464,
4
- "train_runtime": 8.0706,
5
- "train_samples": 30567,
6
- "train_samples_per_second": 2.478,
7
- "train_steps_per_second": 0.248
8
  }
 
1
  {
2
+ "epoch": 1.98,
3
+ "train_loss": 0.01022457688926689,
4
+ "train_runtime": 1632.1621,
5
+ "train_samples": 61135,
6
+ "train_samples_per_second": 1.225,
7
+ "train_steps_per_second": 0.076
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 1.3333333333333333,
5
  "eval_steps": 100,
6
- "global_step": 2,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
- "epoch": 1.0,
13
- "learning_rate": 5e-06,
14
- "logits/chosen": 0.701408326625824,
15
- "logits/rejected": 0.9044769406318665,
16
- "logps/chosen": -268.0211486816406,
17
- "logps/rejected": -240.4591522216797,
18
- "loss": 0.0033,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -23,17 +23,201 @@
23
  "step": 1
24
  },
25
  {
26
- "epoch": 1.33,
27
- "step": 2,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  "total_flos": 0.0,
29
- "train_loss": 0.0022073863074183464,
30
- "train_runtime": 8.0706,
31
- "train_samples_per_second": 2.478,
32
- "train_steps_per_second": 0.248
33
  }
34
  ],
35
  "logging_steps": 10,
36
- "max_steps": 2,
37
  "num_input_tokens_seen": 0,
38
  "num_train_epochs": 2,
39
  "save_steps": 100,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.984,
5
  "eval_steps": 100,
6
+ "global_step": 124,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
+ "epoch": 0.02,
13
+ "learning_rate": 3.846153846153847e-07,
14
+ "logits/chosen": 0.9600980877876282,
15
+ "logits/rejected": 1.094868540763855,
16
+ "logps/chosen": -119.03445434570312,
17
+ "logps/rejected": -112.18000030517578,
18
+ "loss": 0.0102,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
23
  "step": 1
24
  },
25
  {
26
+ "epoch": 0.16,
27
+ "learning_rate": 3.846153846153847e-06,
28
+ "logits/chosen": 0.8411452770233154,
29
+ "logits/rejected": 0.8130205273628235,
30
+ "logps/chosen": -142.57501220703125,
31
+ "logps/rejected": -130.0977325439453,
32
+ "loss": 0.0103,
33
+ "rewards/accuracies": 0.3472222089767456,
34
+ "rewards/chosen": 0.00034333759685978293,
35
+ "rewards/margins": -0.00018329803424421698,
36
+ "rewards/rejected": 0.0005266356747597456,
37
+ "step": 10
38
+ },
39
+ {
40
+ "epoch": 0.32,
41
+ "learning_rate": 4.951096619903317e-06,
42
+ "logits/chosen": 0.8094542622566223,
43
+ "logits/rejected": 0.8163386583328247,
44
+ "logps/chosen": -155.31309509277344,
45
+ "logps/rejected": -143.37014770507812,
46
+ "loss": 0.0103,
47
+ "rewards/accuracies": 0.40625,
48
+ "rewards/chosen": 0.000520143483299762,
49
+ "rewards/margins": -0.0002155094116460532,
50
+ "rewards/rejected": 0.0007356529822573066,
51
+ "step": 20
52
+ },
53
+ {
54
+ "epoch": 0.48,
55
+ "learning_rate": 4.716164218065246e-06,
56
+ "logits/chosen": 0.8769502639770508,
57
+ "logits/rejected": 0.9219558835029602,
58
+ "logps/chosen": -154.8149871826172,
59
+ "logps/rejected": -141.13705444335938,
60
+ "loss": 0.0103,
61
+ "rewards/accuracies": 0.3812499940395355,
62
+ "rewards/chosen": 0.00016751736984588206,
63
+ "rewards/margins": -0.000498370616696775,
64
+ "rewards/rejected": 0.0006658880738541484,
65
+ "step": 30
66
+ },
67
+ {
68
+ "epoch": 0.64,
69
+ "learning_rate": 4.3048902348863116e-06,
70
+ "logits/chosen": 0.8127928972244263,
71
+ "logits/rejected": 0.8382323384284973,
72
+ "logps/chosen": -172.5569610595703,
73
+ "logps/rejected": -157.7318115234375,
74
+ "loss": 0.0102,
75
+ "rewards/accuracies": 0.42500001192092896,
76
+ "rewards/chosen": 0.00022131178411655128,
77
+ "rewards/margins": 0.00017877723439596593,
78
+ "rewards/rejected": 4.253460792824626e-05,
79
+ "step": 40
80
+ },
81
+ {
82
+ "epoch": 0.8,
83
+ "learning_rate": 3.7500000000000005e-06,
84
+ "logits/chosen": 0.8299118280410767,
85
+ "logits/rejected": 0.8132155537605286,
86
+ "logps/chosen": -160.2462615966797,
87
+ "logps/rejected": -146.29754638671875,
88
+ "loss": 0.0102,
89
+ "rewards/accuracies": 0.42500001192092896,
90
+ "rewards/chosen": 0.00015722319949418306,
91
+ "rewards/margins": 7.946729601826519e-05,
92
+ "rewards/rejected": 7.775588892400265e-05,
93
+ "step": 50
94
+ },
95
+ {
96
+ "epoch": 0.96,
97
+ "learning_rate": 3.0956464785579125e-06,
98
+ "logits/chosen": 0.7370177507400513,
99
+ "logits/rejected": 0.7227329611778259,
100
+ "logps/chosen": -147.09487915039062,
101
+ "logps/rejected": -134.1732940673828,
102
+ "loss": 0.0102,
103
+ "rewards/accuracies": 0.4312500059604645,
104
+ "rewards/chosen": -0.000294997647870332,
105
+ "rewards/margins": 6.74725151839084e-06,
106
+ "rewards/rejected": -0.00030174493440426886,
107
+ "step": 60
108
+ },
109
+ {
110
+ "epoch": 1.12,
111
+ "learning_rate": 2.39389699200963e-06,
112
+ "logits/chosen": 0.8728678822517395,
113
+ "logits/rejected": 0.8462715148925781,
114
+ "logps/chosen": -146.96482849121094,
115
+ "logps/rejected": -137.31175231933594,
116
+ "loss": 0.0102,
117
+ "rewards/accuracies": 0.39375001192092896,
118
+ "rewards/chosen": -0.00032994337379932404,
119
+ "rewards/margins": 0.00023689583758823574,
120
+ "rewards/rejected": -0.0005668391240760684,
121
+ "step": 70
122
+ },
123
+ {
124
+ "epoch": 1.28,
125
+ "learning_rate": 1.700590188571887e-06,
126
+ "logits/chosen": 0.852059543132782,
127
+ "logits/rejected": 0.896367073059082,
128
+ "logps/chosen": -156.40220642089844,
129
+ "logps/rejected": -141.6897430419922,
130
+ "loss": 0.0102,
131
+ "rewards/accuracies": 0.41874998807907104,
132
+ "rewards/chosen": -0.0007336369017139077,
133
+ "rewards/margins": 0.00012243367382325232,
134
+ "rewards/rejected": -0.0008560704882256687,
135
+ "step": 80
136
+ },
137
+ {
138
+ "epoch": 1.44,
139
+ "learning_rate": 1.0708929268538034e-06,
140
+ "logits/chosen": 0.8062575459480286,
141
+ "logits/rejected": 0.8183335065841675,
142
+ "logps/chosen": -159.78892517089844,
143
+ "logps/rejected": -148.1704864501953,
144
+ "loss": 0.0101,
145
+ "rewards/accuracies": 0.45625001192092896,
146
+ "rewards/chosen": -0.00028093549190089107,
147
+ "rewards/margins": 0.0007433668943122029,
148
+ "rewards/rejected": -0.001024302444420755,
149
+ "step": 90
150
+ },
151
+ {
152
+ "epoch": 1.6,
153
+ "learning_rate": 5.549106142039018e-07,
154
+ "logits/chosen": 0.7814786434173584,
155
+ "logits/rejected": 0.8064650297164917,
156
+ "logps/chosen": -149.96612548828125,
157
+ "logps/rejected": -137.0812225341797,
158
+ "loss": 0.0101,
159
+ "rewards/accuracies": 0.48750001192092896,
160
+ "rewards/chosen": -0.00046937749721109867,
161
+ "rewards/margins": 0.0005115901003591716,
162
+ "rewards/rejected": -0.0009809675393626094,
163
+ "step": 100
164
+ },
165
+ {
166
+ "epoch": 1.6,
167
+ "eval_logits/chosen": 0.8348284363746643,
168
+ "eval_logits/rejected": 0.8922409415245056,
169
+ "eval_logps/chosen": -256.5745544433594,
170
+ "eval_logps/rejected": -233.52647399902344,
171
+ "eval_loss": 0.010232986882328987,
172
+ "eval_rewards/accuracies": 0.5019999742507935,
173
+ "eval_rewards/chosen": 0.00039203467895276845,
174
+ "eval_rewards/margins": 0.0002707619860302657,
175
+ "eval_rewards/rejected": 0.00012127268564654514,
176
+ "eval_runtime": 532.7373,
177
+ "eval_samples_per_second": 3.754,
178
+ "eval_steps_per_second": 0.939,
179
+ "step": 100
180
+ },
181
+ {
182
+ "epoch": 1.76,
183
+ "learning_rate": 1.937002879188285e-07,
184
+ "logits/chosen": 0.8264827728271484,
185
+ "logits/rejected": 0.797345757484436,
186
+ "logps/chosen": -160.50473022460938,
187
+ "logps/rejected": -143.83724975585938,
188
+ "loss": 0.0102,
189
+ "rewards/accuracies": 0.45625001192092896,
190
+ "rewards/chosen": -0.0012062744935974479,
191
+ "rewards/margins": 5.5822358262958005e-05,
192
+ "rewards/rejected": -0.0012620969209820032,
193
+ "step": 110
194
+ },
195
+ {
196
+ "epoch": 1.92,
197
+ "learning_rate": 1.6003680950742728e-08,
198
+ "logits/chosen": 0.8524961471557617,
199
+ "logits/rejected": 0.8347142934799194,
200
+ "logps/chosen": -145.30233764648438,
201
+ "logps/rejected": -132.08731079101562,
202
+ "loss": 0.0102,
203
+ "rewards/accuracies": 0.4124999940395355,
204
+ "rewards/chosen": -0.0007318368298001587,
205
+ "rewards/margins": 0.0001280030992347747,
206
+ "rewards/rejected": -0.0008598399581387639,
207
+ "step": 120
208
+ },
209
+ {
210
+ "epoch": 1.98,
211
+ "step": 124,
212
  "total_flos": 0.0,
213
+ "train_loss": 0.01022457688926689,
214
+ "train_runtime": 1632.1621,
215
+ "train_samples_per_second": 1.225,
216
+ "train_steps_per_second": 0.076
217
  }
218
  ],
219
  "logging_steps": 10,
220
+ "max_steps": 124,
221
  "num_input_tokens_seen": 0,
222
  "num_train_epochs": 2,
223
  "save_steps": 100,