BraylonDash commited on
Commit
266f7bc
1 Parent(s): 098f8d7

Model save

Browse files
README.md CHANGED
@@ -2,14 +2,10 @@
2
  license: mit
3
  library_name: peft
4
  tags:
5
- - alignment-handbook
6
- - generated_from_trainer
7
  - trl
8
  - dpo
9
  - generated_from_trainer
10
  base_model: microsoft/phi-2
11
- datasets:
12
- - HuggingFaceH4/ultrafeedback_binarized
13
  model-index:
14
  - name: phi-2-gpo-test-longest-iter-2
15
  results: []
@@ -20,7 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
20
 
21
  # phi-2-gpo-test-longest-iter-2
22
 
23
- This model is a fine-tuned version of [DUAL-GPO/phi-2-gpo-test-longest-iter-1](https://huggingface.co/DUAL-GPO/phi-2-gpo-test-longest-iter-1) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 
 
 
 
 
 
 
 
 
 
24
 
25
  ## Model description
26
 
@@ -53,6 +59,9 @@ The following hyperparameters were used during training:
53
 
54
  ### Training results
55
 
 
 
 
56
 
57
 
58
  ### Framework versions
 
2
  license: mit
3
  library_name: peft
4
  tags:
 
 
5
  - trl
6
  - dpo
7
  - generated_from_trainer
8
  base_model: microsoft/phi-2
 
 
9
  model-index:
10
  - name: phi-2-gpo-test-longest-iter-2
11
  results: []
 
16
 
17
  # phi-2-gpo-test-longest-iter-2
18
 
19
+ This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the None dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 0.0108
22
+ - Rewards/chosen: -0.0005
23
+ - Rewards/rejected: -0.0003
24
+ - Rewards/accuracies: 0.4915
25
+ - Rewards/margins: -0.0001
26
+ - Logps/rejected: -278.6774
27
+ - Logps/chosen: -306.4169
28
+ - Logits/rejected: 0.0876
29
+ - Logits/chosen: -0.0097
30
 
31
  ## Model description
32
 
 
59
 
60
  ### Training results
61
 
62
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
63
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
64
+ | 0.0103 | 1.6 | 100 | 0.0108 | 0.0006 | 0.0007 | 0.5005 | -0.0000 | -278.5776 | -306.3062 | 0.0866 | -0.0109 |
65
 
66
 
67
  ### Framework versions
adapter_config.json CHANGED
@@ -20,9 +20,9 @@
20
  "revision": null,
21
  "target_modules": [
22
  "q_proj",
 
23
  "dense",
24
- "k_proj",
25
- "v_proj"
26
  ],
27
  "task_type": "CAUSAL_LM"
28
  }
 
20
  "revision": null,
21
  "target_modules": [
22
  "q_proj",
23
+ "v_proj",
24
  "dense",
25
+ "k_proj"
 
26
  ],
27
  "task_type": "CAUSAL_LM"
28
  }
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:eeddeaf9bf3cfb8fafe6381db179c904f6ea7a811c0473517763dd047c1a364f
3
  size 41977616
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a839cf592811ac151a06351928ea7ec84c69df890e95d78bf41aceaded9f98ef
3
  size 41977616
all_results.json CHANGED
@@ -1,8 +1,21 @@
1
  {
2
- "epoch": 1.33,
3
- "train_loss": 0.003023708879482001,
4
- "train_runtime": 8.1689,
5
- "train_samples": 30567,
6
- "train_samples_per_second": 2.448,
7
- "train_steps_per_second": 0.245
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  }
 
1
  {
2
+ "epoch": 1.98,
3
+ "eval_logits/chosen": -0.009708462283015251,
4
+ "eval_logits/rejected": 0.08755116909742355,
5
+ "eval_logps/chosen": -306.4169006347656,
6
+ "eval_logps/rejected": -278.6773681640625,
7
+ "eval_loss": 0.010828138329088688,
8
+ "eval_rewards/accuracies": 0.49149999022483826,
9
+ "eval_rewards/chosen": -0.00047626858577132225,
10
+ "eval_rewards/margins": -0.00013606167340185493,
11
+ "eval_rewards/rejected": -0.00034020686871372163,
12
+ "eval_runtime": 840.6964,
13
+ "eval_samples": 2000,
14
+ "eval_samples_per_second": 2.379,
15
+ "eval_steps_per_second": 0.595,
16
+ "train_loss": 0.0102488252845022,
17
+ "train_runtime": 3093.4739,
18
+ "train_samples": 61135,
19
+ "train_samples_per_second": 0.647,
20
+ "train_steps_per_second": 0.04
21
  }
eval_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.98,
3
+ "eval_logits/chosen": -0.009708462283015251,
4
+ "eval_logits/rejected": 0.08755116909742355,
5
+ "eval_logps/chosen": -306.4169006347656,
6
+ "eval_logps/rejected": -278.6773681640625,
7
+ "eval_loss": 0.010828138329088688,
8
+ "eval_rewards/accuracies": 0.49149999022483826,
9
+ "eval_rewards/chosen": -0.00047626858577132225,
10
+ "eval_rewards/margins": -0.00013606167340185493,
11
+ "eval_rewards/rejected": -0.00034020686871372163,
12
+ "eval_runtime": 840.6964,
13
+ "eval_samples": 2000,
14
+ "eval_samples_per_second": 2.379,
15
+ "eval_steps_per_second": 0.595
16
+ }
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
- "epoch": 1.33,
3
- "train_loss": 0.003023708879482001,
4
- "train_runtime": 8.1689,
5
- "train_samples": 30567,
6
- "train_samples_per_second": 2.448,
7
- "train_steps_per_second": 0.245
8
  }
 
1
  {
2
+ "epoch": 1.98,
3
+ "train_loss": 0.0102488252845022,
4
+ "train_runtime": 3093.4739,
5
+ "train_samples": 61135,
6
+ "train_samples_per_second": 0.647,
7
+ "train_steps_per_second": 0.04
8
  }
trainer_state.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 1.3333333333333333,
5
  "eval_steps": 100,
6
- "global_step": 2,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
- "epoch": 1.0,
13
- "learning_rate": 5e-06,
14
- "logits/chosen": 0.10384570807218552,
15
- "logits/rejected": 0.28351470828056335,
16
- "logps/chosen": -136.66958618164062,
17
- "logps/rejected": -113.0638427734375,
18
- "loss": 0.0045,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -23,17 +23,201 @@
23
  "step": 1
24
  },
25
  {
26
- "epoch": 1.33,
27
- "step": 2,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  "total_flos": 0.0,
29
- "train_loss": 0.003023708879482001,
30
- "train_runtime": 8.1689,
31
- "train_samples_per_second": 2.448,
32
- "train_steps_per_second": 0.245
33
  }
34
  ],
35
  "logging_steps": 10,
36
- "max_steps": 2,
37
  "num_input_tokens_seen": 0,
38
  "num_train_epochs": 2,
39
  "save_steps": 100,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.984,
5
  "eval_steps": 100,
6
+ "global_step": 124,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
+ "epoch": 0.02,
13
+ "learning_rate": 3.846153846153847e-07,
14
+ "logits/chosen": 0.09327474981546402,
15
+ "logits/rejected": 0.0010143294930458069,
16
+ "logps/chosen": -192.48846435546875,
17
+ "logps/rejected": -100.94705200195312,
18
+ "loss": 0.0102,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
23
  "step": 1
24
  },
25
  {
26
+ "epoch": 0.16,
27
+ "learning_rate": 3.846153846153847e-06,
28
+ "logits/chosen": 0.1767091602087021,
29
+ "logits/rejected": 0.07442302256822586,
30
+ "logps/chosen": -181.4591522216797,
31
+ "logps/rejected": -162.86940002441406,
32
+ "loss": 0.0101,
33
+ "rewards/accuracies": 0.3958333432674408,
34
+ "rewards/chosen": -0.001088320161215961,
35
+ "rewards/margins": 0.0011313353898003697,
36
+ "rewards/rejected": -0.0022196555510163307,
37
+ "step": 10
38
+ },
39
+ {
40
+ "epoch": 0.32,
41
+ "learning_rate": 4.951096619903317e-06,
42
+ "logits/chosen": 0.22487369179725647,
43
+ "logits/rejected": 0.2731098234653473,
44
+ "logps/chosen": -178.03126525878906,
45
+ "logps/rejected": -134.20669555664062,
46
+ "loss": 0.0102,
47
+ "rewards/accuracies": 0.38749998807907104,
48
+ "rewards/chosen": -0.0006419686833396554,
49
+ "rewards/margins": 0.000687784340698272,
50
+ "rewards/rejected": -0.0013297529658302665,
51
+ "step": 20
52
+ },
53
+ {
54
+ "epoch": 0.48,
55
+ "learning_rate": 4.716164218065246e-06,
56
+ "logits/chosen": 0.2530214190483093,
57
+ "logits/rejected": 0.23770615458488464,
58
+ "logps/chosen": -171.042724609375,
59
+ "logps/rejected": -149.58749389648438,
60
+ "loss": 0.0104,
61
+ "rewards/accuracies": 0.375,
62
+ "rewards/chosen": -0.0013910250272601843,
63
+ "rewards/margins": -0.00043516140431165695,
64
+ "rewards/rejected": -0.0009558637393638492,
65
+ "step": 30
66
+ },
67
+ {
68
+ "epoch": 0.64,
69
+ "learning_rate": 4.3048902348863116e-06,
70
+ "logits/chosen": 0.2087957113981247,
71
+ "logits/rejected": 0.2569302022457123,
72
+ "logps/chosen": -174.41390991210938,
73
+ "logps/rejected": -153.81405639648438,
74
+ "loss": 0.0105,
75
+ "rewards/accuracies": 0.41874998807907104,
76
+ "rewards/chosen": 2.615232915559318e-05,
77
+ "rewards/margins": -0.0007305769249796867,
78
+ "rewards/rejected": 0.000756729394197464,
79
+ "step": 40
80
+ },
81
+ {
82
+ "epoch": 0.8,
83
+ "learning_rate": 3.7500000000000005e-06,
84
+ "logits/chosen": 0.22499620914459229,
85
+ "logits/rejected": 0.17940528690814972,
86
+ "logps/chosen": -198.2622833251953,
87
+ "logps/rejected": -162.13284301757812,
88
+ "loss": 0.0105,
89
+ "rewards/accuracies": 0.3687500059604645,
90
+ "rewards/chosen": 0.0005174060934223235,
91
+ "rewards/margins": -0.0008635882404632866,
92
+ "rewards/rejected": 0.0013809942174702883,
93
+ "step": 50
94
+ },
95
+ {
96
+ "epoch": 0.96,
97
+ "learning_rate": 3.0956464785579125e-06,
98
+ "logits/chosen": 0.2462855875492096,
99
+ "logits/rejected": 0.2873149812221527,
100
+ "logps/chosen": -191.82350158691406,
101
+ "logps/rejected": -158.4237823486328,
102
+ "loss": 0.0101,
103
+ "rewards/accuracies": 0.40625,
104
+ "rewards/chosen": -0.0009249815484508872,
105
+ "rewards/margins": 0.0009990528924390674,
106
+ "rewards/rejected": -0.0019240345573052764,
107
+ "step": 60
108
+ },
109
+ {
110
+ "epoch": 1.12,
111
+ "learning_rate": 2.39389699200963e-06,
112
+ "logits/chosen": 0.24232041835784912,
113
+ "logits/rejected": 0.14627447724342346,
114
+ "logps/chosen": -198.8780059814453,
115
+ "logps/rejected": -164.05284118652344,
116
+ "loss": 0.0101,
117
+ "rewards/accuracies": 0.39375001192092896,
118
+ "rewards/chosen": 0.000793910410720855,
119
+ "rewards/margins": 0.001237305928952992,
120
+ "rewards/rejected": -0.0004433956928551197,
121
+ "step": 70
122
+ },
123
+ {
124
+ "epoch": 1.28,
125
+ "learning_rate": 1.700590188571887e-06,
126
+ "logits/chosen": 0.13903482258319855,
127
+ "logits/rejected": 0.18899227678775787,
128
+ "logps/chosen": -202.53575134277344,
129
+ "logps/rejected": -162.55294799804688,
130
+ "loss": 0.0101,
131
+ "rewards/accuracies": 0.39375001192092896,
132
+ "rewards/chosen": 4.065161192556843e-05,
133
+ "rewards/margins": 0.0009858234552666545,
134
+ "rewards/rejected": -0.0009451720979996026,
135
+ "step": 80
136
+ },
137
+ {
138
+ "epoch": 1.44,
139
+ "learning_rate": 1.0708929268538034e-06,
140
+ "logits/chosen": 0.2650024890899658,
141
+ "logits/rejected": 0.17201000452041626,
142
+ "logps/chosen": -175.6930694580078,
143
+ "logps/rejected": -153.9431915283203,
144
+ "loss": 0.0101,
145
+ "rewards/accuracies": 0.4749999940395355,
146
+ "rewards/chosen": -0.00013711625069845468,
147
+ "rewards/margins": 0.0010541726369410753,
148
+ "rewards/rejected": -0.001191288698464632,
149
+ "step": 90
150
+ },
151
+ {
152
+ "epoch": 1.6,
153
+ "learning_rate": 5.549106142039018e-07,
154
+ "logits/chosen": 0.1846569925546646,
155
+ "logits/rejected": 0.1680203378200531,
156
+ "logps/chosen": -177.53756713867188,
157
+ "logps/rejected": -143.4310302734375,
158
+ "loss": 0.0103,
159
+ "rewards/accuracies": 0.3687500059604645,
160
+ "rewards/chosen": -0.0002936289238277823,
161
+ "rewards/margins": 0.00016203436825890094,
162
+ "rewards/rejected": -0.0004556631320156157,
163
+ "step": 100
164
+ },
165
+ {
166
+ "epoch": 1.6,
167
+ "eval_logits/chosen": -0.010850394144654274,
168
+ "eval_logits/rejected": 0.0866025984287262,
169
+ "eval_logps/chosen": -306.3062438964844,
170
+ "eval_logps/rejected": -278.57757568359375,
171
+ "eval_loss": 0.010814609937369823,
172
+ "eval_rewards/accuracies": 0.5005000233650208,
173
+ "eval_rewards/chosen": 0.0006301876856014132,
174
+ "eval_rewards/margins": -2.8262209525564685e-05,
175
+ "eval_rewards/rejected": 0.00065844994969666,
176
+ "eval_runtime": 840.0738,
177
+ "eval_samples_per_second": 2.381,
178
+ "eval_steps_per_second": 0.595,
179
+ "step": 100
180
+ },
181
+ {
182
+ "epoch": 1.76,
183
+ "learning_rate": 1.937002879188285e-07,
184
+ "logits/chosen": 0.2905462980270386,
185
+ "logits/rejected": 0.20439067482948303,
186
+ "logps/chosen": -172.76882934570312,
187
+ "logps/rejected": -152.1476593017578,
188
+ "loss": 0.0102,
189
+ "rewards/accuracies": 0.45625001192092896,
190
+ "rewards/chosen": -0.0004940209328196943,
191
+ "rewards/margins": 0.0009124716743826866,
192
+ "rewards/rejected": -0.0014064926654100418,
193
+ "step": 110
194
+ },
195
+ {
196
+ "epoch": 1.92,
197
+ "learning_rate": 1.6003680950742728e-08,
198
+ "logits/chosen": 0.1698417365550995,
199
+ "logits/rejected": 0.17967729270458221,
200
+ "logps/chosen": -172.40513610839844,
201
+ "logps/rejected": -144.2118377685547,
202
+ "loss": 0.0103,
203
+ "rewards/accuracies": 0.4375,
204
+ "rewards/chosen": 0.0004940934595651925,
205
+ "rewards/margins": 0.0001839537435444072,
206
+ "rewards/rejected": 0.00031013964326120913,
207
+ "step": 120
208
+ },
209
+ {
210
+ "epoch": 1.98,
211
+ "step": 124,
212
  "total_flos": 0.0,
213
+ "train_loss": 0.0102488252845022,
214
+ "train_runtime": 3093.4739,
215
+ "train_samples_per_second": 0.647,
216
+ "train_steps_per_second": 0.04
217
  }
218
  ],
219
  "logging_steps": 10,
220
+ "max_steps": 124,
221
  "num_input_tokens_seen": 0,
222
  "num_train_epochs": 2,
223
  "save_steps": 100,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bbe5ab6340b614d53762370c7e7467a88eeb921f8325cecf428592653477572c
3
  size 5880
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5a1d0e33113b94efdfa8cfe2125b424d7a1d65320be91c739ea5ad638ffd5a0
3
  size 5880