chansung commited on
Commit
b43ab9a
1 Parent(s): d4c9736

Model save

Browse files
README.md ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: gemma
3
+ library_name: peft
4
+ tags:
5
+ - trl
6
+ - sft
7
+ - generated_from_trainer
8
+ base_model: google/gemma-2b
9
+ datasets:
10
+ - generator
11
+ model-index:
12
+ - name: gemma2b-summarize-claude3sonnet-4k
13
+ results: []
14
+ ---
15
+
16
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
17
+ should probably proofread and complete it, then remove this comment. -->
18
+
19
+ # gemma2b-summarize-claude3sonnet-4k
20
+
21
+ This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the generator dataset.
22
+ It achieves the following results on the evaluation set:
23
+ - Loss: 2.5362
24
+
25
+ ## Model description
26
+
27
+ More information needed
28
+
29
+ ## Intended uses & limitations
30
+
31
+ More information needed
32
+
33
+ ## Training and evaluation data
34
+
35
+ More information needed
36
+
37
+ ## Training procedure
38
+
39
+ ### Training hyperparameters
40
+
41
+ The following hyperparameters were used during training:
42
+ - learning_rate: 0.0002
43
+ - train_batch_size: 8
44
+ - eval_batch_size: 8
45
+ - seed: 42
46
+ - distributed_type: multi-GPU
47
+ - num_devices: 3
48
+ - gradient_accumulation_steps: 2
49
+ - total_train_batch_size: 48
50
+ - total_eval_batch_size: 24
51
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
52
+ - lr_scheduler_type: cosine
53
+ - lr_scheduler_warmup_ratio: 0.1
54
+ - num_epochs: 10
55
+
56
+ ### Training results
57
+
58
+ | Training Loss | Epoch | Step | Validation Loss |
59
+ |:-------------:|:-----:|:----:|:---------------:|
60
+ | 2.6184 | 1.0 | 13 | 2.7931 |
61
+ | 1.8166 | 2.0 | 26 | 2.5963 |
62
+ | 1.5365 | 3.0 | 39 | 2.5611 |
63
+ | 1.3191 | 4.0 | 52 | 2.5480 |
64
+ | 1.2353 | 5.0 | 65 | 2.5346 |
65
+ | 1.1927 | 6.0 | 78 | 2.5317 |
66
+ | 1.1527 | 7.0 | 91 | 2.5331 |
67
+ | 1.1355 | 8.0 | 104 | 2.5356 |
68
+ | 1.1336 | 9.0 | 117 | 2.5359 |
69
+ | 1.1358 | 10.0 | 130 | 2.5362 |
70
+
71
+
72
+ ### Framework versions
73
+
74
+ - PEFT 0.10.0
75
+ - Transformers 4.40.0
76
+ - Pytorch 2.2.2+cu121
77
+ - Datasets 2.19.1
78
+ - Tokenizers 0.19.1
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a0c59d34b4f373ce1cac8a3ac05f97653778726d8b8aa88548290bf42097a621
3
  size 19644912
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd66901a8945557eb8e9173c03224849775e276b519bdb50c7d2e5e30bded074
3
  size 19644912
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "total_flos": 7.63585340899328e+16,
4
+ "train_loss": 1.4418765141413763,
5
+ "train_runtime": 452.7766,
6
+ "train_samples": 3968,
7
+ "train_samples_per_second": 13.583,
8
+ "train_steps_per_second": 0.287
9
+ }
runs/Jun05_13-38-00_user-HP-Z8-Fury-G5-Workstation-Desktop-PC/events.out.tfevents.1717562300.user-HP-Z8-Fury-G5-Workstation-Desktop-PC.934194.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d466bf07fa9aa2ab83db473a017a2c79fa2824dacb9ecf321696b424f3462f7e
3
- size 11681
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ba745a05c8ad60509a9aa03f80ce2e4d240320a5267073787d8af1ce452d8c9
3
+ size 14084
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "total_flos": 7.63585340899328e+16,
4
+ "train_loss": 1.4418765141413763,
5
+ "train_runtime": 452.7766,
6
+ "train_samples": 3968,
7
+ "train_samples_per_second": 13.583,
8
+ "train_steps_per_second": 0.287
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,299 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 10.0,
5
+ "eval_steps": 500,
6
+ "global_step": 130,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.07692307692307693,
13
+ "grad_norm": 2.703125,
14
+ "learning_rate": 1.5384615384615387e-05,
15
+ "loss": 2.7432,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.38461538461538464,
20
+ "grad_norm": 1.46875,
21
+ "learning_rate": 7.692307692307693e-05,
22
+ "loss": 2.7967,
23
+ "step": 5
24
+ },
25
+ {
26
+ "epoch": 0.7692307692307693,
27
+ "grad_norm": 1.5078125,
28
+ "learning_rate": 0.00015384615384615385,
29
+ "loss": 2.6184,
30
+ "step": 10
31
+ },
32
+ {
33
+ "epoch": 1.0,
34
+ "eval_loss": 2.7931067943573,
35
+ "eval_runtime": 0.5382,
36
+ "eval_samples_per_second": 18.579,
37
+ "eval_steps_per_second": 1.858,
38
+ "step": 13
39
+ },
40
+ {
41
+ "epoch": 1.1538461538461537,
42
+ "grad_norm": 1.6640625,
43
+ "learning_rate": 0.00019985583705641418,
44
+ "loss": 2.2437,
45
+ "step": 15
46
+ },
47
+ {
48
+ "epoch": 1.5384615384615383,
49
+ "grad_norm": 2.046875,
50
+ "learning_rate": 0.00019823877374156647,
51
+ "loss": 1.9974,
52
+ "step": 20
53
+ },
54
+ {
55
+ "epoch": 1.9230769230769231,
56
+ "grad_norm": 1.2734375,
57
+ "learning_rate": 0.00019485364419471454,
58
+ "loss": 1.8166,
59
+ "step": 25
60
+ },
61
+ {
62
+ "epoch": 2.0,
63
+ "eval_loss": 2.5962724685668945,
64
+ "eval_runtime": 0.5549,
65
+ "eval_samples_per_second": 18.021,
66
+ "eval_steps_per_second": 1.802,
67
+ "step": 26
68
+ },
69
+ {
70
+ "epoch": 2.3076923076923075,
71
+ "grad_norm": 0.65625,
72
+ "learning_rate": 0.0001897613727639014,
73
+ "loss": 1.6601,
74
+ "step": 30
75
+ },
76
+ {
77
+ "epoch": 2.6923076923076925,
78
+ "grad_norm": 0.61328125,
79
+ "learning_rate": 0.00018305360832480117,
80
+ "loss": 1.5365,
81
+ "step": 35
82
+ },
83
+ {
84
+ "epoch": 3.0,
85
+ "eval_loss": 2.561077356338501,
86
+ "eval_runtime": 0.5395,
87
+ "eval_samples_per_second": 18.537,
88
+ "eval_steps_per_second": 1.854,
89
+ "step": 39
90
+ },
91
+ {
92
+ "epoch": 3.076923076923077,
93
+ "grad_norm": 0.326171875,
94
+ "learning_rate": 0.00017485107481711012,
95
+ "loss": 1.4277,
96
+ "step": 40
97
+ },
98
+ {
99
+ "epoch": 3.4615384615384617,
100
+ "grad_norm": 0.34765625,
101
+ "learning_rate": 0.0001653013984983585,
102
+ "loss": 1.3807,
103
+ "step": 45
104
+ },
105
+ {
106
+ "epoch": 3.8461538461538463,
107
+ "grad_norm": 0.337890625,
108
+ "learning_rate": 0.00015457645101945046,
109
+ "loss": 1.3191,
110
+ "step": 50
111
+ },
112
+ {
113
+ "epoch": 4.0,
114
+ "eval_loss": 2.5480408668518066,
115
+ "eval_runtime": 0.5396,
116
+ "eval_samples_per_second": 18.531,
117
+ "eval_steps_per_second": 1.853,
118
+ "step": 52
119
+ },
120
+ {
121
+ "epoch": 4.230769230769231,
122
+ "grad_norm": 0.32421875,
123
+ "learning_rate": 0.00014286925614030542,
124
+ "loss": 1.2714,
125
+ "step": 55
126
+ },
127
+ {
128
+ "epoch": 4.615384615384615,
129
+ "grad_norm": 0.328125,
130
+ "learning_rate": 0.0001303905157574247,
131
+ "loss": 1.2521,
132
+ "step": 60
133
+ },
134
+ {
135
+ "epoch": 5.0,
136
+ "grad_norm": 0.328125,
137
+ "learning_rate": 0.00011736481776669306,
138
+ "loss": 1.2353,
139
+ "step": 65
140
+ },
141
+ {
142
+ "epoch": 5.0,
143
+ "eval_loss": 2.534567356109619,
144
+ "eval_runtime": 0.5331,
145
+ "eval_samples_per_second": 18.76,
146
+ "eval_steps_per_second": 1.876,
147
+ "step": 65
148
+ },
149
+ {
150
+ "epoch": 5.384615384615385,
151
+ "grad_norm": 0.29296875,
152
+ "learning_rate": 0.00010402659401094152,
153
+ "loss": 1.1973,
154
+ "step": 70
155
+ },
156
+ {
157
+ "epoch": 5.769230769230769,
158
+ "grad_norm": 0.349609375,
159
+ "learning_rate": 9.061590105968208e-05,
160
+ "loss": 1.1927,
161
+ "step": 75
162
+ },
163
+ {
164
+ "epoch": 6.0,
165
+ "eval_loss": 2.5317368507385254,
166
+ "eval_runtime": 0.5351,
167
+ "eval_samples_per_second": 18.69,
168
+ "eval_steps_per_second": 1.869,
169
+ "step": 78
170
+ },
171
+ {
172
+ "epoch": 6.153846153846154,
173
+ "grad_norm": 0.265625,
174
+ "learning_rate": 7.73740997570278e-05,
175
+ "loss": 1.1796,
176
+ "step": 80
177
+ },
178
+ {
179
+ "epoch": 6.538461538461538,
180
+ "grad_norm": 0.326171875,
181
+ "learning_rate": 6.453951129574644e-05,
182
+ "loss": 1.1608,
183
+ "step": 85
184
+ },
185
+ {
186
+ "epoch": 6.923076923076923,
187
+ "grad_norm": 0.33203125,
188
+ "learning_rate": 5.234312799786921e-05,
189
+ "loss": 1.1527,
190
+ "step": 90
191
+ },
192
+ {
193
+ "epoch": 7.0,
194
+ "eval_loss": 2.5331029891967773,
195
+ "eval_runtime": 0.536,
196
+ "eval_samples_per_second": 18.658,
197
+ "eval_steps_per_second": 1.866,
198
+ "step": 91
199
+ },
200
+ {
201
+ "epoch": 7.3076923076923075,
202
+ "grad_norm": 0.294921875,
203
+ "learning_rate": 4.100445599768774e-05,
204
+ "loss": 1.1448,
205
+ "step": 95
206
+ },
207
+ {
208
+ "epoch": 7.6923076923076925,
209
+ "grad_norm": 0.28515625,
210
+ "learning_rate": 3.072756464904006e-05,
211
+ "loss": 1.1355,
212
+ "step": 100
213
+ },
214
+ {
215
+ "epoch": 8.0,
216
+ "eval_loss": 2.535613536834717,
217
+ "eval_runtime": 0.5382,
218
+ "eval_samples_per_second": 18.582,
219
+ "eval_steps_per_second": 1.858,
220
+ "step": 104
221
+ },
222
+ {
223
+ "epoch": 8.076923076923077,
224
+ "grad_norm": 0.296875,
225
+ "learning_rate": 2.1697413758237784e-05,
226
+ "loss": 1.1328,
227
+ "step": 105
228
+ },
229
+ {
230
+ "epoch": 8.461538461538462,
231
+ "grad_norm": 0.3125,
232
+ "learning_rate": 1.4076524743778319e-05,
233
+ "loss": 1.1259,
234
+ "step": 110
235
+ },
236
+ {
237
+ "epoch": 8.846153846153847,
238
+ "grad_norm": 0.2734375,
239
+ "learning_rate": 8.002055634117578e-06,
240
+ "loss": 1.1336,
241
+ "step": 115
242
+ },
243
+ {
244
+ "epoch": 9.0,
245
+ "eval_loss": 2.535912275314331,
246
+ "eval_runtime": 0.5424,
247
+ "eval_samples_per_second": 18.437,
248
+ "eval_steps_per_second": 1.844,
249
+ "step": 117
250
+ },
251
+ {
252
+ "epoch": 9.23076923076923,
253
+ "grad_norm": 0.333984375,
254
+ "learning_rate": 3.5833325466437694e-06,
255
+ "loss": 1.139,
256
+ "step": 120
257
+ },
258
+ {
259
+ "epoch": 9.615384615384615,
260
+ "grad_norm": 0.27734375,
261
+ "learning_rate": 8.998820754091531e-07,
262
+ "loss": 1.1132,
263
+ "step": 125
264
+ },
265
+ {
266
+ "epoch": 10.0,
267
+ "grad_norm": 0.302734375,
268
+ "learning_rate": 0.0,
269
+ "loss": 1.1358,
270
+ "step": 130
271
+ },
272
+ {
273
+ "epoch": 10.0,
274
+ "eval_loss": 2.536184310913086,
275
+ "eval_runtime": 0.5396,
276
+ "eval_samples_per_second": 18.533,
277
+ "eval_steps_per_second": 1.853,
278
+ "step": 130
279
+ },
280
+ {
281
+ "epoch": 10.0,
282
+ "step": 130,
283
+ "total_flos": 7.63585340899328e+16,
284
+ "train_loss": 1.4418765141413763,
285
+ "train_runtime": 452.7766,
286
+ "train_samples_per_second": 13.583,
287
+ "train_steps_per_second": 0.287
288
+ }
289
+ ],
290
+ "logging_steps": 5,
291
+ "max_steps": 130,
292
+ "num_input_tokens_seen": 0,
293
+ "num_train_epochs": 10,
294
+ "save_steps": 100,
295
+ "total_flos": 7.63585340899328e+16,
296
+ "train_batch_size": 8,
297
+ "trial_name": null,
298
+ "trial_params": null
299
+ }