Hugging Face's logo

dh-unibe
/
luther-xl

Text Generation
Transformers
PyTorch
Safetensors
gpt2
generated_from_trainer
Inference Endpoints
text-generation-inference
Model card Files Community
1
Jonas Widmer commited on
Commit
a404f0a
•
1 Parent(s): b638227

initial commit

Browse files
Files changed (12) hide show
  1. README.md +84 -0
  2. all_results.json +15 -0
  3. config.json +35 -0
  4. eval_results.json +10 -0
  5. merges.txt +0 -0
  6. special_tokens_map.json +5 -0
  7. tokenizer.json +0 -0
  8. tokenizer_config.json +10 -0
  9. train_results.json +8 -0
  10. trainer_state.json +289 -0
  11. training_args.bin +3 -0
  12. vocab.json +0 -0
README.md CHANGED
@@ -1,3 +1,87 @@
1
  ---
2
  license: mit
 
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  license: mit
3
+ tags:
4
+ - generated_from_trainer
5
+ metrics:
6
+ - accuracy
7
+ model-index:
8
+ - name: 22_12_13_luther_blocks_xl_fp16_5ep
9
+ results: []
10
  ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ # 22_12_13_luther_blocks_xl_fp16_5ep
16
+
17
+ This model is a fine-tuned version of [malteos/gpt2-xl-wechsel-german](https://huggingface.co/malteos/gpt2-xl-wechsel-german) on an unknown dataset.
18
+ It achieves the following results on the evaluation set:
19
+ - Loss: 2.8833
20
+ - Accuracy: 0.4196
21
+
22
+ ## Model description
23
+
24
+ More information needed
25
+
26
+ ## Intended uses & limitations
27
+
28
+ More information needed
29
+
30
+ ## Training and evaluation data
31
+
32
+ More information needed
33
+
34
+ ## Training procedure
35
+
36
+ ### Training hyperparameters
37
+
38
+ The following hyperparameters were used during training:
39
+ - learning_rate: 5e-05
40
+ - train_batch_size: 8
41
+ - eval_batch_size: 8
42
+ - seed: 42
43
+ - gradient_accumulation_steps: 2
44
+ - total_train_batch_size: 16
45
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
46
+ - lr_scheduler_type: linear
47
+ - num_epochs: 5.0
48
+ - mixed_precision_training: Native AMP
49
+
50
+ ### Training results
51
+
52
+ | Training Loss | Epoch | Step | Validation Loss | Accuracy |
53
+ |:-------------:|:-----:|:----:|:---------------:|:--------:|
54
+ | No log | 0.19 | 50 | 3.0276 | 0.3997 |
55
+ | No log | 0.38 | 100 | 2.9185 | 0.4143 |
56
+ | No log | 0.58 | 150 | 2.8846 | 0.4189 |
57
+ | No log | 0.77 | 200 | 2.8833 | 0.4196 |
58
+ | 3.0309 | 0.96 | 250 | 2.8833 | 0.4196 |
59
+ | 3.0309 | 1.15 | 300 | 2.8833 | 0.4196 |
60
+ | 3.0309 | 1.35 | 350 | 2.8833 | 0.4196 |
61
+ | 3.0309 | 1.54 | 400 | 2.8833 | 0.4196 |
62
+ | 3.0309 | 1.73 | 450 | 2.8833 | 0.4196 |
63
+ | 2.8083 | 1.92 | 500 | 2.8833 | 0.4196 |
64
+ | 2.8083 | 2.12 | 550 | 2.8833 | 0.4196 |
65
+ | 2.8083 | 2.31 | 600 | 2.8833 | 0.4196 |
66
+ | 2.8083 | 2.5 | 650 | 2.8833 | 0.4196 |
67
+ | 2.8083 | 2.69 | 700 | 2.8833 | 0.4196 |
68
+ | 2.7992 | 2.88 | 750 | 2.8833 | 0.4196 |
69
+ | 2.7992 | 3.08 | 800 | 2.8833 | 0.4196 |
70
+ | 2.7992 | 3.27 | 850 | 2.8833 | 0.4196 |
71
+ | 2.7992 | 3.46 | 900 | 2.8833 | 0.4196 |
72
+ | 2.7992 | 3.65 | 950 | 2.8833 | 0.4196 |
73
+ | 2.7935 | 3.85 | 1000 | 2.8833 | 0.4196 |
74
+ | 2.7935 | 4.04 | 1050 | 2.8833 | 0.4196 |
75
+ | 2.7935 | 4.23 | 1100 | 2.8833 | 0.4196 |
76
+ | 2.7935 | 4.42 | 1150 | 2.8833 | 0.4196 |
77
+ | 2.7935 | 4.62 | 1200 | 2.8833 | 0.4196 |
78
+ | 2.8129 | 4.81 | 1250 | 2.8833 | 0.4196 |
79
+ | 2.8129 | 5.0 | 1300 | 2.8833 | 0.4196 |
80
+
81
+
82
+ ### Framework versions
83
+
84
+ - Transformers 4.26.0.dev0
85
+ - Pytorch 1.13.0
86
+ - Datasets 2.7.1
87
+ - Tokenizers 0.12.1
all_results.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 5.0,
3
+ "eval_accuracy": 0.4195681151692882,
4
+ "eval_loss": 2.8833088874816895,
5
+ "eval_runtime": 5.8323,
6
+ "eval_samples": 55,
7
+ "eval_samples_per_second": 9.43,
8
+ "eval_steps_per_second": 1.2,
9
+ "perplexity": 17.87331623467394,
10
+ "train_loss": 2.8456746497521035,
11
+ "train_runtime": 1533.618,
12
+ "train_samples": 520,
13
+ "train_samples_per_second": 1.695,
14
+ "train_steps_per_second": 0.848
15
+ }
config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "malteos/gpt2-xl-wechsel-german",
3
+ "activation_function": "gelu",
4
+ "architectures": [
5
+ "GPT2LMHeadModel"
6
+ ],
7
+ "attn_pdrop": 0.1,
8
+ "bos_token_id": 50256,
9
+ "embd_pdrop": 0.1,
10
+ "eos_token_id": 50256,
11
+ "gradient_checkpointing": false,
12
+ "initializer_range": 0.02,
13
+ "layer_norm_epsilon": 1e-05,
14
+ "model_type": "gpt2",
15
+ "n_ctx": 1024,
16
+ "n_embd": 1600,
17
+ "n_head": 25,
18
+ "n_inner": 6400,
19
+ "n_layer": 48,
20
+ "n_positions": 1024,
21
+ "reorder_and_upcast_attn": false,
22
+ "resid_pdrop": 0.1,
23
+ "scale_attn_by_inverse_layer_idx": false,
24
+ "scale_attn_weights": true,
25
+ "summary_activation": null,
26
+ "summary_first_dropout": 0.1,
27
+ "summary_proj_to_labels": true,
28
+ "summary_type": "cls_index",
29
+ "summary_use_proj": true,
30
+ "tokenizer_class": "GPT2TokenizerFast",
31
+ "torch_dtype": "float32",
32
+ "transformers_version": "4.26.0.dev0",
33
+ "use_cache": true,
34
+ "vocab_size": 50304
35
+ }
eval_results.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 5.0,
3
+ "eval_accuracy": 0.4195681151692882,
4
+ "eval_loss": 2.8833088874816895,
5
+ "eval_runtime": 5.8323,
6
+ "eval_samples": 55,
7
+ "eval_samples_per_second": 9.43,
8
+ "eval_steps_per_second": 1.2,
9
+ "perplexity": 17.87331623467394
10
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "unk_token": "<|endoftext|>"
5
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<|endoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "model_max_length": 1000000000000000019884624838656,
6
+ "name_or_path": "malteos/gpt2-xl-wechsel-german",
7
+ "special_tokens_map_file": null,
8
+ "tokenizer_class": "GPT2Tokenizer",
9
+ "unk_token": "<|endoftext|>"
10
+ }
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 5.0,
3
+ "train_loss": 2.8456746497521035,
4
+ "train_runtime": 1533.618,
5
+ "train_samples": 520,
6
+ "train_samples_per_second": 1.695,
7
+ "train_steps_per_second": 0.848
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 2.8833088874816895,
3
+ "best_model_checkpoint": "./models/22_12_13_luther_blocks_xl_fp16_5ep/checkpoint-250",
4
+ "epoch": 5.0,
5
+ "global_step": 1300,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.19,
12
+ "eval_accuracy": 0.39973340442548655,
13
+ "eval_loss": 3.027606964111328,
14
+ "eval_runtime": 5.8811,
15
+ "eval_samples_per_second": 9.352,
16
+ "eval_steps_per_second": 1.19,
17
+ "step": 50
18
+ },
19
+ {
20
+ "epoch": 0.38,
21
+ "eval_accuracy": 0.4143428419088243,
22
+ "eval_loss": 2.9184916019439697,
23
+ "eval_runtime": 5.8392,
24
+ "eval_samples_per_second": 9.419,
25
+ "eval_steps_per_second": 1.199,
26
+ "step": 100
27
+ },
28
+ {
29
+ "epoch": 0.58,
30
+ "eval_accuracy": 0.4188927397138541,
31
+ "eval_loss": 2.8845763206481934,
32
+ "eval_runtime": 5.8499,
33
+ "eval_samples_per_second": 9.402,
34
+ "eval_steps_per_second": 1.197,
35
+ "step": 150
36
+ },
37
+ {
38
+ "epoch": 0.77,
39
+ "eval_accuracy": 0.4195681151692882,
40
+ "eval_loss": 2.8833088874816895,
41
+ "eval_runtime": 5.8518,
42
+ "eval_samples_per_second": 9.399,
43
+ "eval_steps_per_second": 1.196,
44
+ "step": 200
45
+ },
46
+ {
47
+ "epoch": 0.96,
48
+ "learning_rate": 0.0,
49
+ "loss": 3.0309,
50
+ "step": 250
51
+ },
52
+ {
53
+ "epoch": 0.96,
54
+ "eval_accuracy": 0.4195681151692882,
55
+ "eval_loss": 2.8833088874816895,
56
+ "eval_runtime": 5.8496,
57
+ "eval_samples_per_second": 9.402,
58
+ "eval_steps_per_second": 1.197,
59
+ "step": 250
60
+ },
61
+ {
62
+ "epoch": 1.15,
63
+ "eval_accuracy": 0.4195681151692882,
64
+ "eval_loss": 2.8833088874816895,
65
+ "eval_runtime": 5.8476,
66
+ "eval_samples_per_second": 9.405,
67
+ "eval_steps_per_second": 1.197,
68
+ "step": 300
69
+ },
70
+ {
71
+ "epoch": 1.35,
72
+ "eval_accuracy": 0.4195681151692882,
73
+ "eval_loss": 2.8833088874816895,
74
+ "eval_runtime": 5.8513,
75
+ "eval_samples_per_second": 9.4,
76
+ "eval_steps_per_second": 1.196,
77
+ "step": 350
78
+ },
79
+ {
80
+ "epoch": 1.54,
81
+ "eval_accuracy": 0.4195681151692882,
82
+ "eval_loss": 2.8833088874816895,
83
+ "eval_runtime": 5.8506,
84
+ "eval_samples_per_second": 9.401,
85
+ "eval_steps_per_second": 1.196,
86
+ "step": 400
87
+ },
88
+ {
89
+ "epoch": 1.73,
90
+ "eval_accuracy": 0.4195681151692882,
91
+ "eval_loss": 2.8833088874816895,
92
+ "eval_runtime": 5.8498,
93
+ "eval_samples_per_second": 9.402,
94
+ "eval_steps_per_second": 1.197,
95
+ "step": 450
96
+ },
97
+ {
98
+ "epoch": 1.92,
99
+ "learning_rate": 0.0,
100
+ "loss": 2.8083,
101
+ "step": 500
102
+ },
103
+ {
104
+ "epoch": 1.92,
105
+ "eval_accuracy": 0.4195681151692882,
106
+ "eval_loss": 2.8833088874816895,
107
+ "eval_runtime": 5.8488,
108
+ "eval_samples_per_second": 9.404,
109
+ "eval_steps_per_second": 1.197,
110
+ "step": 500
111
+ },
112
+ {
113
+ "epoch": 2.12,
114
+ "eval_accuracy": 0.4195681151692882,
115
+ "eval_loss": 2.8833088874816895,
116
+ "eval_runtime": 5.8482,
117
+ "eval_samples_per_second": 9.405,
118
+ "eval_steps_per_second": 1.197,
119
+ "step": 550
120
+ },
121
+ {
122
+ "epoch": 2.31,
123
+ "eval_accuracy": 0.4195681151692882,
124
+ "eval_loss": 2.8833088874816895,
125
+ "eval_runtime": 5.8537,
126
+ "eval_samples_per_second": 9.396,
127
+ "eval_steps_per_second": 1.196,
128
+ "step": 600
129
+ },
130
+ {
131
+ "epoch": 2.5,
132
+ "eval_accuracy": 0.4195681151692882,
133
+ "eval_loss": 2.8833088874816895,
134
+ "eval_runtime": 5.8517,
135
+ "eval_samples_per_second": 9.399,
136
+ "eval_steps_per_second": 1.196,
137
+ "step": 650
138
+ },
139
+ {
140
+ "epoch": 2.69,
141
+ "eval_accuracy": 0.4195681151692882,
142
+ "eval_loss": 2.8833088874816895,
143
+ "eval_runtime": 5.8492,
144
+ "eval_samples_per_second": 9.403,
145
+ "eval_steps_per_second": 1.197,
146
+ "step": 700
147
+ },
148
+ {
149
+ "epoch": 2.88,
150
+ "learning_rate": 0.0,
151
+ "loss": 2.7992,
152
+ "step": 750
153
+ },
154
+ {
155
+ "epoch": 2.88,
156
+ "eval_accuracy": 0.4195681151692882,
157
+ "eval_loss": 2.8833088874816895,
158
+ "eval_runtime": 5.8497,
159
+ "eval_samples_per_second": 9.402,
160
+ "eval_steps_per_second": 1.197,
161
+ "step": 750
162
+ },
163
+ {
164
+ "epoch": 3.08,
165
+ "eval_accuracy": 0.4195681151692882,
166
+ "eval_loss": 2.8833088874816895,
167
+ "eval_runtime": 5.8451,
168
+ "eval_samples_per_second": 9.41,
169
+ "eval_steps_per_second": 1.198,
170
+ "step": 800
171
+ },
172
+ {
173
+ "epoch": 3.27,
174
+ "eval_accuracy": 0.4195681151692882,
175
+ "eval_loss": 2.8833088874816895,
176
+ "eval_runtime": 5.8463,
177
+ "eval_samples_per_second": 9.408,
178
+ "eval_steps_per_second": 1.197,
179
+ "step": 850
180
+ },
181
+ {
182
+ "epoch": 3.46,
183
+ "eval_accuracy": 0.4195681151692882,
184
+ "eval_loss": 2.8833088874816895,
185
+ "eval_runtime": 5.8445,
186
+ "eval_samples_per_second": 9.411,
187
+ "eval_steps_per_second": 1.198,
188
+ "step": 900
189
+ },
190
+ {
191
+ "epoch": 3.65,
192
+ "eval_accuracy": 0.4195681151692882,
193
+ "eval_loss": 2.8833088874816895,
194
+ "eval_runtime": 5.8463,
195
+ "eval_samples_per_second": 9.408,
196
+ "eval_steps_per_second": 1.197,
197
+ "step": 950
198
+ },
199
+ {
200
+ "epoch": 3.85,
201
+ "learning_rate": 0.0,
202
+ "loss": 2.7935,
203
+ "step": 1000
204
+ },
205
+ {
206
+ "epoch": 3.85,
207
+ "eval_accuracy": 0.4195681151692882,
208
+ "eval_loss": 2.8833088874816895,
209
+ "eval_runtime": 5.8428,
210
+ "eval_samples_per_second": 9.413,
211
+ "eval_steps_per_second": 1.198,
212
+ "step": 1000
213
+ },
214
+ {
215
+ "epoch": 4.04,
216
+ "eval_accuracy": 0.4195681151692882,
217
+ "eval_loss": 2.8833088874816895,
218
+ "eval_runtime": 5.848,
219
+ "eval_samples_per_second": 9.405,
220
+ "eval_steps_per_second": 1.197,
221
+ "step": 1050
222
+ },
223
+ {
224
+ "epoch": 4.23,
225
+ "eval_accuracy": 0.4195681151692882,
226
+ "eval_loss": 2.8833088874816895,
227
+ "eval_runtime": 5.8489,
228
+ "eval_samples_per_second": 9.403,
229
+ "eval_steps_per_second": 1.197,
230
+ "step": 1100
231
+ },
232
+ {
233
+ "epoch": 4.42,
234
+ "eval_accuracy": 0.4195681151692882,
235
+ "eval_loss": 2.8833088874816895,
236
+ "eval_runtime": 5.8501,
237
+ "eval_samples_per_second": 9.402,
238
+ "eval_steps_per_second": 1.197,
239
+ "step": 1150
240
+ },
241
+ {
242
+ "epoch": 4.62,
243
+ "eval_accuracy": 0.4195681151692882,
244
+ "eval_loss": 2.8833088874816895,
245
+ "eval_runtime": 5.8484,
246
+ "eval_samples_per_second": 9.404,
247
+ "eval_steps_per_second": 1.197,
248
+ "step": 1200
249
+ },
250
+ {
251
+ "epoch": 4.81,
252
+ "learning_rate": 0.0,
253
+ "loss": 2.8129,
254
+ "step": 1250
255
+ },
256
+ {
257
+ "epoch": 4.81,
258
+ "eval_accuracy": 0.4195681151692882,
259
+ "eval_loss": 2.8833088874816895,
260
+ "eval_runtime": 5.8457,
261
+ "eval_samples_per_second": 9.409,
262
+ "eval_steps_per_second": 1.197,
263
+ "step": 1250
264
+ },
265
+ {
266
+ "epoch": 5.0,
267
+ "eval_accuracy": 0.4195681151692882,
268
+ "eval_loss": 2.8833088874816895,
269
+ "eval_runtime": 5.8475,
270
+ "eval_samples_per_second": 9.406,
271
+ "eval_steps_per_second": 1.197,
272
+ "step": 1300
273
+ },
274
+ {
275
+ "epoch": 5.0,
276
+ "step": 1300,
277
+ "total_flos": 2.36074746249216e+16,
278
+ "train_loss": 2.8456746497521035,
279
+ "train_runtime": 1533.618,
280
+ "train_samples_per_second": 1.695,
281
+ "train_steps_per_second": 0.848
282
+ }
283
+ ],
284
+ "max_steps": 1300,
285
+ "num_train_epochs": 5,
286
+ "total_flos": 2.36074746249216e+16,
287
+ "trial_name": null,
288
+ "trial_params": null
289
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8627d10fe54dee7f02c449cd49564549cb4e83e7c1465b856951d8812754b1a8
3
+ size 3515
vocab.json ADDED
The diff for this file is too large to render. See raw diff