Hugging Face's logo

DrNicefellow
/
GPT-2-Large-51k-steps

Text Generation
Transformers
PyTorch
gpt2
text-generation-inference
Inference Endpoints
Model card Files Community
DrNicefellow commited on
Commit
2a0ea00
·
verified ·
1 Parent(s): 370f9ab

Upload 13 files

Browse files
Files changed (13) hide show
  1. config.json +38 -0
  2. generation_config.json +6 -0
  3. merges.txt +0 -0
  4. optimizer.pt +3 -0
  5. pytorch_model.bin +3 -0
  6. rng_state.pth +3 -0
  7. scheduler.pt +3 -0
  8. special_tokens_map.json +5 -0
  9. tokenizer.json +0 -0
  10. tokenizer_config.json +19 -0
  11. trainer_state.json +261 -0
  12. training_args.bin +3 -0
  13. vocab.json +0 -0
config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 50256,
10
+ "initializer_range": 0.02,
11
+ "layer_norm_epsilon": 1e-05,
12
+ "model_type": "gpt2",
13
+ "n_ctx": 512,
14
+ "n_embd": 1280,
15
+ "n_head": 20,
16
+ "n_inner": null,
17
+ "n_layer": 36,
18
+ "n_positions": 512,
19
+ "reorder_and_upcast_attn": false,
20
+ "resid_pdrop": 0.1,
21
+ "scale_attn_by_inverse_layer_idx": false,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.38.0.dev0",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.38.0.dev0"
6
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0a3e23acf29cef0701d54d3cd44f40537af61930866da0155bc63b6b9f34d4b
3
+ size 6187371110
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b2c3ddafef6b02429344cf78d40b5f49a1e919e92099890660b1706ae4f9a41
3
+ size 3093641690
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:637c70c0ada711e0f2bdec28b2036ce3758f5b80717c1efb8295918572f259fb
3
+ size 14244
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d440d11c4154736725fed90366fa187c8cc6e3ec9f75395a9637f2917fc7df56
3
+ size 1064
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "unk_token": "<|endoftext|>"
5
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ }
12
+ },
13
+ "bos_token": "<|endoftext|>",
14
+ "clean_up_tokenization_spaces": true,
15
+ "eos_token": "<|endoftext|>",
16
+ "model_max_length": 1024,
17
+ "tokenizer_class": "GPT2Tokenizer",
18
+ "unk_token": "<|endoftext|>"
19
+ }
trainer_state.json ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.018587602440923952,
5
+ "eval_steps": 100000000,
6
+ "global_step": 20000,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "learning_rate": 5e-05,
14
+ "loss": 7.7296,
15
+ "step": 500
16
+ },
17
+ {
18
+ "epoch": 0.0,
19
+ "learning_rate": 4.9999535305620204e-05,
20
+ "loss": 6.5786,
21
+ "step": 1000
22
+ },
23
+ {
24
+ "epoch": 0.0,
25
+ "learning_rate": 4.99990706112404e-05,
26
+ "loss": 6.2317,
27
+ "step": 1500
28
+ },
29
+ {
30
+ "epoch": 0.0,
31
+ "learning_rate": 4.99986059168606e-05,
32
+ "loss": 6.0053,
33
+ "step": 2000
34
+ },
35
+ {
36
+ "epoch": 0.0,
37
+ "learning_rate": 4.99981412224808e-05,
38
+ "loss": 5.8615,
39
+ "step": 2500
40
+ },
41
+ {
42
+ "epoch": 0.0,
43
+ "learning_rate": 4.9997676528101e-05,
44
+ "loss": 5.6794,
45
+ "step": 3000
46
+ },
47
+ {
48
+ "epoch": 0.0,
49
+ "learning_rate": 4.99972118337212e-05,
50
+ "loss": 5.5481,
51
+ "step": 3500
52
+ },
53
+ {
54
+ "epoch": 0.0,
55
+ "learning_rate": 4.9996747139341396e-05,
56
+ "loss": 5.4375,
57
+ "step": 4000
58
+ },
59
+ {
60
+ "epoch": 0.0,
61
+ "learning_rate": 4.99962824449616e-05,
62
+ "loss": 5.3391,
63
+ "step": 4500
64
+ },
65
+ {
66
+ "epoch": 0.0,
67
+ "learning_rate": 4.99958177505818e-05,
68
+ "loss": 5.2598,
69
+ "step": 5000
70
+ },
71
+ {
72
+ "epoch": 0.01,
73
+ "learning_rate": 4.9995353056202e-05,
74
+ "loss": 5.17,
75
+ "step": 5500
76
+ },
77
+ {
78
+ "epoch": 0.01,
79
+ "learning_rate": 4.9994888361822204e-05,
80
+ "loss": 5.0886,
81
+ "step": 6000
82
+ },
83
+ {
84
+ "epoch": 0.01,
85
+ "learning_rate": 4.99944236674424e-05,
86
+ "loss": 5.0128,
87
+ "step": 6500
88
+ },
89
+ {
90
+ "epoch": 0.01,
91
+ "learning_rate": 4.99939589730626e-05,
92
+ "loss": 4.9504,
93
+ "step": 7000
94
+ },
95
+ {
96
+ "epoch": 0.01,
97
+ "learning_rate": 4.99934942786828e-05,
98
+ "loss": 4.8755,
99
+ "step": 7500
100
+ },
101
+ {
102
+ "epoch": 0.01,
103
+ "learning_rate": 4.9993029584303e-05,
104
+ "loss": 4.8402,
105
+ "step": 8000
106
+ },
107
+ {
108
+ "epoch": 0.01,
109
+ "learning_rate": 4.99925648899232e-05,
110
+ "loss": 4.7674,
111
+ "step": 8500
112
+ },
113
+ {
114
+ "epoch": 0.01,
115
+ "learning_rate": 4.9992100195543396e-05,
116
+ "loss": 4.7254,
117
+ "step": 9000
118
+ },
119
+ {
120
+ "epoch": 0.01,
121
+ "learning_rate": 4.99916355011636e-05,
122
+ "loss": 4.6749,
123
+ "step": 9500
124
+ },
125
+ {
126
+ "epoch": 0.01,
127
+ "learning_rate": 4.99911708067838e-05,
128
+ "loss": 4.6135,
129
+ "step": 10000
130
+ },
131
+ {
132
+ "epoch": 0.01,
133
+ "learning_rate": 4.9990706112403995e-05,
134
+ "loss": 4.5239,
135
+ "step": 10500
136
+ },
137
+ {
138
+ "epoch": 0.01,
139
+ "learning_rate": 4.99902414180242e-05,
140
+ "loss": 4.4729,
141
+ "step": 11000
142
+ },
143
+ {
144
+ "epoch": 0.01,
145
+ "learning_rate": 4.998977672364439e-05,
146
+ "loss": 4.4434,
147
+ "step": 11500
148
+ },
149
+ {
150
+ "epoch": 0.01,
151
+ "learning_rate": 4.9989312029264594e-05,
152
+ "loss": 4.3587,
153
+ "step": 12000
154
+ },
155
+ {
156
+ "epoch": 0.01,
157
+ "learning_rate": 4.9988847334884796e-05,
158
+ "loss": 4.3126,
159
+ "step": 12500
160
+ },
161
+ {
162
+ "epoch": 0.01,
163
+ "learning_rate": 4.998838264050499e-05,
164
+ "loss": 4.2658,
165
+ "step": 13000
166
+ },
167
+ {
168
+ "epoch": 0.01,
169
+ "learning_rate": 4.998791794612519e-05,
170
+ "loss": 4.2412,
171
+ "step": 13500
172
+ },
173
+ {
174
+ "epoch": 0.01,
175
+ "learning_rate": 4.9987453251745395e-05,
176
+ "loss": 4.2035,
177
+ "step": 14000
178
+ },
179
+ {
180
+ "epoch": 0.01,
181
+ "learning_rate": 4.998698855736559e-05,
182
+ "loss": 4.1695,
183
+ "step": 14500
184
+ },
185
+ {
186
+ "epoch": 0.01,
187
+ "learning_rate": 4.998652386298579e-05,
188
+ "loss": 4.1406,
189
+ "step": 15000
190
+ },
191
+ {
192
+ "epoch": 0.01,
193
+ "learning_rate": 4.9986059168605994e-05,
194
+ "loss": 4.1275,
195
+ "step": 15500
196
+ },
197
+ {
198
+ "epoch": 0.01,
199
+ "learning_rate": 4.9985594474226196e-05,
200
+ "loss": 4.1032,
201
+ "step": 16000
202
+ },
203
+ {
204
+ "epoch": 0.02,
205
+ "learning_rate": 4.99851297798464e-05,
206
+ "loss": 4.0588,
207
+ "step": 16500
208
+ },
209
+ {
210
+ "epoch": 0.02,
211
+ "learning_rate": 4.9984665085466594e-05,
212
+ "loss": 4.0536,
213
+ "step": 17000
214
+ },
215
+ {
216
+ "epoch": 0.02,
217
+ "learning_rate": 4.9984200391086796e-05,
218
+ "loss": 3.9979,
219
+ "step": 17500
220
+ },
221
+ {
222
+ "epoch": 0.02,
223
+ "learning_rate": 4.998373569670699e-05,
224
+ "loss": 3.9913,
225
+ "step": 18000
226
+ },
227
+ {
228
+ "epoch": 0.02,
229
+ "learning_rate": 4.998327100232719e-05,
230
+ "loss": 3.988,
231
+ "step": 18500
232
+ },
233
+ {
234
+ "epoch": 0.02,
235
+ "learning_rate": 4.9982806307947395e-05,
236
+ "loss": 3.9575,
237
+ "step": 19000
238
+ },
239
+ {
240
+ "epoch": 0.02,
241
+ "learning_rate": 4.998234161356759e-05,
242
+ "loss": 3.942,
243
+ "step": 19500
244
+ },
245
+ {
246
+ "epoch": 0.02,
247
+ "learning_rate": 4.998187691918779e-05,
248
+ "loss": 3.9258,
249
+ "step": 20000
250
+ }
251
+ ],
252
+ "logging_steps": 500,
253
+ "max_steps": 53799300,
254
+ "num_input_tokens_seen": 0,
255
+ "num_train_epochs": 50,
256
+ "save_steps": 1000,
257
+ "total_flos": 3.04664543232e+17,
258
+ "train_batch_size": 7,
259
+ "trial_name": null,
260
+ "trial_params": null
261
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86f29bf77a8fce3925866627ee09b308d44523a9a0f9e0b8d56f38d693a27954
3
+ size 4664
vocab.json ADDED
The diff for this file is too large to render. See raw diff