add tokenizer
Browse files- checkpoint_100/meta.json +1 -0
- checkpoint_100/prank_0_checkpoint.pt +3 -0
- checkpoint_100/prank_0_optimizer.pt +3 -0
- checkpoint_100/prank_0_scheduler.pt +3 -0
- checkpoint_100/prank_1_checkpoint.pt +3 -0
- checkpoint_100/prank_1_optimizer.pt +3 -0
- checkpoint_100/prank_1_scheduler.pt +3 -0
- checkpoint_200/meta.json +1 -0
- checkpoint_200/prank_0_checkpoint.pt +3 -0
- checkpoint_200/prank_0_optimizer.pt +3 -0
- checkpoint_200/prank_0_scheduler.pt +3 -0
- checkpoint_200/prank_1_checkpoint.pt +3 -0
- checkpoint_200/prank_1_optimizer.pt +3 -0
- checkpoint_200/prank_1_scheduler.pt +3 -0
- config.json +29 -0
- latest +1 -0
- merges.txt +0 -0
- pytorch_model.bin +3 -0
- special_tokens_map.json +30 -0
- tokenizer.json +0 -0
- tokenizer_config.json +40 -0
- vocab.json +0 -0
checkpoint_100/meta.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"step": 100}
|
checkpoint_100/prank_0_checkpoint.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0c1ae82e6f503987b1f0cdedde72b9b9e821968904f4aa7f4969b44d80916500
|
3 |
+
size 415755723
|
checkpoint_100/prank_0_optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3082f092ce85b061032277751826a1ad6a944f252a14bac40bec94e8d743767c
|
3 |
+
size 1663005189
|
checkpoint_100/prank_0_scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:778041f2b02da59705117b00ad6ce16f148e7179e8552dc1d96e70af9b6a23ba
|
3 |
+
size 643
|
checkpoint_100/prank_1_checkpoint.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ee86e1af83e182da2adf9742c0731c36449e23ca80213deb427fc707c670978d
|
3 |
+
size 407367379
|
checkpoint_100/prank_1_optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0f37ff798d289db6beeb1cc11bd748e978c5186ee27b197a4c785649e61c177e
|
3 |
+
size 1629451613
|
checkpoint_100/prank_1_scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4e023481b905766990ed421b86df6e13f80a25ee31cf10f6e45b8148699f2a78
|
3 |
+
size 643
|
checkpoint_200/meta.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"step": 200}
|
checkpoint_200/prank_0_checkpoint.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b40d061aa49ae85fbd6211a72438f90c7ee0da82a8608c093c9fa8ad41be29a8
|
3 |
+
size 415755723
|
checkpoint_200/prank_0_optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ccb7b55af07e56e2b3bcb56eada887273eef70f93298c0acf8697d4df77393a8
|
3 |
+
size 1663005189
|
checkpoint_200/prank_0_scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a2a89ce5aebc1ce5c6ba80d07f7a431a0ac53419fe5bfb8bd2c214c28be8630b
|
3 |
+
size 643
|
checkpoint_200/prank_1_checkpoint.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:570a6a2863b505177429b57f89e9ea61e6a32d5d4c0b9d8a992fdcffb5fba8db
|
3 |
+
size 407367379
|
checkpoint_200/prank_1_optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9afbd9b54e1ec57963c8091234235edbe573511b95dc8d5bbff93815a250a800
|
3 |
+
size 1629451613
|
checkpoint_200/prank_1_scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1e21aa3e51853cf52a0839ab39e46ef524f5fa9cd3cc3535296fbac38a388ff6
|
3 |
+
size 643
|
config.json
ADDED
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "facebook/opt-1.3b",
|
3 |
+
"_remove_final_layer_norm": false,
|
4 |
+
"activation_dropout": 0.0,
|
5 |
+
"activation_function": "relu",
|
6 |
+
"architectures": [
|
7 |
+
"OPTForCausalLM"
|
8 |
+
],
|
9 |
+
"attention_dropout": 0.0,
|
10 |
+
"bos_token_id": 2,
|
11 |
+
"do_layer_norm_before": true,
|
12 |
+
"dropout": 0.1,
|
13 |
+
"eos_token_id": 2,
|
14 |
+
"ffn_dim": 8192,
|
15 |
+
"hidden_size": 2048,
|
16 |
+
"init_std": 0.02,
|
17 |
+
"layerdrop": 0.0,
|
18 |
+
"max_position_embeddings": 2048,
|
19 |
+
"model_type": "opt",
|
20 |
+
"num_attention_heads": 32,
|
21 |
+
"num_hidden_layers": 24,
|
22 |
+
"pad_token_id": 1,
|
23 |
+
"prefix": "</s>",
|
24 |
+
"torch_dtype": "float16",
|
25 |
+
"transformers_version": "4.21.1",
|
26 |
+
"use_cache": true,
|
27 |
+
"vocab_size": 50272,
|
28 |
+
"word_embed_proj_dim": 2048
|
29 |
+
}
|
latest
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
200
|
merges.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:30422915e54022654e23ea344f06223de3c9d63056d9f835944f1bbe952f07fd
|
3 |
+
size 407367578
|
special_tokens_map.json
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token": {
|
3 |
+
"content": "</s>",
|
4 |
+
"lstrip": false,
|
5 |
+
"normalized": true,
|
6 |
+
"rstrip": false,
|
7 |
+
"single_word": false
|
8 |
+
},
|
9 |
+
"eos_token": {
|
10 |
+
"content": "</s>",
|
11 |
+
"lstrip": false,
|
12 |
+
"normalized": true,
|
13 |
+
"rstrip": false,
|
14 |
+
"single_word": false
|
15 |
+
},
|
16 |
+
"pad_token": {
|
17 |
+
"content": "<pad>",
|
18 |
+
"lstrip": false,
|
19 |
+
"normalized": true,
|
20 |
+
"rstrip": false,
|
21 |
+
"single_word": false
|
22 |
+
},
|
23 |
+
"unk_token": {
|
24 |
+
"content": "</s>",
|
25 |
+
"lstrip": false,
|
26 |
+
"normalized": true,
|
27 |
+
"rstrip": false,
|
28 |
+
"single_word": false
|
29 |
+
}
|
30 |
+
}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
ADDED
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_bos_token": false,
|
3 |
+
"add_prefix_space": false,
|
4 |
+
"bos_token": {
|
5 |
+
"__type": "AddedToken",
|
6 |
+
"content": "</s>",
|
7 |
+
"lstrip": false,
|
8 |
+
"normalized": true,
|
9 |
+
"rstrip": false,
|
10 |
+
"single_word": false
|
11 |
+
},
|
12 |
+
"eos_token": {
|
13 |
+
"__type": "AddedToken",
|
14 |
+
"content": "</s>",
|
15 |
+
"lstrip": false,
|
16 |
+
"normalized": true,
|
17 |
+
"rstrip": false,
|
18 |
+
"single_word": false
|
19 |
+
},
|
20 |
+
"errors": "replace",
|
21 |
+
"name_or_path": "model_checkpoints/4STDUF0DNHDJHNUKI11UZI9LDQF5A88PVEA0HKX74WSZ0TYPKI",
|
22 |
+
"pad_token": {
|
23 |
+
"__type": "AddedToken",
|
24 |
+
"content": "<pad>",
|
25 |
+
"lstrip": false,
|
26 |
+
"normalized": true,
|
27 |
+
"rstrip": false,
|
28 |
+
"single_word": false
|
29 |
+
},
|
30 |
+
"special_tokens_map_file": null,
|
31 |
+
"tokenizer_class": "GPT2Tokenizer",
|
32 |
+
"unk_token": {
|
33 |
+
"__type": "AddedToken",
|
34 |
+
"content": "</s>",
|
35 |
+
"lstrip": false,
|
36 |
+
"normalized": true,
|
37 |
+
"rstrip": false,
|
38 |
+
"single_word": false
|
39 |
+
}
|
40 |
+
}
|
vocab.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|