BigSalmon commited on
Commit
ceede33
1 Parent(s): bcf5f34

Initial commit

Browse files
Files changed (5) hide show
  1. config.json +67 -0
  2. merges.txt +0 -0
  3. pytorch_model.bin +3 -0
  4. tokenizer.json +0 -0
  5. vocab.json +0 -0
config.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "xhyi/PT_GPTNEO350_ATG",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "GPTNeoForCausalLM"
6
+ ],
7
+ "attention_dropout": 0,
8
+ "attention_layers": [
9
+ "global",
10
+ "local",
11
+ "global",
12
+ "local",
13
+ "global",
14
+ "local",
15
+ "global",
16
+ "local",
17
+ "global",
18
+ "local",
19
+ "global",
20
+ "local",
21
+ "global",
22
+ "local",
23
+ "global",
24
+ "local",
25
+ "global",
26
+ "local",
27
+ "global",
28
+ "local",
29
+ "global",
30
+ "local",
31
+ "global",
32
+ "local"
33
+ ],
34
+ "attention_types": [
35
+ [
36
+ [
37
+ "global",
38
+ "local"
39
+ ],
40
+ 12
41
+ ]
42
+ ],
43
+ "bos_token_id": 50256,
44
+ "embed_dropout": 0,
45
+ "eos_token_id": 50256,
46
+ "gradient_checkpointing": false,
47
+ "hidden_size": 1024,
48
+ "initializer_range": 0.02,
49
+ "intermediate_size": null,
50
+ "layer_norm_epsilon": 1e-05,
51
+ "max_position_embeddings": 2048,
52
+ "model_friendly_id": "PT_GPTNEO350_ATG",
53
+ "model_type": "gpt_neo",
54
+ "num_heads": 16,
55
+ "num_layers": 24,
56
+ "resid_dropout": 0,
57
+ "summary_activation": null,
58
+ "summary_first_dropout": 0.1,
59
+ "summary_proj_to_labels": true,
60
+ "summary_type": "cls_index",
61
+ "summary_use_proj": true,
62
+ "torch_dtype": "float32",
63
+ "transformers_version": "4.17.0.dev0",
64
+ "use_cache": true,
65
+ "vocab_size": 50257,
66
+ "window_size": 256
67
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:076ca1dca24cd83aa885e30ff9b12adb87451397a1888aff94fcd2b835d5f389
3
+ size 1523994281
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
vocab.json ADDED
The diff for this file is too large to render. See raw diff