Chua, Vui Seng
commited on
Commit
•
d96569c
1
Parent(s):
694812a
Add collaterals
Browse files- .gitattributes +2 -0
- README.md +26 -0
- all_results.json +5 -0
- bert-base-squadv1.onnx +3 -0
- eval_nbest_predictions.json +3 -0
- eval_predictions.json +0 -0
- eval_results.json +5 -0
- layer_wise_sparsity_global_rate_0.00.csv +200 -0
- layer_wise_sparsity_global_rate_0.00.md +201 -0
- linear_layer_sparsity_85M_params_0.00_sparsity.csv +73 -0
- linear_layer_sparsity_85M_params_0.00_sparsity.md +74 -0
.gitattributes
CHANGED
@@ -25,3 +25,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
bert-base-squadv1.onnx filter=lfs diff=lfs merge=lfs -text
|
29 |
+
eval_nbest_predictions.json filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
This model is a fork of [```csarron/bert-base-uncased-squad-v1```](https://huggingface.co/csarron/bert-base-uncased-squad-v1).
|
2 |
+
|
3 |
+
```
|
4 |
+
eval_exact_match = 80.9082
|
5 |
+
eval_f1 = 88.2275
|
6 |
+
eval_samples = 10784
|
7 |
+
```
|
8 |
+
|
9 |
+
# Eval
|
10 |
+
```bash
|
11 |
+
export CUDA_VISIBLE_DEVICES=0
|
12 |
+
|
13 |
+
OUTDIR=eval-bert-base-squadv1
|
14 |
+
WORKDIR=transformers/examples/pytorch/question-answering
|
15 |
+
cd $WORKDIR
|
16 |
+
|
17 |
+
nohup python run_qa.py \
|
18 |
+
--model_name_or_path vuiseng9/bert-base-squadv1 \
|
19 |
+
--dataset_name squad \
|
20 |
+
--do_eval \
|
21 |
+
--per_device_eval_batch_size 128 \
|
22 |
+
--max_seq_length 384 \
|
23 |
+
--doc_stride 128 \
|
24 |
+
--overwrite_output_dir \
|
25 |
+
--output_dir $OUTDIR 2>&1 | tee $OUTDIR/run.log &
|
26 |
+
```
|
all_results.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"eval_exact_match": 80.90823084200568,
|
3 |
+
"eval_f1": 88.22754061399627,
|
4 |
+
"eval_samples": 10784
|
5 |
+
}
|
bert-base-squadv1.onnx
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:95dc1f18387c59847a7d4e63b18ab78f87ec3e04d6c65f6c492d9bba9681622e
|
3 |
+
size 435644640
|
eval_nbest_predictions.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d30c9ccf3e7b843265ac27aa7189c8a5ac89af1e8dfc8fd7bbcfa3c9f7a83979
|
3 |
+
size 49496690
|
eval_predictions.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
eval_results.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"eval_exact_match": 80.90823084200568,
|
3 |
+
"eval_f1": 88.22754061399627,
|
4 |
+
"eval_samples": 10784
|
5 |
+
}
|
layer_wise_sparsity_global_rate_0.00.csv
ADDED
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
,layer_id,layer_type,param_type,shape,nparam,nnz,sparsity
|
2 |
+
0,bert.embeddings.word_embeddings,Embedding,weight,"[30522, 768]",23440896,23440896,0.0
|
3 |
+
1,bert.embeddings.position_embeddings,Embedding,weight,"[512, 768]",393216,393216,0.0
|
4 |
+
2,bert.embeddings.token_type_embeddings,Embedding,weight,"[2, 768]",1536,1536,0.0
|
5 |
+
3,bert.embeddings.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
6 |
+
4,bert.embeddings.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
7 |
+
5,bert.encoder.layer.0.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
8 |
+
6,bert.encoder.layer.0.attention.self.query,Linear,bias,[768],768,768,0.0
|
9 |
+
7,bert.encoder.layer.0.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
10 |
+
8,bert.encoder.layer.0.attention.self.key,Linear,bias,[768],768,768,0.0
|
11 |
+
9,bert.encoder.layer.0.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
12 |
+
10,bert.encoder.layer.0.attention.self.value,Linear,bias,[768],768,768,0.0
|
13 |
+
11,bert.encoder.layer.0.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
14 |
+
12,bert.encoder.layer.0.attention.output.dense,Linear,bias,[768],768,768,0.0
|
15 |
+
13,bert.encoder.layer.0.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
16 |
+
14,bert.encoder.layer.0.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
17 |
+
15,bert.encoder.layer.0.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
18 |
+
16,bert.encoder.layer.0.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
19 |
+
17,bert.encoder.layer.0.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
20 |
+
18,bert.encoder.layer.0.output.dense,Linear,bias,[768],768,768,0.0
|
21 |
+
19,bert.encoder.layer.0.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
22 |
+
20,bert.encoder.layer.0.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
23 |
+
21,bert.encoder.layer.1.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
24 |
+
22,bert.encoder.layer.1.attention.self.query,Linear,bias,[768],768,768,0.0
|
25 |
+
23,bert.encoder.layer.1.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
26 |
+
24,bert.encoder.layer.1.attention.self.key,Linear,bias,[768],768,768,0.0
|
27 |
+
25,bert.encoder.layer.1.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
28 |
+
26,bert.encoder.layer.1.attention.self.value,Linear,bias,[768],768,768,0.0
|
29 |
+
27,bert.encoder.layer.1.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
30 |
+
28,bert.encoder.layer.1.attention.output.dense,Linear,bias,[768],768,768,0.0
|
31 |
+
29,bert.encoder.layer.1.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
32 |
+
30,bert.encoder.layer.1.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
33 |
+
31,bert.encoder.layer.1.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
34 |
+
32,bert.encoder.layer.1.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
35 |
+
33,bert.encoder.layer.1.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
36 |
+
34,bert.encoder.layer.1.output.dense,Linear,bias,[768],768,768,0.0
|
37 |
+
35,bert.encoder.layer.1.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
38 |
+
36,bert.encoder.layer.1.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
39 |
+
37,bert.encoder.layer.2.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
40 |
+
38,bert.encoder.layer.2.attention.self.query,Linear,bias,[768],768,768,0.0
|
41 |
+
39,bert.encoder.layer.2.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
42 |
+
40,bert.encoder.layer.2.attention.self.key,Linear,bias,[768],768,768,0.0
|
43 |
+
41,bert.encoder.layer.2.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
44 |
+
42,bert.encoder.layer.2.attention.self.value,Linear,bias,[768],768,768,0.0
|
45 |
+
43,bert.encoder.layer.2.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
46 |
+
44,bert.encoder.layer.2.attention.output.dense,Linear,bias,[768],768,768,0.0
|
47 |
+
45,bert.encoder.layer.2.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
48 |
+
46,bert.encoder.layer.2.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
49 |
+
47,bert.encoder.layer.2.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
50 |
+
48,bert.encoder.layer.2.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
51 |
+
49,bert.encoder.layer.2.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
52 |
+
50,bert.encoder.layer.2.output.dense,Linear,bias,[768],768,768,0.0
|
53 |
+
51,bert.encoder.layer.2.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
54 |
+
52,bert.encoder.layer.2.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
55 |
+
53,bert.encoder.layer.3.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
56 |
+
54,bert.encoder.layer.3.attention.self.query,Linear,bias,[768],768,768,0.0
|
57 |
+
55,bert.encoder.layer.3.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
58 |
+
56,bert.encoder.layer.3.attention.self.key,Linear,bias,[768],768,768,0.0
|
59 |
+
57,bert.encoder.layer.3.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
60 |
+
58,bert.encoder.layer.3.attention.self.value,Linear,bias,[768],768,768,0.0
|
61 |
+
59,bert.encoder.layer.3.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
62 |
+
60,bert.encoder.layer.3.attention.output.dense,Linear,bias,[768],768,768,0.0
|
63 |
+
61,bert.encoder.layer.3.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
64 |
+
62,bert.encoder.layer.3.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
65 |
+
63,bert.encoder.layer.3.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
66 |
+
64,bert.encoder.layer.3.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
67 |
+
65,bert.encoder.layer.3.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
68 |
+
66,bert.encoder.layer.3.output.dense,Linear,bias,[768],768,768,0.0
|
69 |
+
67,bert.encoder.layer.3.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
70 |
+
68,bert.encoder.layer.3.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
71 |
+
69,bert.encoder.layer.4.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
72 |
+
70,bert.encoder.layer.4.attention.self.query,Linear,bias,[768],768,768,0.0
|
73 |
+
71,bert.encoder.layer.4.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
74 |
+
72,bert.encoder.layer.4.attention.self.key,Linear,bias,[768],768,768,0.0
|
75 |
+
73,bert.encoder.layer.4.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
76 |
+
74,bert.encoder.layer.4.attention.self.value,Linear,bias,[768],768,768,0.0
|
77 |
+
75,bert.encoder.layer.4.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
78 |
+
76,bert.encoder.layer.4.attention.output.dense,Linear,bias,[768],768,768,0.0
|
79 |
+
77,bert.encoder.layer.4.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
80 |
+
78,bert.encoder.layer.4.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
81 |
+
79,bert.encoder.layer.4.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
82 |
+
80,bert.encoder.layer.4.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
83 |
+
81,bert.encoder.layer.4.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
84 |
+
82,bert.encoder.layer.4.output.dense,Linear,bias,[768],768,768,0.0
|
85 |
+
83,bert.encoder.layer.4.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
86 |
+
84,bert.encoder.layer.4.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
87 |
+
85,bert.encoder.layer.5.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
88 |
+
86,bert.encoder.layer.5.attention.self.query,Linear,bias,[768],768,768,0.0
|
89 |
+
87,bert.encoder.layer.5.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
90 |
+
88,bert.encoder.layer.5.attention.self.key,Linear,bias,[768],768,768,0.0
|
91 |
+
89,bert.encoder.layer.5.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
92 |
+
90,bert.encoder.layer.5.attention.self.value,Linear,bias,[768],768,768,0.0
|
93 |
+
91,bert.encoder.layer.5.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
94 |
+
92,bert.encoder.layer.5.attention.output.dense,Linear,bias,[768],768,768,0.0
|
95 |
+
93,bert.encoder.layer.5.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
96 |
+
94,bert.encoder.layer.5.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
97 |
+
95,bert.encoder.layer.5.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
98 |
+
96,bert.encoder.layer.5.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
99 |
+
97,bert.encoder.layer.5.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
100 |
+
98,bert.encoder.layer.5.output.dense,Linear,bias,[768],768,768,0.0
|
101 |
+
99,bert.encoder.layer.5.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
102 |
+
100,bert.encoder.layer.5.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
103 |
+
101,bert.encoder.layer.6.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
104 |
+
102,bert.encoder.layer.6.attention.self.query,Linear,bias,[768],768,768,0.0
|
105 |
+
103,bert.encoder.layer.6.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
106 |
+
104,bert.encoder.layer.6.attention.self.key,Linear,bias,[768],768,768,0.0
|
107 |
+
105,bert.encoder.layer.6.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
108 |
+
106,bert.encoder.layer.6.attention.self.value,Linear,bias,[768],768,768,0.0
|
109 |
+
107,bert.encoder.layer.6.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
110 |
+
108,bert.encoder.layer.6.attention.output.dense,Linear,bias,[768],768,768,0.0
|
111 |
+
109,bert.encoder.layer.6.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
112 |
+
110,bert.encoder.layer.6.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
113 |
+
111,bert.encoder.layer.6.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
114 |
+
112,bert.encoder.layer.6.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
115 |
+
113,bert.encoder.layer.6.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
116 |
+
114,bert.encoder.layer.6.output.dense,Linear,bias,[768],768,768,0.0
|
117 |
+
115,bert.encoder.layer.6.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
118 |
+
116,bert.encoder.layer.6.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
119 |
+
117,bert.encoder.layer.7.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
120 |
+
118,bert.encoder.layer.7.attention.self.query,Linear,bias,[768],768,768,0.0
|
121 |
+
119,bert.encoder.layer.7.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
122 |
+
120,bert.encoder.layer.7.attention.self.key,Linear,bias,[768],768,768,0.0
|
123 |
+
121,bert.encoder.layer.7.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
124 |
+
122,bert.encoder.layer.7.attention.self.value,Linear,bias,[768],768,768,0.0
|
125 |
+
123,bert.encoder.layer.7.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
126 |
+
124,bert.encoder.layer.7.attention.output.dense,Linear,bias,[768],768,768,0.0
|
127 |
+
125,bert.encoder.layer.7.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
128 |
+
126,bert.encoder.layer.7.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
129 |
+
127,bert.encoder.layer.7.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
130 |
+
128,bert.encoder.layer.7.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
131 |
+
129,bert.encoder.layer.7.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
132 |
+
130,bert.encoder.layer.7.output.dense,Linear,bias,[768],768,768,0.0
|
133 |
+
131,bert.encoder.layer.7.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
134 |
+
132,bert.encoder.layer.7.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
135 |
+
133,bert.encoder.layer.8.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
136 |
+
134,bert.encoder.layer.8.attention.self.query,Linear,bias,[768],768,768,0.0
|
137 |
+
135,bert.encoder.layer.8.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
138 |
+
136,bert.encoder.layer.8.attention.self.key,Linear,bias,[768],768,768,0.0
|
139 |
+
137,bert.encoder.layer.8.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
140 |
+
138,bert.encoder.layer.8.attention.self.value,Linear,bias,[768],768,768,0.0
|
141 |
+
139,bert.encoder.layer.8.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
142 |
+
140,bert.encoder.layer.8.attention.output.dense,Linear,bias,[768],768,768,0.0
|
143 |
+
141,bert.encoder.layer.8.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
144 |
+
142,bert.encoder.layer.8.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
145 |
+
143,bert.encoder.layer.8.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
146 |
+
144,bert.encoder.layer.8.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
147 |
+
145,bert.encoder.layer.8.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
148 |
+
146,bert.encoder.layer.8.output.dense,Linear,bias,[768],768,768,0.0
|
149 |
+
147,bert.encoder.layer.8.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
150 |
+
148,bert.encoder.layer.8.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
151 |
+
149,bert.encoder.layer.9.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
152 |
+
150,bert.encoder.layer.9.attention.self.query,Linear,bias,[768],768,768,0.0
|
153 |
+
151,bert.encoder.layer.9.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
154 |
+
152,bert.encoder.layer.9.attention.self.key,Linear,bias,[768],768,768,0.0
|
155 |
+
153,bert.encoder.layer.9.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
156 |
+
154,bert.encoder.layer.9.attention.self.value,Linear,bias,[768],768,768,0.0
|
157 |
+
155,bert.encoder.layer.9.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
158 |
+
156,bert.encoder.layer.9.attention.output.dense,Linear,bias,[768],768,768,0.0
|
159 |
+
157,bert.encoder.layer.9.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
160 |
+
158,bert.encoder.layer.9.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
161 |
+
159,bert.encoder.layer.9.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
162 |
+
160,bert.encoder.layer.9.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
163 |
+
161,bert.encoder.layer.9.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
164 |
+
162,bert.encoder.layer.9.output.dense,Linear,bias,[768],768,768,0.0
|
165 |
+
163,bert.encoder.layer.9.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
166 |
+
164,bert.encoder.layer.9.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
167 |
+
165,bert.encoder.layer.10.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
168 |
+
166,bert.encoder.layer.10.attention.self.query,Linear,bias,[768],768,768,0.0
|
169 |
+
167,bert.encoder.layer.10.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
170 |
+
168,bert.encoder.layer.10.attention.self.key,Linear,bias,[768],768,768,0.0
|
171 |
+
169,bert.encoder.layer.10.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
172 |
+
170,bert.encoder.layer.10.attention.self.value,Linear,bias,[768],768,768,0.0
|
173 |
+
171,bert.encoder.layer.10.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
174 |
+
172,bert.encoder.layer.10.attention.output.dense,Linear,bias,[768],768,768,0.0
|
175 |
+
173,bert.encoder.layer.10.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
176 |
+
174,bert.encoder.layer.10.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
177 |
+
175,bert.encoder.layer.10.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
178 |
+
176,bert.encoder.layer.10.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
179 |
+
177,bert.encoder.layer.10.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
180 |
+
178,bert.encoder.layer.10.output.dense,Linear,bias,[768],768,768,0.0
|
181 |
+
179,bert.encoder.layer.10.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
182 |
+
180,bert.encoder.layer.10.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
183 |
+
181,bert.encoder.layer.11.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
184 |
+
182,bert.encoder.layer.11.attention.self.query,Linear,bias,[768],768,768,0.0
|
185 |
+
183,bert.encoder.layer.11.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
186 |
+
184,bert.encoder.layer.11.attention.self.key,Linear,bias,[768],768,768,0.0
|
187 |
+
185,bert.encoder.layer.11.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
188 |
+
186,bert.encoder.layer.11.attention.self.value,Linear,bias,[768],768,768,0.0
|
189 |
+
187,bert.encoder.layer.11.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
190 |
+
188,bert.encoder.layer.11.attention.output.dense,Linear,bias,[768],768,768,0.0
|
191 |
+
189,bert.encoder.layer.11.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
192 |
+
190,bert.encoder.layer.11.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
193 |
+
191,bert.encoder.layer.11.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
194 |
+
192,bert.encoder.layer.11.intermediate.dense,Linear,bias,[3072],3072,3072,0.0
|
195 |
+
193,bert.encoder.layer.11.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
196 |
+
194,bert.encoder.layer.11.output.dense,Linear,bias,[768],768,768,0.0
|
197 |
+
195,bert.encoder.layer.11.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
198 |
+
196,bert.encoder.layer.11.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
199 |
+
197,qa_outputs,Linear,weight,"[2, 768]",1536,1536,0.0
|
200 |
+
198,qa_outputs,Linear,bias,[2],2,2,0.0
|
layer_wise_sparsity_global_rate_0.00.md
ADDED
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
| | layer_id | layer_type | param_type | shape | nparam | nnz | sparsity |
|
2 |
+
|----:|:-------------------------------------------------|:-------------|:-------------|:-------------|---------:|---------:|-----------:|
|
3 |
+
| 0 | bert.embeddings.word_embeddings | Embedding | weight | [30522, 768] | 23440896 | 23440896 | 0 |
|
4 |
+
| 1 | bert.embeddings.position_embeddings | Embedding | weight | [512, 768] | 393216 | 393216 | 0 |
|
5 |
+
| 2 | bert.embeddings.token_type_embeddings | Embedding | weight | [2, 768] | 1536 | 1536 | 0 |
|
6 |
+
| 3 | bert.embeddings.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
7 |
+
| 4 | bert.embeddings.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
8 |
+
| 5 | bert.encoder.layer.0.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
9 |
+
| 6 | bert.encoder.layer.0.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
10 |
+
| 7 | bert.encoder.layer.0.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
11 |
+
| 8 | bert.encoder.layer.0.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
12 |
+
| 9 | bert.encoder.layer.0.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
13 |
+
| 10 | bert.encoder.layer.0.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
14 |
+
| 11 | bert.encoder.layer.0.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
15 |
+
| 12 | bert.encoder.layer.0.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
16 |
+
| 13 | bert.encoder.layer.0.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
17 |
+
| 14 | bert.encoder.layer.0.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
18 |
+
| 15 | bert.encoder.layer.0.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
19 |
+
| 16 | bert.encoder.layer.0.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
20 |
+
| 17 | bert.encoder.layer.0.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
21 |
+
| 18 | bert.encoder.layer.0.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
22 |
+
| 19 | bert.encoder.layer.0.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
23 |
+
| 20 | bert.encoder.layer.0.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
24 |
+
| 21 | bert.encoder.layer.1.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
25 |
+
| 22 | bert.encoder.layer.1.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
26 |
+
| 23 | bert.encoder.layer.1.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
27 |
+
| 24 | bert.encoder.layer.1.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
28 |
+
| 25 | bert.encoder.layer.1.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
29 |
+
| 26 | bert.encoder.layer.1.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
30 |
+
| 27 | bert.encoder.layer.1.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
31 |
+
| 28 | bert.encoder.layer.1.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
32 |
+
| 29 | bert.encoder.layer.1.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
33 |
+
| 30 | bert.encoder.layer.1.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
34 |
+
| 31 | bert.encoder.layer.1.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
35 |
+
| 32 | bert.encoder.layer.1.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
36 |
+
| 33 | bert.encoder.layer.1.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
37 |
+
| 34 | bert.encoder.layer.1.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
38 |
+
| 35 | bert.encoder.layer.1.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
39 |
+
| 36 | bert.encoder.layer.1.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
40 |
+
| 37 | bert.encoder.layer.2.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
41 |
+
| 38 | bert.encoder.layer.2.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
42 |
+
| 39 | bert.encoder.layer.2.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
43 |
+
| 40 | bert.encoder.layer.2.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
44 |
+
| 41 | bert.encoder.layer.2.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
45 |
+
| 42 | bert.encoder.layer.2.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
46 |
+
| 43 | bert.encoder.layer.2.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
47 |
+
| 44 | bert.encoder.layer.2.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
48 |
+
| 45 | bert.encoder.layer.2.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
49 |
+
| 46 | bert.encoder.layer.2.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
50 |
+
| 47 | bert.encoder.layer.2.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
51 |
+
| 48 | bert.encoder.layer.2.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
52 |
+
| 49 | bert.encoder.layer.2.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
53 |
+
| 50 | bert.encoder.layer.2.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
54 |
+
| 51 | bert.encoder.layer.2.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
55 |
+
| 52 | bert.encoder.layer.2.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
56 |
+
| 53 | bert.encoder.layer.3.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
57 |
+
| 54 | bert.encoder.layer.3.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
58 |
+
| 55 | bert.encoder.layer.3.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
59 |
+
| 56 | bert.encoder.layer.3.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
60 |
+
| 57 | bert.encoder.layer.3.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
61 |
+
| 58 | bert.encoder.layer.3.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
62 |
+
| 59 | bert.encoder.layer.3.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
63 |
+
| 60 | bert.encoder.layer.3.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
64 |
+
| 61 | bert.encoder.layer.3.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
65 |
+
| 62 | bert.encoder.layer.3.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
66 |
+
| 63 | bert.encoder.layer.3.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
67 |
+
| 64 | bert.encoder.layer.3.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
68 |
+
| 65 | bert.encoder.layer.3.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
69 |
+
| 66 | bert.encoder.layer.3.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
70 |
+
| 67 | bert.encoder.layer.3.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
71 |
+
| 68 | bert.encoder.layer.3.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
72 |
+
| 69 | bert.encoder.layer.4.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
73 |
+
| 70 | bert.encoder.layer.4.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
74 |
+
| 71 | bert.encoder.layer.4.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
75 |
+
| 72 | bert.encoder.layer.4.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
76 |
+
| 73 | bert.encoder.layer.4.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
77 |
+
| 74 | bert.encoder.layer.4.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
78 |
+
| 75 | bert.encoder.layer.4.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
79 |
+
| 76 | bert.encoder.layer.4.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
80 |
+
| 77 | bert.encoder.layer.4.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
81 |
+
| 78 | bert.encoder.layer.4.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
82 |
+
| 79 | bert.encoder.layer.4.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
83 |
+
| 80 | bert.encoder.layer.4.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
84 |
+
| 81 | bert.encoder.layer.4.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
85 |
+
| 82 | bert.encoder.layer.4.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
86 |
+
| 83 | bert.encoder.layer.4.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
87 |
+
| 84 | bert.encoder.layer.4.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
88 |
+
| 85 | bert.encoder.layer.5.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
89 |
+
| 86 | bert.encoder.layer.5.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
90 |
+
| 87 | bert.encoder.layer.5.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
91 |
+
| 88 | bert.encoder.layer.5.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
92 |
+
| 89 | bert.encoder.layer.5.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
93 |
+
| 90 | bert.encoder.layer.5.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
94 |
+
| 91 | bert.encoder.layer.5.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
95 |
+
| 92 | bert.encoder.layer.5.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
96 |
+
| 93 | bert.encoder.layer.5.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
97 |
+
| 94 | bert.encoder.layer.5.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
98 |
+
| 95 | bert.encoder.layer.5.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
99 |
+
| 96 | bert.encoder.layer.5.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
100 |
+
| 97 | bert.encoder.layer.5.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
101 |
+
| 98 | bert.encoder.layer.5.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
102 |
+
| 99 | bert.encoder.layer.5.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
103 |
+
| 100 | bert.encoder.layer.5.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
104 |
+
| 101 | bert.encoder.layer.6.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
105 |
+
| 102 | bert.encoder.layer.6.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
106 |
+
| 103 | bert.encoder.layer.6.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
107 |
+
| 104 | bert.encoder.layer.6.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
108 |
+
| 105 | bert.encoder.layer.6.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
109 |
+
| 106 | bert.encoder.layer.6.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
110 |
+
| 107 | bert.encoder.layer.6.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
111 |
+
| 108 | bert.encoder.layer.6.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
112 |
+
| 109 | bert.encoder.layer.6.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
113 |
+
| 110 | bert.encoder.layer.6.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
114 |
+
| 111 | bert.encoder.layer.6.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
115 |
+
| 112 | bert.encoder.layer.6.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
116 |
+
| 113 | bert.encoder.layer.6.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
117 |
+
| 114 | bert.encoder.layer.6.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
118 |
+
| 115 | bert.encoder.layer.6.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
119 |
+
| 116 | bert.encoder.layer.6.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
120 |
+
| 117 | bert.encoder.layer.7.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
121 |
+
| 118 | bert.encoder.layer.7.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
122 |
+
| 119 | bert.encoder.layer.7.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
123 |
+
| 120 | bert.encoder.layer.7.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
124 |
+
| 121 | bert.encoder.layer.7.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
125 |
+
| 122 | bert.encoder.layer.7.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
126 |
+
| 123 | bert.encoder.layer.7.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
127 |
+
| 124 | bert.encoder.layer.7.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
128 |
+
| 125 | bert.encoder.layer.7.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
129 |
+
| 126 | bert.encoder.layer.7.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
130 |
+
| 127 | bert.encoder.layer.7.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
131 |
+
| 128 | bert.encoder.layer.7.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
132 |
+
| 129 | bert.encoder.layer.7.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
133 |
+
| 130 | bert.encoder.layer.7.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
134 |
+
| 131 | bert.encoder.layer.7.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
135 |
+
| 132 | bert.encoder.layer.7.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
136 |
+
| 133 | bert.encoder.layer.8.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
137 |
+
| 134 | bert.encoder.layer.8.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
138 |
+
| 135 | bert.encoder.layer.8.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
139 |
+
| 136 | bert.encoder.layer.8.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
140 |
+
| 137 | bert.encoder.layer.8.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
141 |
+
| 138 | bert.encoder.layer.8.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
142 |
+
| 139 | bert.encoder.layer.8.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
143 |
+
| 140 | bert.encoder.layer.8.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
144 |
+
| 141 | bert.encoder.layer.8.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
145 |
+
| 142 | bert.encoder.layer.8.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
146 |
+
| 143 | bert.encoder.layer.8.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
147 |
+
| 144 | bert.encoder.layer.8.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
148 |
+
| 145 | bert.encoder.layer.8.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
149 |
+
| 146 | bert.encoder.layer.8.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
150 |
+
| 147 | bert.encoder.layer.8.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
151 |
+
| 148 | bert.encoder.layer.8.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
152 |
+
| 149 | bert.encoder.layer.9.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
153 |
+
| 150 | bert.encoder.layer.9.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
154 |
+
| 151 | bert.encoder.layer.9.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
155 |
+
| 152 | bert.encoder.layer.9.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
156 |
+
| 153 | bert.encoder.layer.9.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
157 |
+
| 154 | bert.encoder.layer.9.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
158 |
+
| 155 | bert.encoder.layer.9.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
159 |
+
| 156 | bert.encoder.layer.9.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
160 |
+
| 157 | bert.encoder.layer.9.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
161 |
+
| 158 | bert.encoder.layer.9.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
162 |
+
| 159 | bert.encoder.layer.9.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
163 |
+
| 160 | bert.encoder.layer.9.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
164 |
+
| 161 | bert.encoder.layer.9.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
165 |
+
| 162 | bert.encoder.layer.9.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
166 |
+
| 163 | bert.encoder.layer.9.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
167 |
+
| 164 | bert.encoder.layer.9.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
168 |
+
| 165 | bert.encoder.layer.10.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
169 |
+
| 166 | bert.encoder.layer.10.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
170 |
+
| 167 | bert.encoder.layer.10.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
171 |
+
| 168 | bert.encoder.layer.10.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
172 |
+
| 169 | bert.encoder.layer.10.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
173 |
+
| 170 | bert.encoder.layer.10.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
174 |
+
| 171 | bert.encoder.layer.10.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
175 |
+
| 172 | bert.encoder.layer.10.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
176 |
+
| 173 | bert.encoder.layer.10.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
177 |
+
| 174 | bert.encoder.layer.10.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
178 |
+
| 175 | bert.encoder.layer.10.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
179 |
+
| 176 | bert.encoder.layer.10.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
180 |
+
| 177 | bert.encoder.layer.10.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
181 |
+
| 178 | bert.encoder.layer.10.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
182 |
+
| 179 | bert.encoder.layer.10.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
183 |
+
| 180 | bert.encoder.layer.10.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
184 |
+
| 181 | bert.encoder.layer.11.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
185 |
+
| 182 | bert.encoder.layer.11.attention.self.query | Linear | bias | [768] | 768 | 768 | 0 |
|
186 |
+
| 183 | bert.encoder.layer.11.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
187 |
+
| 184 | bert.encoder.layer.11.attention.self.key | Linear | bias | [768] | 768 | 768 | 0 |
|
188 |
+
| 185 | bert.encoder.layer.11.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
189 |
+
| 186 | bert.encoder.layer.11.attention.self.value | Linear | bias | [768] | 768 | 768 | 0 |
|
190 |
+
| 187 | bert.encoder.layer.11.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
191 |
+
| 188 | bert.encoder.layer.11.attention.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
192 |
+
| 189 | bert.encoder.layer.11.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
193 |
+
| 190 | bert.encoder.layer.11.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
194 |
+
| 191 | bert.encoder.layer.11.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
195 |
+
| 192 | bert.encoder.layer.11.intermediate.dense | Linear | bias | [3072] | 3072 | 3072 | 0 |
|
196 |
+
| 193 | bert.encoder.layer.11.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
197 |
+
| 194 | bert.encoder.layer.11.output.dense | Linear | bias | [768] | 768 | 768 | 0 |
|
198 |
+
| 195 | bert.encoder.layer.11.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
199 |
+
| 196 | bert.encoder.layer.11.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
200 |
+
| 197 | qa_outputs | Linear | weight | [2, 768] | 1536 | 1536 | 0 |
|
201 |
+
| 198 | qa_outputs | Linear | bias | [2] | 2 | 2 | 0 |
|
linear_layer_sparsity_85M_params_0.00_sparsity.csv
ADDED
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
,layer_id,layer_type,param_type,shape,nparam,nnz,sparsity
|
2 |
+
5,bert.encoder.layer.0.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
3 |
+
7,bert.encoder.layer.0.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
4 |
+
9,bert.encoder.layer.0.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
5 |
+
11,bert.encoder.layer.0.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
6 |
+
15,bert.encoder.layer.0.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
7 |
+
17,bert.encoder.layer.0.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
8 |
+
21,bert.encoder.layer.1.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
9 |
+
23,bert.encoder.layer.1.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
10 |
+
25,bert.encoder.layer.1.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
11 |
+
27,bert.encoder.layer.1.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
12 |
+
31,bert.encoder.layer.1.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
13 |
+
33,bert.encoder.layer.1.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
14 |
+
37,bert.encoder.layer.2.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
15 |
+
39,bert.encoder.layer.2.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
16 |
+
41,bert.encoder.layer.2.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
17 |
+
43,bert.encoder.layer.2.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
18 |
+
47,bert.encoder.layer.2.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
19 |
+
49,bert.encoder.layer.2.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
20 |
+
53,bert.encoder.layer.3.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
21 |
+
55,bert.encoder.layer.3.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
22 |
+
57,bert.encoder.layer.3.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
23 |
+
59,bert.encoder.layer.3.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
24 |
+
63,bert.encoder.layer.3.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
25 |
+
65,bert.encoder.layer.3.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
26 |
+
69,bert.encoder.layer.4.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
27 |
+
71,bert.encoder.layer.4.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
28 |
+
73,bert.encoder.layer.4.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
29 |
+
75,bert.encoder.layer.4.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
30 |
+
79,bert.encoder.layer.4.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
31 |
+
81,bert.encoder.layer.4.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
32 |
+
85,bert.encoder.layer.5.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
33 |
+
87,bert.encoder.layer.5.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
34 |
+
89,bert.encoder.layer.5.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
35 |
+
91,bert.encoder.layer.5.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
36 |
+
95,bert.encoder.layer.5.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
37 |
+
97,bert.encoder.layer.5.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
38 |
+
101,bert.encoder.layer.6.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
39 |
+
103,bert.encoder.layer.6.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
40 |
+
105,bert.encoder.layer.6.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
41 |
+
107,bert.encoder.layer.6.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
42 |
+
111,bert.encoder.layer.6.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
43 |
+
113,bert.encoder.layer.6.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
44 |
+
117,bert.encoder.layer.7.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
45 |
+
119,bert.encoder.layer.7.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
46 |
+
121,bert.encoder.layer.7.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
47 |
+
123,bert.encoder.layer.7.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
48 |
+
127,bert.encoder.layer.7.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
49 |
+
129,bert.encoder.layer.7.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
50 |
+
133,bert.encoder.layer.8.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
51 |
+
135,bert.encoder.layer.8.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
52 |
+
137,bert.encoder.layer.8.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
53 |
+
139,bert.encoder.layer.8.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
54 |
+
143,bert.encoder.layer.8.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
55 |
+
145,bert.encoder.layer.8.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
56 |
+
149,bert.encoder.layer.9.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
57 |
+
151,bert.encoder.layer.9.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
58 |
+
153,bert.encoder.layer.9.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
59 |
+
155,bert.encoder.layer.9.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
60 |
+
159,bert.encoder.layer.9.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
61 |
+
161,bert.encoder.layer.9.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
62 |
+
165,bert.encoder.layer.10.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
63 |
+
167,bert.encoder.layer.10.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
64 |
+
169,bert.encoder.layer.10.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
65 |
+
171,bert.encoder.layer.10.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
66 |
+
175,bert.encoder.layer.10.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
67 |
+
177,bert.encoder.layer.10.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
68 |
+
181,bert.encoder.layer.11.attention.self.query,Linear,weight,"[768, 768]",589824,589824,0.0
|
69 |
+
183,bert.encoder.layer.11.attention.self.key,Linear,weight,"[768, 768]",589824,589824,0.0
|
70 |
+
185,bert.encoder.layer.11.attention.self.value,Linear,weight,"[768, 768]",589824,589824,0.0
|
71 |
+
187,bert.encoder.layer.11.attention.output.dense,Linear,weight,"[768, 768]",589824,589824,0.0
|
72 |
+
191,bert.encoder.layer.11.intermediate.dense,Linear,weight,"[3072, 768]",2359296,2359296,0.0
|
73 |
+
193,bert.encoder.layer.11.output.dense,Linear,weight,"[768, 3072]",2359296,2359296,0.0
|
linear_layer_sparsity_85M_params_0.00_sparsity.md
ADDED
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
| | layer_id | layer_type | param_type | shape | nparam | nnz | sparsity |
|
2 |
+
|----:|:---------------------------------------------|:-------------|:-------------|:------------|---------:|--------:|-----------:|
|
3 |
+
| 5 | bert.encoder.layer.0.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
4 |
+
| 7 | bert.encoder.layer.0.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
5 |
+
| 9 | bert.encoder.layer.0.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
6 |
+
| 11 | bert.encoder.layer.0.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
7 |
+
| 15 | bert.encoder.layer.0.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
8 |
+
| 17 | bert.encoder.layer.0.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
9 |
+
| 21 | bert.encoder.layer.1.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
10 |
+
| 23 | bert.encoder.layer.1.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
11 |
+
| 25 | bert.encoder.layer.1.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
12 |
+
| 27 | bert.encoder.layer.1.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
13 |
+
| 31 | bert.encoder.layer.1.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
14 |
+
| 33 | bert.encoder.layer.1.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
15 |
+
| 37 | bert.encoder.layer.2.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
16 |
+
| 39 | bert.encoder.layer.2.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
17 |
+
| 41 | bert.encoder.layer.2.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
18 |
+
| 43 | bert.encoder.layer.2.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
19 |
+
| 47 | bert.encoder.layer.2.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
20 |
+
| 49 | bert.encoder.layer.2.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
21 |
+
| 53 | bert.encoder.layer.3.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
22 |
+
| 55 | bert.encoder.layer.3.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
23 |
+
| 57 | bert.encoder.layer.3.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
24 |
+
| 59 | bert.encoder.layer.3.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
25 |
+
| 63 | bert.encoder.layer.3.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
26 |
+
| 65 | bert.encoder.layer.3.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
27 |
+
| 69 | bert.encoder.layer.4.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
28 |
+
| 71 | bert.encoder.layer.4.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
29 |
+
| 73 | bert.encoder.layer.4.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
30 |
+
| 75 | bert.encoder.layer.4.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
31 |
+
| 79 | bert.encoder.layer.4.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
32 |
+
| 81 | bert.encoder.layer.4.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
33 |
+
| 85 | bert.encoder.layer.5.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
34 |
+
| 87 | bert.encoder.layer.5.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
35 |
+
| 89 | bert.encoder.layer.5.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
36 |
+
| 91 | bert.encoder.layer.5.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
37 |
+
| 95 | bert.encoder.layer.5.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
38 |
+
| 97 | bert.encoder.layer.5.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
39 |
+
| 101 | bert.encoder.layer.6.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
40 |
+
| 103 | bert.encoder.layer.6.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
41 |
+
| 105 | bert.encoder.layer.6.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
42 |
+
| 107 | bert.encoder.layer.6.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
43 |
+
| 111 | bert.encoder.layer.6.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
44 |
+
| 113 | bert.encoder.layer.6.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
45 |
+
| 117 | bert.encoder.layer.7.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
46 |
+
| 119 | bert.encoder.layer.7.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
47 |
+
| 121 | bert.encoder.layer.7.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
48 |
+
| 123 | bert.encoder.layer.7.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
49 |
+
| 127 | bert.encoder.layer.7.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
50 |
+
| 129 | bert.encoder.layer.7.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
51 |
+
| 133 | bert.encoder.layer.8.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
52 |
+
| 135 | bert.encoder.layer.8.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
53 |
+
| 137 | bert.encoder.layer.8.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
54 |
+
| 139 | bert.encoder.layer.8.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
55 |
+
| 143 | bert.encoder.layer.8.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
56 |
+
| 145 | bert.encoder.layer.8.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
57 |
+
| 149 | bert.encoder.layer.9.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
58 |
+
| 151 | bert.encoder.layer.9.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
59 |
+
| 153 | bert.encoder.layer.9.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
60 |
+
| 155 | bert.encoder.layer.9.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
61 |
+
| 159 | bert.encoder.layer.9.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
62 |
+
| 161 | bert.encoder.layer.9.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
63 |
+
| 165 | bert.encoder.layer.10.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
64 |
+
| 167 | bert.encoder.layer.10.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
65 |
+
| 169 | bert.encoder.layer.10.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
66 |
+
| 171 | bert.encoder.layer.10.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
67 |
+
| 175 | bert.encoder.layer.10.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
68 |
+
| 177 | bert.encoder.layer.10.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|
69 |
+
| 181 | bert.encoder.layer.11.attention.self.query | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
70 |
+
| 183 | bert.encoder.layer.11.attention.self.key | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
71 |
+
| 185 | bert.encoder.layer.11.attention.self.value | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
72 |
+
| 187 | bert.encoder.layer.11.attention.output.dense | Linear | weight | [768, 768] | 589824 | 589824 | 0 |
|
73 |
+
| 191 | bert.encoder.layer.11.intermediate.dense | Linear | weight | [3072, 768] | 2359296 | 2359296 | 0 |
|
74 |
+
| 193 | bert.encoder.layer.11.output.dense | Linear | weight | [768, 3072] | 2359296 | 2359296 | 0 |
|