Chua, Vui Seng
commited on
Commit
•
41d4286
1
Parent(s):
054669e
Add collaterals
Browse files- .gitattributes +6 -0
- README.md +112 -0
- XP_layer_wise_sparsity_global_rate_26.51.csv +200 -0
- XP_layer_wise_sparsity_global_rate_26.51.md +201 -0
- XP_linear_layer_sparsity_20M_params_57.92_sparsity.csv +73 -0
- XP_linear_layer_sparsity_20M_params_57.92_sparsity.md +74 -0
- all_results.json +5 -0
- bert-base-squadv1-block-pruning-hybrid-filled-lt-nncf-57.92sparse-lt.onnx +3 -0
- checkpoint-20000/config.json +115 -0
- checkpoint-20000/optimizer.pt +3 -0
- checkpoint-20000/pytorch_model.bin +3 -0
- checkpoint-20000/rng_state.pth +3 -0
- checkpoint-20000/scheduler.pt +3 -0
- checkpoint-20000/special_tokens_map.json +1 -0
- checkpoint-20000/tokenizer.json +0 -0
- checkpoint-20000/tokenizer_config.json +1 -0
- checkpoint-20000/trainer_state.json +3 -0
- checkpoint-20000/training_args.bin +3 -0
- checkpoint-20000/vocab.txt +0 -0
- compressed_graph.dot +0 -0
- config.json +115 -0
- eval_XP_results.json +5 -0
- eval_nbest_predictions.json +3 -0
- eval_predictions.json +0 -0
- nncf_bert_squad_sparsity.json +72 -0
- original_graph.dot +0 -0
- pytorch_model.bin +3 -0
- special_tokens_map.json +1 -0
- tokenizer.json +0 -0
- tokenizer_config.json +1 -0
- train_results.json +8 -0
- trainer_state.json +3 -0
- training_args.bin +3 -0
- vocab.txt +0 -0
.gitattributes
CHANGED
@@ -25,3 +25,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
|
29 |
+
trainer_state.json filter=lfs diff=lfs merge=lfs -text
|
30 |
+
checkpoint-20000/pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
|
31 |
+
checkpoint-20000/optimizer.pt filter=lfs diff=lfs merge=lfs -text
|
32 |
+
bert-base-squadv1-block-pruning-hybrid-filled-lt-nncf-57.92sparse-lt.onnx filter=lfs diff=lfs merge=lfs -text
|
33 |
+
eval_nbest_predictions.json filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
This model is a downstream optimization of [```vuiseng9/bert-base-squadv1-block-pruning-hybrid-filled-lt```](https://huggingface.co/vuiseng9/bert-base-squadv1-block-pruning-hybrid-filled-lt) using [OpenVINO/NNCF](https://github.com/openvinotoolkit/nncf). Applied optimization includes:
|
2 |
+
1. magnitude sparsification at 57.92% upon initialization so that sparsity over all linear layers of bert-base is at 90%. Parameters are ranked globally via thier absolute norm. Only linear layers of self-attention and ffnn are targeted.
|
3 |
+
2. Custom distillation with large model ```bert-large-uncased-whole-word-masking-finetuned-squad```
|
4 |
+
|
5 |
+
```
|
6 |
+
eval_exact_match = 80.4447
|
7 |
+
eval_f1 = 87.7678
|
8 |
+
eval_samples = 10784
|
9 |
+
```
|
10 |
+
|
11 |
+
# Setup
|
12 |
+
```bash
|
13 |
+
# OpenVINO/NNCF
|
14 |
+
git clone https://github.com/vuiseng9/nncf && cd nncf
|
15 |
+
git checkout tld-poc
|
16 |
+
git reset --hard 1dec7afe7a4b567c059fcf287ea2c234980fded2
|
17 |
+
python setup.py develop
|
18 |
+
pip install -r examples/torch/requirements.txt
|
19 |
+
|
20 |
+
# Huggingface nn_pruning
|
21 |
+
git clone https://github.com/vuiseng9/nn_pruning && cd nn_pruning
|
22 |
+
git checkout reproduce-evaluation
|
23 |
+
git reset --hard 2d4e196d694c465e43e5fbce6c3836d0a60e1446
|
24 |
+
pip install -e ".[dev]"
|
25 |
+
|
26 |
+
# Huggingface Transformers
|
27 |
+
git clone https://github.com/vuiseng9/transformers && cd transformers
|
28 |
+
git checkout tld-poc
|
29 |
+
git reset --hard 10a1e29d84484e48fd106f58957d9ffc89dc43c5
|
30 |
+
pip install -e .
|
31 |
+
head -n 1 examples/pytorch/question-answering/requirements.txt | xargs -i pip install {}
|
32 |
+
|
33 |
+
# Additional dependencies
|
34 |
+
pip install onnx
|
35 |
+
```
|
36 |
+
|
37 |
+
# Train
|
38 |
+
|
39 |
+
```bash
|
40 |
+
git clone https://huggingface.co/vuiseng9/bert-base-squadv1-block-pruning-hybrid-filled-lt
|
41 |
+
BASE_MODEL=/path/to/cloned_repo_above #to-revise
|
42 |
+
|
43 |
+
wget https://huggingface.co/vuiseng9/bert-base-squadv1-block-pruning-hybrid-filled-lt-nncf-57.92sparse-lt/raw/main/nncf_bert_squad_sparsity.json
|
44 |
+
NNCF_CFG=/path/to/downloaded_nncf_cfg_above #to-revise
|
45 |
+
|
46 |
+
OUTROOT=/path/to/train_output_root #to-revise
|
47 |
+
WORKDIR=transformers/examples/pytorch/question-answering #to-revise
|
48 |
+
RUNID=bert-base-squadv1-block-pruning-hybrid-filled-lt-nncf-57.92sparse-lt
|
49 |
+
|
50 |
+
cd $WORKDIR
|
51 |
+
|
52 |
+
OUTDIR=$OUTROOT/$RUNID
|
53 |
+
mkdir -p $OUTDIR
|
54 |
+
|
55 |
+
export CUDA_VISIBLE_DEVICES=0
|
56 |
+
NEPOCH=5
|
57 |
+
|
58 |
+
python run_qa.py \
|
59 |
+
--model_name_or_path vuiseng9/bert-base-squadv1-block-pruning-hybrid \
|
60 |
+
--optimize_model_before_eval \
|
61 |
+
--optimized_checkpoint $BASE_MODEL \
|
62 |
+
--dataset_name squad \
|
63 |
+
--do_eval \
|
64 |
+
--do_train \
|
65 |
+
--evaluation_strategy steps \
|
66 |
+
--eval_steps 250 \
|
67 |
+
--learning_rate 3e-5 \
|
68 |
+
--lr_scheduler_type cosine_with_restarts \
|
69 |
+
--warmup_ratio 0.25 \
|
70 |
+
--cosine_cycles 1 \
|
71 |
+
--teacher bert-large-uncased-whole-word-masking-finetuned-squad \
|
72 |
+
--teacher_ratio 0.9 \
|
73 |
+
--num_train_epochs $NEPOCH \
|
74 |
+
--per_device_eval_batch_size 128 \
|
75 |
+
--per_device_train_batch_size 16 \
|
76 |
+
--max_seq_length 384 \
|
77 |
+
--doc_stride 128 \
|
78 |
+
--save_steps 250 \
|
79 |
+
--nncf_config $NNCF_CFG \
|
80 |
+
--logging_steps 1 \
|
81 |
+
--overwrite_output_dir \
|
82 |
+
--run_name $RUNID \
|
83 |
+
--output_dir $OUTDIR
|
84 |
+
```
|
85 |
+
|
86 |
+
# Eval
|
87 |
+
This repo must be cloned locally.
|
88 |
+
```bash
|
89 |
+
git clone https://huggingface.co/vuiseng9/bert-base-squadv1-block-pruning-hybrid-filled-lt-nncf-57.92sparse-lt
|
90 |
+
MODELROOT=/path/to/cloned_repo_above #to-revise
|
91 |
+
|
92 |
+
export CUDA_VISIBLE_DEVICES=0
|
93 |
+
|
94 |
+
OUTDIR=eval-bert-base-squadv1-block-pruning-hybrid-filled-lt-nncf-57.92sparse-lt
|
95 |
+
WORKDIR=transformers/examples/pytorch/question-answering #to-revise
|
96 |
+
cd $WORKDIR
|
97 |
+
mkdir $OUTDIR
|
98 |
+
|
99 |
+
nohup python run_qa.py \
|
100 |
+
--model_name_or_path vuiseng9/bert-base-squadv1-block-pruning-hybrid \
|
101 |
+
--dataset_name squad \
|
102 |
+
--optimize_model_before_eval \
|
103 |
+
--qat_checkpoint $MODELROOT/checkpoint-20000 \
|
104 |
+
--nncf_config $MODELROOT/nncf_bert_squad_sparsity.json \
|
105 |
+
--to_onnx $OUTDIR/bert-base-squadv1-block-pruning-hybrid-filled-lt-nncf-57.92sparse-lt.onnx \
|
106 |
+
--do_eval \
|
107 |
+
--per_device_eval_batch_size 128 \
|
108 |
+
--max_seq_length 384 \
|
109 |
+
--doc_stride 128 \
|
110 |
+
--overwrite_output_dir \
|
111 |
+
--output_dir $OUTDIR 2>&1 | tee $OUTDIR/run.log &
|
112 |
+
```
|
XP_layer_wise_sparsity_global_rate_26.51.csv
ADDED
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
,layer_id,layer_type,param_type,shape,nparam,nnz,sparsity
|
2 |
+
0,nncf_module.bert.embeddings.word_embeddings,NNCFEmbedding,weight,"[30522, 768]",23440896,23440896,0.0
|
3 |
+
1,nncf_module.bert.embeddings.position_embeddings,NNCFEmbedding,weight,"[512, 768]",393216,393216,0.0
|
4 |
+
2,nncf_module.bert.embeddings.token_type_embeddings,NNCFEmbedding,weight,"[2, 768]",1536,1536,0.0
|
5 |
+
3,nncf_module.bert.embeddings.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
6 |
+
4,nncf_module.bert.embeddings.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
7 |
+
5,nncf_module.bert.encoder.layer.0.attention.self.query,NNCFLinear,weight,"[320, 768]",245760,93583,0.6192097663879395
|
8 |
+
6,nncf_module.bert.encoder.layer.0.attention.self.query,NNCFLinear,bias,[320],320,320,0.0
|
9 |
+
7,nncf_module.bert.encoder.layer.0.attention.self.key,NNCFLinear,weight,"[320, 768]",245760,98270,0.6001383066177368
|
10 |
+
8,nncf_module.bert.encoder.layer.0.attention.self.key,NNCFLinear,bias,[320],320,320,0.0
|
11 |
+
9,nncf_module.bert.encoder.layer.0.attention.self.value,NNCFLinear,weight,"[320, 768]",245760,113605,0.5377400517463684
|
12 |
+
10,nncf_module.bert.encoder.layer.0.attention.self.value,NNCFLinear,bias,[320],320,320,0.0
|
13 |
+
11,nncf_module.bert.encoder.layer.0.attention.output.dense,NNCFLinear,weight,"[768, 320]",245760,117208,0.5230793952941895
|
14 |
+
12,nncf_module.bert.encoder.layer.0.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
15 |
+
13,nncf_module.bert.encoder.layer.0.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
16 |
+
14,nncf_module.bert.encoder.layer.0.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
17 |
+
15,nncf_module.bert.encoder.layer.0.intermediate.dense,NNCFLinear,weight,"[185, 768]",142080,97073,0.3167722225189209
|
18 |
+
16,nncf_module.bert.encoder.layer.0.intermediate.dense,NNCFLinear,bias,[185],185,185,0.0
|
19 |
+
17,nncf_module.bert.encoder.layer.0.output.dense,NNCFLinear,weight,"[768, 185]",142080,94692,0.33353036642074585
|
20 |
+
18,nncf_module.bert.encoder.layer.0.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
21 |
+
19,nncf_module.bert.encoder.layer.0.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
22 |
+
20,nncf_module.bert.encoder.layer.0.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
23 |
+
21,nncf_module.bert.encoder.layer.1.attention.self.query,NNCFLinear,weight,"[320, 768]",245760,118436,0.5180826187133789
|
24 |
+
22,nncf_module.bert.encoder.layer.1.attention.self.query,NNCFLinear,bias,[320],320,320,0.0
|
25 |
+
23,nncf_module.bert.encoder.layer.1.attention.self.key,NNCFLinear,weight,"[320, 768]",245760,118116,0.5193847417831421
|
26 |
+
24,nncf_module.bert.encoder.layer.1.attention.self.key,NNCFLinear,bias,[320],320,320,0.0
|
27 |
+
25,nncf_module.bert.encoder.layer.1.attention.self.value,NNCFLinear,weight,"[320, 768]",245760,107511,0.5625365972518921
|
28 |
+
26,nncf_module.bert.encoder.layer.1.attention.self.value,NNCFLinear,bias,[320],320,320,0.0
|
29 |
+
27,nncf_module.bert.encoder.layer.1.attention.output.dense,NNCFLinear,weight,"[768, 320]",245760,111189,0.5475707650184631
|
30 |
+
28,nncf_module.bert.encoder.layer.1.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
31 |
+
29,nncf_module.bert.encoder.layer.1.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
32 |
+
30,nncf_module.bert.encoder.layer.1.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
33 |
+
31,nncf_module.bert.encoder.layer.1.intermediate.dense,NNCFLinear,weight,"[315, 768]",241920,148783,0.3849908709526062
|
34 |
+
32,nncf_module.bert.encoder.layer.1.intermediate.dense,NNCFLinear,bias,[315],315,315,0.0
|
35 |
+
33,nncf_module.bert.encoder.layer.1.output.dense,NNCFLinear,weight,"[768, 315]",241920,143166,0.40820932388305664
|
36 |
+
34,nncf_module.bert.encoder.layer.1.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
37 |
+
35,nncf_module.bert.encoder.layer.1.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
38 |
+
36,nncf_module.bert.encoder.layer.1.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
39 |
+
37,nncf_module.bert.encoder.layer.2.attention.self.query,NNCFLinear,weight,"[576, 768]",442368,162735,0.6321275234222412
|
40 |
+
38,nncf_module.bert.encoder.layer.2.attention.self.query,NNCFLinear,bias,[576],576,576,0.0
|
41 |
+
39,nncf_module.bert.encoder.layer.2.attention.self.key,NNCFLinear,weight,"[576, 768]",442368,164795,0.6274707913398743
|
42 |
+
40,nncf_module.bert.encoder.layer.2.attention.self.key,NNCFLinear,bias,[576],576,576,0.0
|
43 |
+
41,nncf_module.bert.encoder.layer.2.attention.self.value,NNCFLinear,weight,"[576, 768]",442368,135670,0.6933096647262573
|
44 |
+
42,nncf_module.bert.encoder.layer.2.attention.self.value,NNCFLinear,bias,[576],576,576,0.0
|
45 |
+
43,nncf_module.bert.encoder.layer.2.attention.output.dense,NNCFLinear,weight,"[768, 576]",442368,138445,0.6870365738868713
|
46 |
+
44,nncf_module.bert.encoder.layer.2.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
47 |
+
45,nncf_module.bert.encoder.layer.2.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
48 |
+
46,nncf_module.bert.encoder.layer.2.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
49 |
+
47,nncf_module.bert.encoder.layer.2.intermediate.dense,NNCFLinear,weight,"[339, 768]",260352,154035,0.408358633518219
|
50 |
+
48,nncf_module.bert.encoder.layer.2.intermediate.dense,NNCFLinear,bias,[339],339,339,0.0
|
51 |
+
49,nncf_module.bert.encoder.layer.2.output.dense,NNCFLinear,weight,"[768, 339]",260352,150816,0.4207226634025574
|
52 |
+
50,nncf_module.bert.encoder.layer.2.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
53 |
+
51,nncf_module.bert.encoder.layer.2.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
54 |
+
52,nncf_module.bert.encoder.layer.2.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
55 |
+
53,nncf_module.bert.encoder.layer.3.attention.self.query,NNCFLinear,weight,"[576, 768]",442368,170623,0.6142961978912354
|
56 |
+
54,nncf_module.bert.encoder.layer.3.attention.self.query,NNCFLinear,bias,[576],576,576,0.0
|
57 |
+
55,nncf_module.bert.encoder.layer.3.attention.self.key,NNCFLinear,weight,"[576, 768]",442368,178401,0.5967136025428772
|
58 |
+
56,nncf_module.bert.encoder.layer.3.attention.self.key,NNCFLinear,bias,[576],576,576,0.0
|
59 |
+
57,nncf_module.bert.encoder.layer.3.attention.self.value,NNCFLinear,weight,"[576, 768]",442368,171905,0.6113982200622559
|
60 |
+
58,nncf_module.bert.encoder.layer.3.attention.self.value,NNCFLinear,bias,[576],576,576,0.0
|
61 |
+
59,nncf_module.bert.encoder.layer.3.attention.output.dense,NNCFLinear,weight,"[768, 576]",442368,169172,0.6175763010978699
|
62 |
+
60,nncf_module.bert.encoder.layer.3.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
63 |
+
61,nncf_module.bert.encoder.layer.3.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
64 |
+
62,nncf_module.bert.encoder.layer.3.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
65 |
+
63,nncf_module.bert.encoder.layer.3.intermediate.dense,NNCFLinear,weight,"[368, 768]",282624,163163,0.42268526554107666
|
66 |
+
64,nncf_module.bert.encoder.layer.3.intermediate.dense,NNCFLinear,bias,[368],368,368,0.0
|
67 |
+
65,nncf_module.bert.encoder.layer.3.output.dense,NNCFLinear,weight,"[768, 368]",282624,157506,0.44270122051239014
|
68 |
+
66,nncf_module.bert.encoder.layer.3.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
69 |
+
67,nncf_module.bert.encoder.layer.3.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
70 |
+
68,nncf_module.bert.encoder.layer.3.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
71 |
+
69,nncf_module.bert.encoder.layer.4.attention.self.query,NNCFLinear,weight,"[576, 768]",442368,175772,0.6026566028594971
|
72 |
+
70,nncf_module.bert.encoder.layer.4.attention.self.query,NNCFLinear,bias,[576],576,576,0.0
|
73 |
+
71,nncf_module.bert.encoder.layer.4.attention.self.key,NNCFLinear,weight,"[576, 768]",442368,177087,0.5996840000152588
|
74 |
+
72,nncf_module.bert.encoder.layer.4.attention.self.key,NNCFLinear,bias,[576],576,576,0.0
|
75 |
+
73,nncf_module.bert.encoder.layer.4.attention.self.value,NNCFLinear,weight,"[576, 768]",442368,163996,0.6292769908905029
|
76 |
+
74,nncf_module.bert.encoder.layer.4.attention.self.value,NNCFLinear,bias,[576],576,576,0.0
|
77 |
+
75,nncf_module.bert.encoder.layer.4.attention.output.dense,NNCFLinear,weight,"[768, 576]",442368,159335,0.6398134231567383
|
78 |
+
76,nncf_module.bert.encoder.layer.4.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
79 |
+
77,nncf_module.bert.encoder.layer.4.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
80 |
+
78,nncf_module.bert.encoder.layer.4.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
81 |
+
79,nncf_module.bert.encoder.layer.4.intermediate.dense,NNCFLinear,weight,"[386, 768]",296448,167726,0.43421441316604614
|
82 |
+
80,nncf_module.bert.encoder.layer.4.intermediate.dense,NNCFLinear,bias,[386],386,386,0.0
|
83 |
+
81,nncf_module.bert.encoder.layer.4.output.dense,NNCFLinear,weight,"[768, 386]",296448,159865,0.46073174476623535
|
84 |
+
82,nncf_module.bert.encoder.layer.4.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
85 |
+
83,nncf_module.bert.encoder.layer.4.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
86 |
+
84,nncf_module.bert.encoder.layer.4.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
87 |
+
85,nncf_module.bert.encoder.layer.5.attention.self.query,NNCFLinear,weight,"[384, 768]",294912,114186,0.6128133535385132
|
88 |
+
86,nncf_module.bert.encoder.layer.5.attention.self.query,NNCFLinear,bias,[384],384,384,0.0
|
89 |
+
87,nncf_module.bert.encoder.layer.5.attention.self.key,NNCFLinear,weight,"[384, 768]",294912,132782,0.5497572422027588
|
90 |
+
88,nncf_module.bert.encoder.layer.5.attention.self.key,NNCFLinear,bias,[384],384,384,0.0
|
91 |
+
89,nncf_module.bert.encoder.layer.5.attention.self.value,NNCFLinear,weight,"[384, 768]",294912,134830,0.5428127646446228
|
92 |
+
90,nncf_module.bert.encoder.layer.5.attention.self.value,NNCFLinear,bias,[384],384,384,0.0
|
93 |
+
91,nncf_module.bert.encoder.layer.5.attention.output.dense,NNCFLinear,weight,"[768, 384]",294912,131941,0.5526089072227478
|
94 |
+
92,nncf_module.bert.encoder.layer.5.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
95 |
+
93,nncf_module.bert.encoder.layer.5.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
96 |
+
94,nncf_module.bert.encoder.layer.5.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
97 |
+
95,nncf_module.bert.encoder.layer.5.intermediate.dense,NNCFLinear,weight,"[336, 768]",258048,153916,0.4035372734069824
|
98 |
+
96,nncf_module.bert.encoder.layer.5.intermediate.dense,NNCFLinear,bias,[336],336,336,0.0
|
99 |
+
97,nncf_module.bert.encoder.layer.5.output.dense,NNCFLinear,weight,"[768, 336]",258048,145794,0.4350120425224304
|
100 |
+
98,nncf_module.bert.encoder.layer.5.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
101 |
+
99,nncf_module.bert.encoder.layer.5.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
102 |
+
100,nncf_module.bert.encoder.layer.5.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
103 |
+
101,nncf_module.bert.encoder.layer.6.attention.self.query,NNCFLinear,weight,"[448, 768]",344064,131878,0.616705060005188
|
104 |
+
102,nncf_module.bert.encoder.layer.6.attention.self.query,NNCFLinear,bias,[448],448,448,0.0
|
105 |
+
103,nncf_module.bert.encoder.layer.6.attention.self.key,NNCFLinear,weight,"[448, 768]",344064,144502,0.580014169216156
|
106 |
+
104,nncf_module.bert.encoder.layer.6.attention.self.key,NNCFLinear,bias,[448],448,448,0.0
|
107 |
+
105,nncf_module.bert.encoder.layer.6.attention.self.value,NNCFLinear,weight,"[448, 768]",344064,130911,0.6195155382156372
|
108 |
+
106,nncf_module.bert.encoder.layer.6.attention.self.value,NNCFLinear,bias,[448],448,448,0.0
|
109 |
+
107,nncf_module.bert.encoder.layer.6.attention.output.dense,NNCFLinear,weight,"[768, 448]",344064,125928,0.6339982748031616
|
110 |
+
108,nncf_module.bert.encoder.layer.6.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
111 |
+
109,nncf_module.bert.encoder.layer.6.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
112 |
+
110,nncf_module.bert.encoder.layer.6.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
113 |
+
111,nncf_module.bert.encoder.layer.6.intermediate.dense,NNCFLinear,weight,"[280, 768]",215040,135283,0.37089377641677856
|
114 |
+
112,nncf_module.bert.encoder.layer.6.intermediate.dense,NNCFLinear,bias,[280],280,280,0.0
|
115 |
+
113,nncf_module.bert.encoder.layer.6.output.dense,NNCFLinear,weight,"[768, 280]",215040,131619,0.3879324793815613
|
116 |
+
114,nncf_module.bert.encoder.layer.6.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
117 |
+
115,nncf_module.bert.encoder.layer.6.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
118 |
+
116,nncf_module.bert.encoder.layer.6.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
119 |
+
117,nncf_module.bert.encoder.layer.7.attention.self.query,NNCFLinear,weight,"[448, 768]",344064,132120,0.6160016655921936
|
120 |
+
118,nncf_module.bert.encoder.layer.7.attention.self.query,NNCFLinear,bias,[448],448,448,0.0
|
121 |
+
119,nncf_module.bert.encoder.layer.7.attention.self.key,NNCFLinear,weight,"[448, 768]",344064,152223,0.5575735569000244
|
122 |
+
120,nncf_module.bert.encoder.layer.7.attention.self.key,NNCFLinear,bias,[448],448,448,0.0
|
123 |
+
121,nncf_module.bert.encoder.layer.7.attention.self.value,NNCFLinear,weight,"[448, 768]",344064,141066,0.5900006890296936
|
124 |
+
122,nncf_module.bert.encoder.layer.7.attention.self.value,NNCFLinear,bias,[448],448,448,0.0
|
125 |
+
123,nncf_module.bert.encoder.layer.7.attention.output.dense,NNCFLinear,weight,"[768, 448]",344064,135662,0.605707049369812
|
126 |
+
124,nncf_module.bert.encoder.layer.7.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
127 |
+
125,nncf_module.bert.encoder.layer.7.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
128 |
+
126,nncf_module.bert.encoder.layer.7.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
129 |
+
127,nncf_module.bert.encoder.layer.7.intermediate.dense,NNCFLinear,weight,"[211, 768]",162048,109590,0.3237189054489136
|
130 |
+
128,nncf_module.bert.encoder.layer.7.intermediate.dense,NNCFLinear,bias,[211],211,211,0.0
|
131 |
+
129,nncf_module.bert.encoder.layer.7.output.dense,NNCFLinear,weight,"[768, 211]",162048,107335,0.33763450384140015
|
132 |
+
130,nncf_module.bert.encoder.layer.7.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
133 |
+
131,nncf_module.bert.encoder.layer.7.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
134 |
+
132,nncf_module.bert.encoder.layer.7.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
135 |
+
133,nncf_module.bert.encoder.layer.8.attention.self.query,NNCFLinear,weight,"[448, 768]",344064,129148,0.624639630317688
|
136 |
+
134,nncf_module.bert.encoder.layer.8.attention.self.query,NNCFLinear,bias,[448],448,448,0.0
|
137 |
+
135,nncf_module.bert.encoder.layer.8.attention.self.key,NNCFLinear,weight,"[448, 768]",344064,130060,0.6219888925552368
|
138 |
+
136,nncf_module.bert.encoder.layer.8.attention.self.key,NNCFLinear,bias,[448],448,448,0.0
|
139 |
+
137,nncf_module.bert.encoder.layer.8.attention.self.value,NNCFLinear,weight,"[448, 768]",344064,108162,0.6856340765953064
|
140 |
+
138,nncf_module.bert.encoder.layer.8.attention.self.value,NNCFLinear,bias,[448],448,448,0.0
|
141 |
+
139,nncf_module.bert.encoder.layer.8.attention.output.dense,NNCFLinear,weight,"[768, 448]",344064,103447,0.699337899684906
|
142 |
+
140,nncf_module.bert.encoder.layer.8.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
143 |
+
141,nncf_module.bert.encoder.layer.8.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
144 |
+
142,nncf_module.bert.encoder.layer.8.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
145 |
+
143,nncf_module.bert.encoder.layer.8.intermediate.dense,NNCFLinear,weight,"[108, 768]",82944,63275,0.2371358871459961
|
146 |
+
144,nncf_module.bert.encoder.layer.8.intermediate.dense,NNCFLinear,bias,[108],108,108,0.0
|
147 |
+
145,nncf_module.bert.encoder.layer.8.output.dense,NNCFLinear,weight,"[768, 108]",82944,62725,0.24376684427261353
|
148 |
+
146,nncf_module.bert.encoder.layer.8.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
149 |
+
147,nncf_module.bert.encoder.layer.8.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
150 |
+
148,nncf_module.bert.encoder.layer.8.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
151 |
+
149,nncf_module.bert.encoder.layer.9.attention.self.query,NNCFLinear,weight,"[320, 768]",245760,107145,0.56402587890625
|
152 |
+
150,nncf_module.bert.encoder.layer.9.attention.self.query,NNCFLinear,bias,[320],320,320,0.0
|
153 |
+
151,nncf_module.bert.encoder.layer.9.attention.self.key,NNCFLinear,weight,"[320, 768]",245760,101811,0.5857299566268921
|
154 |
+
152,nncf_module.bert.encoder.layer.9.attention.self.key,NNCFLinear,bias,[320],320,320,0.0
|
155 |
+
153,nncf_module.bert.encoder.layer.9.attention.self.value,NNCFLinear,weight,"[320, 768]",245760,52182,0.7876708507537842
|
156 |
+
154,nncf_module.bert.encoder.layer.9.attention.self.value,NNCFLinear,bias,[320],320,320,0.0
|
157 |
+
155,nncf_module.bert.encoder.layer.9.attention.output.dense,NNCFLinear,weight,"[768, 320]",245760,53210,0.7834879159927368
|
158 |
+
156,nncf_module.bert.encoder.layer.9.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
159 |
+
157,nncf_module.bert.encoder.layer.9.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
160 |
+
158,nncf_module.bert.encoder.layer.9.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
161 |
+
159,nncf_module.bert.encoder.layer.9.intermediate.dense,NNCFLinear,weight,"[53, 768]",40704,33461,0.17794322967529297
|
162 |
+
160,nncf_module.bert.encoder.layer.9.intermediate.dense,NNCFLinear,bias,[53],53,53,0.0
|
163 |
+
161,nncf_module.bert.encoder.layer.9.output.dense,NNCFLinear,weight,"[768, 53]",40704,32551,0.20029973983764648
|
164 |
+
162,nncf_module.bert.encoder.layer.9.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
165 |
+
163,nncf_module.bert.encoder.layer.9.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
166 |
+
164,nncf_module.bert.encoder.layer.9.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
167 |
+
165,nncf_module.bert.encoder.layer.10.attention.self.query,NNCFLinear,weight,"[384, 768]",294912,112430,0.6187676191329956
|
168 |
+
166,nncf_module.bert.encoder.layer.10.attention.self.query,NNCFLinear,bias,[384],384,384,0.0
|
169 |
+
167,nncf_module.bert.encoder.layer.10.attention.self.key,NNCFLinear,weight,"[384, 768]",294912,109594,0.6283840537071228
|
170 |
+
168,nncf_module.bert.encoder.layer.10.attention.self.key,NNCFLinear,bias,[384],384,384,0.0
|
171 |
+
169,nncf_module.bert.encoder.layer.10.attention.self.value,NNCFLinear,weight,"[384, 768]",294912,61774,0.7905341386795044
|
172 |
+
170,nncf_module.bert.encoder.layer.10.attention.self.value,NNCFLinear,bias,[384],384,384,0.0
|
173 |
+
171,nncf_module.bert.encoder.layer.10.attention.output.dense,NNCFLinear,weight,"[768, 384]",294912,64183,0.7823655605316162
|
174 |
+
172,nncf_module.bert.encoder.layer.10.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
175 |
+
173,nncf_module.bert.encoder.layer.10.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
176 |
+
174,nncf_module.bert.encoder.layer.10.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
177 |
+
175,nncf_module.bert.encoder.layer.10.intermediate.dense,NNCFLinear,weight,"[86, 768]",66048,50455,0.2360858917236328
|
178 |
+
176,nncf_module.bert.encoder.layer.10.intermediate.dense,NNCFLinear,bias,[86],86,86,0.0
|
179 |
+
177,nncf_module.bert.encoder.layer.10.output.dense,NNCFLinear,weight,"[768, 86]",66048,49741,0.24689620733261108
|
180 |
+
178,nncf_module.bert.encoder.layer.10.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
181 |
+
179,nncf_module.bert.encoder.layer.10.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
182 |
+
180,nncf_module.bert.encoder.layer.10.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
183 |
+
181,nncf_module.bert.encoder.layer.11.attention.self.query,NNCFLinear,weight,"[384, 768]",294912,88129,0.7011684775352478
|
184 |
+
182,nncf_module.bert.encoder.layer.11.attention.self.query,NNCFLinear,bias,[384],384,384,0.0
|
185 |
+
183,nncf_module.bert.encoder.layer.11.attention.self.key,NNCFLinear,weight,"[384, 768]",294912,85288,0.7108018398284912
|
186 |
+
184,nncf_module.bert.encoder.layer.11.attention.self.key,NNCFLinear,bias,[384],384,384,0.0
|
187 |
+
185,nncf_module.bert.encoder.layer.11.attention.self.value,NNCFLinear,weight,"[384, 768]",294912,47258,0.8397555947303772
|
188 |
+
186,nncf_module.bert.encoder.layer.11.attention.self.value,NNCFLinear,bias,[384],384,384,0.0
|
189 |
+
187,nncf_module.bert.encoder.layer.11.attention.output.dense,NNCFLinear,weight,"[768, 384]",294912,49311,0.832794189453125
|
190 |
+
188,nncf_module.bert.encoder.layer.11.attention.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
191 |
+
189,nncf_module.bert.encoder.layer.11.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
192 |
+
190,nncf_module.bert.encoder.layer.11.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
193 |
+
191,nncf_module.bert.encoder.layer.11.intermediate.dense,NNCFLinear,weight,"[105, 768]",80640,62254,0.22800099849700928
|
194 |
+
192,nncf_module.bert.encoder.layer.11.intermediate.dense,NNCFLinear,bias,[105],105,105,0.0
|
195 |
+
193,nncf_module.bert.encoder.layer.11.output.dense,NNCFLinear,weight,"[768, 105]",80640,61669,0.2352554202079773
|
196 |
+
194,nncf_module.bert.encoder.layer.11.output.dense,NNCFLinear,bias,[768],768,768,0.0
|
197 |
+
195,nncf_module.bert.encoder.layer.11.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
|
198 |
+
196,nncf_module.bert.encoder.layer.11.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
|
199 |
+
197,nncf_module.qa_outputs,NNCFLinear,weight,"[2, 768]",1536,1536,0.0
|
200 |
+
198,nncf_module.qa_outputs,NNCFLinear,bias,[2],2,2,0.0
|
XP_layer_wise_sparsity_global_rate_26.51.md
ADDED
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
| | layer_id | layer_type | param_type | shape | nparam | nnz | sparsity |
|
2 |
+
|----:|:-------------------------------------------------------------|:--------------|:-------------|:-------------|---------:|---------:|-----------:|
|
3 |
+
| 0 | nncf_module.bert.embeddings.word_embeddings | NNCFEmbedding | weight | [30522, 768] | 23440896 | 23440896 | 0 |
|
4 |
+
| 1 | nncf_module.bert.embeddings.position_embeddings | NNCFEmbedding | weight | [512, 768] | 393216 | 393216 | 0 |
|
5 |
+
| 2 | nncf_module.bert.embeddings.token_type_embeddings | NNCFEmbedding | weight | [2, 768] | 1536 | 1536 | 0 |
|
6 |
+
| 3 | nncf_module.bert.embeddings.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
7 |
+
| 4 | nncf_module.bert.embeddings.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
8 |
+
| 5 | nncf_module.bert.encoder.layer.0.attention.self.query | NNCFLinear | weight | [320, 768] | 245760 | 93583 | 0.61921 |
|
9 |
+
| 6 | nncf_module.bert.encoder.layer.0.attention.self.query | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
10 |
+
| 7 | nncf_module.bert.encoder.layer.0.attention.self.key | NNCFLinear | weight | [320, 768] | 245760 | 98270 | 0.600138 |
|
11 |
+
| 8 | nncf_module.bert.encoder.layer.0.attention.self.key | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
12 |
+
| 9 | nncf_module.bert.encoder.layer.0.attention.self.value | NNCFLinear | weight | [320, 768] | 245760 | 113605 | 0.53774 |
|
13 |
+
| 10 | nncf_module.bert.encoder.layer.0.attention.self.value | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
14 |
+
| 11 | nncf_module.bert.encoder.layer.0.attention.output.dense | NNCFLinear | weight | [768, 320] | 245760 | 117208 | 0.523079 |
|
15 |
+
| 12 | nncf_module.bert.encoder.layer.0.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
16 |
+
| 13 | nncf_module.bert.encoder.layer.0.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
17 |
+
| 14 | nncf_module.bert.encoder.layer.0.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
18 |
+
| 15 | nncf_module.bert.encoder.layer.0.intermediate.dense | NNCFLinear | weight | [185, 768] | 142080 | 97073 | 0.316772 |
|
19 |
+
| 16 | nncf_module.bert.encoder.layer.0.intermediate.dense | NNCFLinear | bias | [185] | 185 | 185 | 0 |
|
20 |
+
| 17 | nncf_module.bert.encoder.layer.0.output.dense | NNCFLinear | weight | [768, 185] | 142080 | 94692 | 0.33353 |
|
21 |
+
| 18 | nncf_module.bert.encoder.layer.0.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
22 |
+
| 19 | nncf_module.bert.encoder.layer.0.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
23 |
+
| 20 | nncf_module.bert.encoder.layer.0.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
24 |
+
| 21 | nncf_module.bert.encoder.layer.1.attention.self.query | NNCFLinear | weight | [320, 768] | 245760 | 118436 | 0.518083 |
|
25 |
+
| 22 | nncf_module.bert.encoder.layer.1.attention.self.query | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
26 |
+
| 23 | nncf_module.bert.encoder.layer.1.attention.self.key | NNCFLinear | weight | [320, 768] | 245760 | 118116 | 0.519385 |
|
27 |
+
| 24 | nncf_module.bert.encoder.layer.1.attention.self.key | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
28 |
+
| 25 | nncf_module.bert.encoder.layer.1.attention.self.value | NNCFLinear | weight | [320, 768] | 245760 | 107511 | 0.562537 |
|
29 |
+
| 26 | nncf_module.bert.encoder.layer.1.attention.self.value | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
30 |
+
| 27 | nncf_module.bert.encoder.layer.1.attention.output.dense | NNCFLinear | weight | [768, 320] | 245760 | 111189 | 0.547571 |
|
31 |
+
| 28 | nncf_module.bert.encoder.layer.1.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
32 |
+
| 29 | nncf_module.bert.encoder.layer.1.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
33 |
+
| 30 | nncf_module.bert.encoder.layer.1.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
34 |
+
| 31 | nncf_module.bert.encoder.layer.1.intermediate.dense | NNCFLinear | weight | [315, 768] | 241920 | 148783 | 0.384991 |
|
35 |
+
| 32 | nncf_module.bert.encoder.layer.1.intermediate.dense | NNCFLinear | bias | [315] | 315 | 315 | 0 |
|
36 |
+
| 33 | nncf_module.bert.encoder.layer.1.output.dense | NNCFLinear | weight | [768, 315] | 241920 | 143166 | 0.408209 |
|
37 |
+
| 34 | nncf_module.bert.encoder.layer.1.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
38 |
+
| 35 | nncf_module.bert.encoder.layer.1.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
39 |
+
| 36 | nncf_module.bert.encoder.layer.1.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
40 |
+
| 37 | nncf_module.bert.encoder.layer.2.attention.self.query | NNCFLinear | weight | [576, 768] | 442368 | 162735 | 0.632128 |
|
41 |
+
| 38 | nncf_module.bert.encoder.layer.2.attention.self.query | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
42 |
+
| 39 | nncf_module.bert.encoder.layer.2.attention.self.key | NNCFLinear | weight | [576, 768] | 442368 | 164795 | 0.627471 |
|
43 |
+
| 40 | nncf_module.bert.encoder.layer.2.attention.self.key | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
44 |
+
| 41 | nncf_module.bert.encoder.layer.2.attention.self.value | NNCFLinear | weight | [576, 768] | 442368 | 135670 | 0.69331 |
|
45 |
+
| 42 | nncf_module.bert.encoder.layer.2.attention.self.value | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
46 |
+
| 43 | nncf_module.bert.encoder.layer.2.attention.output.dense | NNCFLinear | weight | [768, 576] | 442368 | 138445 | 0.687037 |
|
47 |
+
| 44 | nncf_module.bert.encoder.layer.2.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
48 |
+
| 45 | nncf_module.bert.encoder.layer.2.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
49 |
+
| 46 | nncf_module.bert.encoder.layer.2.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
50 |
+
| 47 | nncf_module.bert.encoder.layer.2.intermediate.dense | NNCFLinear | weight | [339, 768] | 260352 | 154035 | 0.408359 |
|
51 |
+
| 48 | nncf_module.bert.encoder.layer.2.intermediate.dense | NNCFLinear | bias | [339] | 339 | 339 | 0 |
|
52 |
+
| 49 | nncf_module.bert.encoder.layer.2.output.dense | NNCFLinear | weight | [768, 339] | 260352 | 150816 | 0.420723 |
|
53 |
+
| 50 | nncf_module.bert.encoder.layer.2.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
54 |
+
| 51 | nncf_module.bert.encoder.layer.2.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
55 |
+
| 52 | nncf_module.bert.encoder.layer.2.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
56 |
+
| 53 | nncf_module.bert.encoder.layer.3.attention.self.query | NNCFLinear | weight | [576, 768] | 442368 | 170623 | 0.614296 |
|
57 |
+
| 54 | nncf_module.bert.encoder.layer.3.attention.self.query | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
58 |
+
| 55 | nncf_module.bert.encoder.layer.3.attention.self.key | NNCFLinear | weight | [576, 768] | 442368 | 178401 | 0.596714 |
|
59 |
+
| 56 | nncf_module.bert.encoder.layer.3.attention.self.key | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
60 |
+
| 57 | nncf_module.bert.encoder.layer.3.attention.self.value | NNCFLinear | weight | [576, 768] | 442368 | 171905 | 0.611398 |
|
61 |
+
| 58 | nncf_module.bert.encoder.layer.3.attention.self.value | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
62 |
+
| 59 | nncf_module.bert.encoder.layer.3.attention.output.dense | NNCFLinear | weight | [768, 576] | 442368 | 169172 | 0.617576 |
|
63 |
+
| 60 | nncf_module.bert.encoder.layer.3.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
64 |
+
| 61 | nncf_module.bert.encoder.layer.3.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
65 |
+
| 62 | nncf_module.bert.encoder.layer.3.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
66 |
+
| 63 | nncf_module.bert.encoder.layer.3.intermediate.dense | NNCFLinear | weight | [368, 768] | 282624 | 163163 | 0.422685 |
|
67 |
+
| 64 | nncf_module.bert.encoder.layer.3.intermediate.dense | NNCFLinear | bias | [368] | 368 | 368 | 0 |
|
68 |
+
| 65 | nncf_module.bert.encoder.layer.3.output.dense | NNCFLinear | weight | [768, 368] | 282624 | 157506 | 0.442701 |
|
69 |
+
| 66 | nncf_module.bert.encoder.layer.3.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
70 |
+
| 67 | nncf_module.bert.encoder.layer.3.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
71 |
+
| 68 | nncf_module.bert.encoder.layer.3.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
72 |
+
| 69 | nncf_module.bert.encoder.layer.4.attention.self.query | NNCFLinear | weight | [576, 768] | 442368 | 175772 | 0.602657 |
|
73 |
+
| 70 | nncf_module.bert.encoder.layer.4.attention.self.query | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
74 |
+
| 71 | nncf_module.bert.encoder.layer.4.attention.self.key | NNCFLinear | weight | [576, 768] | 442368 | 177087 | 0.599684 |
|
75 |
+
| 72 | nncf_module.bert.encoder.layer.4.attention.self.key | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
76 |
+
| 73 | nncf_module.bert.encoder.layer.4.attention.self.value | NNCFLinear | weight | [576, 768] | 442368 | 163996 | 0.629277 |
|
77 |
+
| 74 | nncf_module.bert.encoder.layer.4.attention.self.value | NNCFLinear | bias | [576] | 576 | 576 | 0 |
|
78 |
+
| 75 | nncf_module.bert.encoder.layer.4.attention.output.dense | NNCFLinear | weight | [768, 576] | 442368 | 159335 | 0.639813 |
|
79 |
+
| 76 | nncf_module.bert.encoder.layer.4.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
80 |
+
| 77 | nncf_module.bert.encoder.layer.4.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
81 |
+
| 78 | nncf_module.bert.encoder.layer.4.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
82 |
+
| 79 | nncf_module.bert.encoder.layer.4.intermediate.dense | NNCFLinear | weight | [386, 768] | 296448 | 167726 | 0.434214 |
|
83 |
+
| 80 | nncf_module.bert.encoder.layer.4.intermediate.dense | NNCFLinear | bias | [386] | 386 | 386 | 0 |
|
84 |
+
| 81 | nncf_module.bert.encoder.layer.4.output.dense | NNCFLinear | weight | [768, 386] | 296448 | 159865 | 0.460732 |
|
85 |
+
| 82 | nncf_module.bert.encoder.layer.4.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
86 |
+
| 83 | nncf_module.bert.encoder.layer.4.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
87 |
+
| 84 | nncf_module.bert.encoder.layer.4.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
88 |
+
| 85 | nncf_module.bert.encoder.layer.5.attention.self.query | NNCFLinear | weight | [384, 768] | 294912 | 114186 | 0.612813 |
|
89 |
+
| 86 | nncf_module.bert.encoder.layer.5.attention.self.query | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
90 |
+
| 87 | nncf_module.bert.encoder.layer.5.attention.self.key | NNCFLinear | weight | [384, 768] | 294912 | 132782 | 0.549757 |
|
91 |
+
| 88 | nncf_module.bert.encoder.layer.5.attention.self.key | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
92 |
+
| 89 | nncf_module.bert.encoder.layer.5.attention.self.value | NNCFLinear | weight | [384, 768] | 294912 | 134830 | 0.542813 |
|
93 |
+
| 90 | nncf_module.bert.encoder.layer.5.attention.self.value | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
94 |
+
| 91 | nncf_module.bert.encoder.layer.5.attention.output.dense | NNCFLinear | weight | [768, 384] | 294912 | 131941 | 0.552609 |
|
95 |
+
| 92 | nncf_module.bert.encoder.layer.5.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
96 |
+
| 93 | nncf_module.bert.encoder.layer.5.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
97 |
+
| 94 | nncf_module.bert.encoder.layer.5.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
98 |
+
| 95 | nncf_module.bert.encoder.layer.5.intermediate.dense | NNCFLinear | weight | [336, 768] | 258048 | 153916 | 0.403537 |
|
99 |
+
| 96 | nncf_module.bert.encoder.layer.5.intermediate.dense | NNCFLinear | bias | [336] | 336 | 336 | 0 |
|
100 |
+
| 97 | nncf_module.bert.encoder.layer.5.output.dense | NNCFLinear | weight | [768, 336] | 258048 | 145794 | 0.435012 |
|
101 |
+
| 98 | nncf_module.bert.encoder.layer.5.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
102 |
+
| 99 | nncf_module.bert.encoder.layer.5.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
103 |
+
| 100 | nncf_module.bert.encoder.layer.5.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
104 |
+
| 101 | nncf_module.bert.encoder.layer.6.attention.self.query | NNCFLinear | weight | [448, 768] | 344064 | 131878 | 0.616705 |
|
105 |
+
| 102 | nncf_module.bert.encoder.layer.6.attention.self.query | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
106 |
+
| 103 | nncf_module.bert.encoder.layer.6.attention.self.key | NNCFLinear | weight | [448, 768] | 344064 | 144502 | 0.580014 |
|
107 |
+
| 104 | nncf_module.bert.encoder.layer.6.attention.self.key | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
108 |
+
| 105 | nncf_module.bert.encoder.layer.6.attention.self.value | NNCFLinear | weight | [448, 768] | 344064 | 130911 | 0.619516 |
|
109 |
+
| 106 | nncf_module.bert.encoder.layer.6.attention.self.value | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
110 |
+
| 107 | nncf_module.bert.encoder.layer.6.attention.output.dense | NNCFLinear | weight | [768, 448] | 344064 | 125928 | 0.633998 |
|
111 |
+
| 108 | nncf_module.bert.encoder.layer.6.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
112 |
+
| 109 | nncf_module.bert.encoder.layer.6.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
113 |
+
| 110 | nncf_module.bert.encoder.layer.6.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
114 |
+
| 111 | nncf_module.bert.encoder.layer.6.intermediate.dense | NNCFLinear | weight | [280, 768] | 215040 | 135283 | 0.370894 |
|
115 |
+
| 112 | nncf_module.bert.encoder.layer.6.intermediate.dense | NNCFLinear | bias | [280] | 280 | 280 | 0 |
|
116 |
+
| 113 | nncf_module.bert.encoder.layer.6.output.dense | NNCFLinear | weight | [768, 280] | 215040 | 131619 | 0.387932 |
|
117 |
+
| 114 | nncf_module.bert.encoder.layer.6.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
118 |
+
| 115 | nncf_module.bert.encoder.layer.6.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
119 |
+
| 116 | nncf_module.bert.encoder.layer.6.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
120 |
+
| 117 | nncf_module.bert.encoder.layer.7.attention.self.query | NNCFLinear | weight | [448, 768] | 344064 | 132120 | 0.616002 |
|
121 |
+
| 118 | nncf_module.bert.encoder.layer.7.attention.self.query | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
122 |
+
| 119 | nncf_module.bert.encoder.layer.7.attention.self.key | NNCFLinear | weight | [448, 768] | 344064 | 152223 | 0.557574 |
|
123 |
+
| 120 | nncf_module.bert.encoder.layer.7.attention.self.key | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
124 |
+
| 121 | nncf_module.bert.encoder.layer.7.attention.self.value | NNCFLinear | weight | [448, 768] | 344064 | 141066 | 0.590001 |
|
125 |
+
| 122 | nncf_module.bert.encoder.layer.7.attention.self.value | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
126 |
+
| 123 | nncf_module.bert.encoder.layer.7.attention.output.dense | NNCFLinear | weight | [768, 448] | 344064 | 135662 | 0.605707 |
|
127 |
+
| 124 | nncf_module.bert.encoder.layer.7.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
128 |
+
| 125 | nncf_module.bert.encoder.layer.7.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
129 |
+
| 126 | nncf_module.bert.encoder.layer.7.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
130 |
+
| 127 | nncf_module.bert.encoder.layer.7.intermediate.dense | NNCFLinear | weight | [211, 768] | 162048 | 109590 | 0.323719 |
|
131 |
+
| 128 | nncf_module.bert.encoder.layer.7.intermediate.dense | NNCFLinear | bias | [211] | 211 | 211 | 0 |
|
132 |
+
| 129 | nncf_module.bert.encoder.layer.7.output.dense | NNCFLinear | weight | [768, 211] | 162048 | 107335 | 0.337635 |
|
133 |
+
| 130 | nncf_module.bert.encoder.layer.7.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
134 |
+
| 131 | nncf_module.bert.encoder.layer.7.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
135 |
+
| 132 | nncf_module.bert.encoder.layer.7.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
136 |
+
| 133 | nncf_module.bert.encoder.layer.8.attention.self.query | NNCFLinear | weight | [448, 768] | 344064 | 129148 | 0.62464 |
|
137 |
+
| 134 | nncf_module.bert.encoder.layer.8.attention.self.query | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
138 |
+
| 135 | nncf_module.bert.encoder.layer.8.attention.self.key | NNCFLinear | weight | [448, 768] | 344064 | 130060 | 0.621989 |
|
139 |
+
| 136 | nncf_module.bert.encoder.layer.8.attention.self.key | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
140 |
+
| 137 | nncf_module.bert.encoder.layer.8.attention.self.value | NNCFLinear | weight | [448, 768] | 344064 | 108162 | 0.685634 |
|
141 |
+
| 138 | nncf_module.bert.encoder.layer.8.attention.self.value | NNCFLinear | bias | [448] | 448 | 448 | 0 |
|
142 |
+
| 139 | nncf_module.bert.encoder.layer.8.attention.output.dense | NNCFLinear | weight | [768, 448] | 344064 | 103447 | 0.699338 |
|
143 |
+
| 140 | nncf_module.bert.encoder.layer.8.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
144 |
+
| 141 | nncf_module.bert.encoder.layer.8.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
145 |
+
| 142 | nncf_module.bert.encoder.layer.8.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
146 |
+
| 143 | nncf_module.bert.encoder.layer.8.intermediate.dense | NNCFLinear | weight | [108, 768] | 82944 | 63275 | 0.237136 |
|
147 |
+
| 144 | nncf_module.bert.encoder.layer.8.intermediate.dense | NNCFLinear | bias | [108] | 108 | 108 | 0 |
|
148 |
+
| 145 | nncf_module.bert.encoder.layer.8.output.dense | NNCFLinear | weight | [768, 108] | 82944 | 62725 | 0.243767 |
|
149 |
+
| 146 | nncf_module.bert.encoder.layer.8.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
150 |
+
| 147 | nncf_module.bert.encoder.layer.8.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
151 |
+
| 148 | nncf_module.bert.encoder.layer.8.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
152 |
+
| 149 | nncf_module.bert.encoder.layer.9.attention.self.query | NNCFLinear | weight | [320, 768] | 245760 | 107145 | 0.564026 |
|
153 |
+
| 150 | nncf_module.bert.encoder.layer.9.attention.self.query | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
154 |
+
| 151 | nncf_module.bert.encoder.layer.9.attention.self.key | NNCFLinear | weight | [320, 768] | 245760 | 101811 | 0.58573 |
|
155 |
+
| 152 | nncf_module.bert.encoder.layer.9.attention.self.key | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
156 |
+
| 153 | nncf_module.bert.encoder.layer.9.attention.self.value | NNCFLinear | weight | [320, 768] | 245760 | 52182 | 0.787671 |
|
157 |
+
| 154 | nncf_module.bert.encoder.layer.9.attention.self.value | NNCFLinear | bias | [320] | 320 | 320 | 0 |
|
158 |
+
| 155 | nncf_module.bert.encoder.layer.9.attention.output.dense | NNCFLinear | weight | [768, 320] | 245760 | 53210 | 0.783488 |
|
159 |
+
| 156 | nncf_module.bert.encoder.layer.9.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
160 |
+
| 157 | nncf_module.bert.encoder.layer.9.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
161 |
+
| 158 | nncf_module.bert.encoder.layer.9.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
162 |
+
| 159 | nncf_module.bert.encoder.layer.9.intermediate.dense | NNCFLinear | weight | [53, 768] | 40704 | 33461 | 0.177943 |
|
163 |
+
| 160 | nncf_module.bert.encoder.layer.9.intermediate.dense | NNCFLinear | bias | [53] | 53 | 53 | 0 |
|
164 |
+
| 161 | nncf_module.bert.encoder.layer.9.output.dense | NNCFLinear | weight | [768, 53] | 40704 | 32551 | 0.2003 |
|
165 |
+
| 162 | nncf_module.bert.encoder.layer.9.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
166 |
+
| 163 | nncf_module.bert.encoder.layer.9.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
167 |
+
| 164 | nncf_module.bert.encoder.layer.9.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
168 |
+
| 165 | nncf_module.bert.encoder.layer.10.attention.self.query | NNCFLinear | weight | [384, 768] | 294912 | 112430 | 0.618768 |
|
169 |
+
| 166 | nncf_module.bert.encoder.layer.10.attention.self.query | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
170 |
+
| 167 | nncf_module.bert.encoder.layer.10.attention.self.key | NNCFLinear | weight | [384, 768] | 294912 | 109594 | 0.628384 |
|
171 |
+
| 168 | nncf_module.bert.encoder.layer.10.attention.self.key | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
172 |
+
| 169 | nncf_module.bert.encoder.layer.10.attention.self.value | NNCFLinear | weight | [384, 768] | 294912 | 61774 | 0.790534 |
|
173 |
+
| 170 | nncf_module.bert.encoder.layer.10.attention.self.value | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
174 |
+
| 171 | nncf_module.bert.encoder.layer.10.attention.output.dense | NNCFLinear | weight | [768, 384] | 294912 | 64183 | 0.782366 |
|
175 |
+
| 172 | nncf_module.bert.encoder.layer.10.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
176 |
+
| 173 | nncf_module.bert.encoder.layer.10.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
177 |
+
| 174 | nncf_module.bert.encoder.layer.10.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
178 |
+
| 175 | nncf_module.bert.encoder.layer.10.intermediate.dense | NNCFLinear | weight | [86, 768] | 66048 | 50455 | 0.236086 |
|
179 |
+
| 176 | nncf_module.bert.encoder.layer.10.intermediate.dense | NNCFLinear | bias | [86] | 86 | 86 | 0 |
|
180 |
+
| 177 | nncf_module.bert.encoder.layer.10.output.dense | NNCFLinear | weight | [768, 86] | 66048 | 49741 | 0.246896 |
|
181 |
+
| 178 | nncf_module.bert.encoder.layer.10.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
182 |
+
| 179 | nncf_module.bert.encoder.layer.10.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
183 |
+
| 180 | nncf_module.bert.encoder.layer.10.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
184 |
+
| 181 | nncf_module.bert.encoder.layer.11.attention.self.query | NNCFLinear | weight | [384, 768] | 294912 | 88129 | 0.701168 |
|
185 |
+
| 182 | nncf_module.bert.encoder.layer.11.attention.self.query | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
186 |
+
| 183 | nncf_module.bert.encoder.layer.11.attention.self.key | NNCFLinear | weight | [384, 768] | 294912 | 85288 | 0.710802 |
|
187 |
+
| 184 | nncf_module.bert.encoder.layer.11.attention.self.key | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
188 |
+
| 185 | nncf_module.bert.encoder.layer.11.attention.self.value | NNCFLinear | weight | [384, 768] | 294912 | 47258 | 0.839756 |
|
189 |
+
| 186 | nncf_module.bert.encoder.layer.11.attention.self.value | NNCFLinear | bias | [384] | 384 | 384 | 0 |
|
190 |
+
| 187 | nncf_module.bert.encoder.layer.11.attention.output.dense | NNCFLinear | weight | [768, 384] | 294912 | 49311 | 0.832794 |
|
191 |
+
| 188 | nncf_module.bert.encoder.layer.11.attention.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
192 |
+
| 189 | nncf_module.bert.encoder.layer.11.attention.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
193 |
+
| 190 | nncf_module.bert.encoder.layer.11.attention.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
194 |
+
| 191 | nncf_module.bert.encoder.layer.11.intermediate.dense | NNCFLinear | weight | [105, 768] | 80640 | 62254 | 0.228001 |
|
195 |
+
| 192 | nncf_module.bert.encoder.layer.11.intermediate.dense | NNCFLinear | bias | [105] | 105 | 105 | 0 |
|
196 |
+
| 193 | nncf_module.bert.encoder.layer.11.output.dense | NNCFLinear | weight | [768, 105] | 80640 | 61669 | 0.235255 |
|
197 |
+
| 194 | nncf_module.bert.encoder.layer.11.output.dense | NNCFLinear | bias | [768] | 768 | 768 | 0 |
|
198 |
+
| 195 | nncf_module.bert.encoder.layer.11.output.LayerNorm | LayerNorm | weight | [768] | 768 | 768 | 0 |
|
199 |
+
| 196 | nncf_module.bert.encoder.layer.11.output.LayerNorm | LayerNorm | bias | [768] | 768 | 768 | 0 |
|
200 |
+
| 197 | nncf_module.qa_outputs | NNCFLinear | weight | [2, 768] | 1536 | 1536 | 0 |
|
201 |
+
| 198 | nncf_module.qa_outputs | NNCFLinear | bias | [2] | 2 | 2 | 0 |
|
XP_linear_layer_sparsity_20M_params_57.92_sparsity.csv
ADDED
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
,layer_id,layer_type,param_type,shape,nparam,nnz,sparsity
|
2 |
+
5,nncf_module.bert.encoder.layer.0.attention.self.query,NNCFLinear,weight,"[320, 768]",245760,93583,0.6192097663879395
|
3 |
+
7,nncf_module.bert.encoder.layer.0.attention.self.key,NNCFLinear,weight,"[320, 768]",245760,98270,0.6001383066177368
|
4 |
+
9,nncf_module.bert.encoder.layer.0.attention.self.value,NNCFLinear,weight,"[320, 768]",245760,113605,0.5377400517463684
|
5 |
+
11,nncf_module.bert.encoder.layer.0.attention.output.dense,NNCFLinear,weight,"[768, 320]",245760,117208,0.5230793952941895
|
6 |
+
15,nncf_module.bert.encoder.layer.0.intermediate.dense,NNCFLinear,weight,"[185, 768]",142080,97073,0.3167722225189209
|
7 |
+
17,nncf_module.bert.encoder.layer.0.output.dense,NNCFLinear,weight,"[768, 185]",142080,94692,0.33353036642074585
|
8 |
+
21,nncf_module.bert.encoder.layer.1.attention.self.query,NNCFLinear,weight,"[320, 768]",245760,118436,0.5180826187133789
|
9 |
+
23,nncf_module.bert.encoder.layer.1.attention.self.key,NNCFLinear,weight,"[320, 768]",245760,118116,0.5193847417831421
|
10 |
+
25,nncf_module.bert.encoder.layer.1.attention.self.value,NNCFLinear,weight,"[320, 768]",245760,107511,0.5625365972518921
|
11 |
+
27,nncf_module.bert.encoder.layer.1.attention.output.dense,NNCFLinear,weight,"[768, 320]",245760,111189,0.5475707650184631
|
12 |
+
31,nncf_module.bert.encoder.layer.1.intermediate.dense,NNCFLinear,weight,"[315, 768]",241920,148783,0.3849908709526062
|
13 |
+
33,nncf_module.bert.encoder.layer.1.output.dense,NNCFLinear,weight,"[768, 315]",241920,143166,0.40820932388305664
|
14 |
+
37,nncf_module.bert.encoder.layer.2.attention.self.query,NNCFLinear,weight,"[576, 768]",442368,162735,0.6321275234222412
|
15 |
+
39,nncf_module.bert.encoder.layer.2.attention.self.key,NNCFLinear,weight,"[576, 768]",442368,164795,0.6274707913398743
|
16 |
+
41,nncf_module.bert.encoder.layer.2.attention.self.value,NNCFLinear,weight,"[576, 768]",442368,135670,0.6933096647262573
|
17 |
+
43,nncf_module.bert.encoder.layer.2.attention.output.dense,NNCFLinear,weight,"[768, 576]",442368,138445,0.6870365738868713
|
18 |
+
47,nncf_module.bert.encoder.layer.2.intermediate.dense,NNCFLinear,weight,"[339, 768]",260352,154035,0.408358633518219
|
19 |
+
49,nncf_module.bert.encoder.layer.2.output.dense,NNCFLinear,weight,"[768, 339]",260352,150816,0.4207226634025574
|
20 |
+
53,nncf_module.bert.encoder.layer.3.attention.self.query,NNCFLinear,weight,"[576, 768]",442368,170623,0.6142961978912354
|
21 |
+
55,nncf_module.bert.encoder.layer.3.attention.self.key,NNCFLinear,weight,"[576, 768]",442368,178401,0.5967136025428772
|
22 |
+
57,nncf_module.bert.encoder.layer.3.attention.self.value,NNCFLinear,weight,"[576, 768]",442368,171905,0.6113982200622559
|
23 |
+
59,nncf_module.bert.encoder.layer.3.attention.output.dense,NNCFLinear,weight,"[768, 576]",442368,169172,0.6175763010978699
|
24 |
+
63,nncf_module.bert.encoder.layer.3.intermediate.dense,NNCFLinear,weight,"[368, 768]",282624,163163,0.42268526554107666
|
25 |
+
65,nncf_module.bert.encoder.layer.3.output.dense,NNCFLinear,weight,"[768, 368]",282624,157506,0.44270122051239014
|
26 |
+
69,nncf_module.bert.encoder.layer.4.attention.self.query,NNCFLinear,weight,"[576, 768]",442368,175772,0.6026566028594971
|
27 |
+
71,nncf_module.bert.encoder.layer.4.attention.self.key,NNCFLinear,weight,"[576, 768]",442368,177087,0.5996840000152588
|
28 |
+
73,nncf_module.bert.encoder.layer.4.attention.self.value,NNCFLinear,weight,"[576, 768]",442368,163996,0.6292769908905029
|
29 |
+
75,nncf_module.bert.encoder.layer.4.attention.output.dense,NNCFLinear,weight,"[768, 576]",442368,159335,0.6398134231567383
|
30 |
+
79,nncf_module.bert.encoder.layer.4.intermediate.dense,NNCFLinear,weight,"[386, 768]",296448,167726,0.43421441316604614
|
31 |
+
81,nncf_module.bert.encoder.layer.4.output.dense,NNCFLinear,weight,"[768, 386]",296448,159865,0.46073174476623535
|
32 |
+
85,nncf_module.bert.encoder.layer.5.attention.self.query,NNCFLinear,weight,"[384, 768]",294912,114186,0.6128133535385132
|
33 |
+
87,nncf_module.bert.encoder.layer.5.attention.self.key,NNCFLinear,weight,"[384, 768]",294912,132782,0.5497572422027588
|
34 |
+
89,nncf_module.bert.encoder.layer.5.attention.self.value,NNCFLinear,weight,"[384, 768]",294912,134830,0.5428127646446228
|
35 |
+
91,nncf_module.bert.encoder.layer.5.attention.output.dense,NNCFLinear,weight,"[768, 384]",294912,131941,0.5526089072227478
|
36 |
+
95,nncf_module.bert.encoder.layer.5.intermediate.dense,NNCFLinear,weight,"[336, 768]",258048,153916,0.4035372734069824
|
37 |
+
97,nncf_module.bert.encoder.layer.5.output.dense,NNCFLinear,weight,"[768, 336]",258048,145794,0.4350120425224304
|
38 |
+
101,nncf_module.bert.encoder.layer.6.attention.self.query,NNCFLinear,weight,"[448, 768]",344064,131878,0.616705060005188
|
39 |
+
103,nncf_module.bert.encoder.layer.6.attention.self.key,NNCFLinear,weight,"[448, 768]",344064,144502,0.580014169216156
|
40 |
+
105,nncf_module.bert.encoder.layer.6.attention.self.value,NNCFLinear,weight,"[448, 768]",344064,130911,0.6195155382156372
|
41 |
+
107,nncf_module.bert.encoder.layer.6.attention.output.dense,NNCFLinear,weight,"[768, 448]",344064,125928,0.6339982748031616
|
42 |
+
111,nncf_module.bert.encoder.layer.6.intermediate.dense,NNCFLinear,weight,"[280, 768]",215040,135283,0.37089377641677856
|
43 |
+
113,nncf_module.bert.encoder.layer.6.output.dense,NNCFLinear,weight,"[768, 280]",215040,131619,0.3879324793815613
|
44 |
+
117,nncf_module.bert.encoder.layer.7.attention.self.query,NNCFLinear,weight,"[448, 768]",344064,132120,0.6160016655921936
|
45 |
+
119,nncf_module.bert.encoder.layer.7.attention.self.key,NNCFLinear,weight,"[448, 768]",344064,152223,0.5575735569000244
|
46 |
+
121,nncf_module.bert.encoder.layer.7.attention.self.value,NNCFLinear,weight,"[448, 768]",344064,141066,0.5900006890296936
|
47 |
+
123,nncf_module.bert.encoder.layer.7.attention.output.dense,NNCFLinear,weight,"[768, 448]",344064,135662,0.605707049369812
|
48 |
+
127,nncf_module.bert.encoder.layer.7.intermediate.dense,NNCFLinear,weight,"[211, 768]",162048,109590,0.3237189054489136
|
49 |
+
129,nncf_module.bert.encoder.layer.7.output.dense,NNCFLinear,weight,"[768, 211]",162048,107335,0.33763450384140015
|
50 |
+
133,nncf_module.bert.encoder.layer.8.attention.self.query,NNCFLinear,weight,"[448, 768]",344064,129148,0.624639630317688
|
51 |
+
135,nncf_module.bert.encoder.layer.8.attention.self.key,NNCFLinear,weight,"[448, 768]",344064,130060,0.6219888925552368
|
52 |
+
137,nncf_module.bert.encoder.layer.8.attention.self.value,NNCFLinear,weight,"[448, 768]",344064,108162,0.6856340765953064
|
53 |
+
139,nncf_module.bert.encoder.layer.8.attention.output.dense,NNCFLinear,weight,"[768, 448]",344064,103447,0.699337899684906
|
54 |
+
143,nncf_module.bert.encoder.layer.8.intermediate.dense,NNCFLinear,weight,"[108, 768]",82944,63275,0.2371358871459961
|
55 |
+
145,nncf_module.bert.encoder.layer.8.output.dense,NNCFLinear,weight,"[768, 108]",82944,62725,0.24376684427261353
|
56 |
+
149,nncf_module.bert.encoder.layer.9.attention.self.query,NNCFLinear,weight,"[320, 768]",245760,107145,0.56402587890625
|
57 |
+
151,nncf_module.bert.encoder.layer.9.attention.self.key,NNCFLinear,weight,"[320, 768]",245760,101811,0.5857299566268921
|
58 |
+
153,nncf_module.bert.encoder.layer.9.attention.self.value,NNCFLinear,weight,"[320, 768]",245760,52182,0.7876708507537842
|
59 |
+
155,nncf_module.bert.encoder.layer.9.attention.output.dense,NNCFLinear,weight,"[768, 320]",245760,53210,0.7834879159927368
|
60 |
+
159,nncf_module.bert.encoder.layer.9.intermediate.dense,NNCFLinear,weight,"[53, 768]",40704,33461,0.17794322967529297
|
61 |
+
161,nncf_module.bert.encoder.layer.9.output.dense,NNCFLinear,weight,"[768, 53]",40704,32551,0.20029973983764648
|
62 |
+
165,nncf_module.bert.encoder.layer.10.attention.self.query,NNCFLinear,weight,"[384, 768]",294912,112430,0.6187676191329956
|
63 |
+
167,nncf_module.bert.encoder.layer.10.attention.self.key,NNCFLinear,weight,"[384, 768]",294912,109594,0.6283840537071228
|
64 |
+
169,nncf_module.bert.encoder.layer.10.attention.self.value,NNCFLinear,weight,"[384, 768]",294912,61774,0.7905341386795044
|
65 |
+
171,nncf_module.bert.encoder.layer.10.attention.output.dense,NNCFLinear,weight,"[768, 384]",294912,64183,0.7823655605316162
|
66 |
+
175,nncf_module.bert.encoder.layer.10.intermediate.dense,NNCFLinear,weight,"[86, 768]",66048,50455,0.2360858917236328
|
67 |
+
177,nncf_module.bert.encoder.layer.10.output.dense,NNCFLinear,weight,"[768, 86]",66048,49741,0.24689620733261108
|
68 |
+
181,nncf_module.bert.encoder.layer.11.attention.self.query,NNCFLinear,weight,"[384, 768]",294912,88129,0.7011684775352478
|
69 |
+
183,nncf_module.bert.encoder.layer.11.attention.self.key,NNCFLinear,weight,"[384, 768]",294912,85288,0.7108018398284912
|
70 |
+
185,nncf_module.bert.encoder.layer.11.attention.self.value,NNCFLinear,weight,"[384, 768]",294912,47258,0.8397555947303772
|
71 |
+
187,nncf_module.bert.encoder.layer.11.attention.output.dense,NNCFLinear,weight,"[768, 384]",294912,49311,0.832794189453125
|
72 |
+
191,nncf_module.bert.encoder.layer.11.intermediate.dense,NNCFLinear,weight,"[105, 768]",80640,62254,0.22800099849700928
|
73 |
+
193,nncf_module.bert.encoder.layer.11.output.dense,NNCFLinear,weight,"[768, 105]",80640,61669,0.2352554202079773
|
XP_linear_layer_sparsity_20M_params_57.92_sparsity.md
ADDED
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
| | layer_id | layer_type | param_type | shape | nparam | nnz | sparsity |
|
2 |
+
|----:|:---------------------------------------------------------|:-------------|:-------------|:-----------|---------:|-------:|-----------:|
|
3 |
+
| 5 | nncf_module.bert.encoder.layer.0.attention.self.query | NNCFLinear | weight | [320, 768] | 245760 | 93583 | 0.61921 |
|
4 |
+
| 7 | nncf_module.bert.encoder.layer.0.attention.self.key | NNCFLinear | weight | [320, 768] | 245760 | 98270 | 0.600138 |
|
5 |
+
| 9 | nncf_module.bert.encoder.layer.0.attention.self.value | NNCFLinear | weight | [320, 768] | 245760 | 113605 | 0.53774 |
|
6 |
+
| 11 | nncf_module.bert.encoder.layer.0.attention.output.dense | NNCFLinear | weight | [768, 320] | 245760 | 117208 | 0.523079 |
|
7 |
+
| 15 | nncf_module.bert.encoder.layer.0.intermediate.dense | NNCFLinear | weight | [185, 768] | 142080 | 97073 | 0.316772 |
|
8 |
+
| 17 | nncf_module.bert.encoder.layer.0.output.dense | NNCFLinear | weight | [768, 185] | 142080 | 94692 | 0.33353 |
|
9 |
+
| 21 | nncf_module.bert.encoder.layer.1.attention.self.query | NNCFLinear | weight | [320, 768] | 245760 | 118436 | 0.518083 |
|
10 |
+
| 23 | nncf_module.bert.encoder.layer.1.attention.self.key | NNCFLinear | weight | [320, 768] | 245760 | 118116 | 0.519385 |
|
11 |
+
| 25 | nncf_module.bert.encoder.layer.1.attention.self.value | NNCFLinear | weight | [320, 768] | 245760 | 107511 | 0.562537 |
|
12 |
+
| 27 | nncf_module.bert.encoder.layer.1.attention.output.dense | NNCFLinear | weight | [768, 320] | 245760 | 111189 | 0.547571 |
|
13 |
+
| 31 | nncf_module.bert.encoder.layer.1.intermediate.dense | NNCFLinear | weight | [315, 768] | 241920 | 148783 | 0.384991 |
|
14 |
+
| 33 | nncf_module.bert.encoder.layer.1.output.dense | NNCFLinear | weight | [768, 315] | 241920 | 143166 | 0.408209 |
|
15 |
+
| 37 | nncf_module.bert.encoder.layer.2.attention.self.query | NNCFLinear | weight | [576, 768] | 442368 | 162735 | 0.632128 |
|
16 |
+
| 39 | nncf_module.bert.encoder.layer.2.attention.self.key | NNCFLinear | weight | [576, 768] | 442368 | 164795 | 0.627471 |
|
17 |
+
| 41 | nncf_module.bert.encoder.layer.2.attention.self.value | NNCFLinear | weight | [576, 768] | 442368 | 135670 | 0.69331 |
|
18 |
+
| 43 | nncf_module.bert.encoder.layer.2.attention.output.dense | NNCFLinear | weight | [768, 576] | 442368 | 138445 | 0.687037 |
|
19 |
+
| 47 | nncf_module.bert.encoder.layer.2.intermediate.dense | NNCFLinear | weight | [339, 768] | 260352 | 154035 | 0.408359 |
|
20 |
+
| 49 | nncf_module.bert.encoder.layer.2.output.dense | NNCFLinear | weight | [768, 339] | 260352 | 150816 | 0.420723 |
|
21 |
+
| 53 | nncf_module.bert.encoder.layer.3.attention.self.query | NNCFLinear | weight | [576, 768] | 442368 | 170623 | 0.614296 |
|
22 |
+
| 55 | nncf_module.bert.encoder.layer.3.attention.self.key | NNCFLinear | weight | [576, 768] | 442368 | 178401 | 0.596714 |
|
23 |
+
| 57 | nncf_module.bert.encoder.layer.3.attention.self.value | NNCFLinear | weight | [576, 768] | 442368 | 171905 | 0.611398 |
|
24 |
+
| 59 | nncf_module.bert.encoder.layer.3.attention.output.dense | NNCFLinear | weight | [768, 576] | 442368 | 169172 | 0.617576 |
|
25 |
+
| 63 | nncf_module.bert.encoder.layer.3.intermediate.dense | NNCFLinear | weight | [368, 768] | 282624 | 163163 | 0.422685 |
|
26 |
+
| 65 | nncf_module.bert.encoder.layer.3.output.dense | NNCFLinear | weight | [768, 368] | 282624 | 157506 | 0.442701 |
|
27 |
+
| 69 | nncf_module.bert.encoder.layer.4.attention.self.query | NNCFLinear | weight | [576, 768] | 442368 | 175772 | 0.602657 |
|
28 |
+
| 71 | nncf_module.bert.encoder.layer.4.attention.self.key | NNCFLinear | weight | [576, 768] | 442368 | 177087 | 0.599684 |
|
29 |
+
| 73 | nncf_module.bert.encoder.layer.4.attention.self.value | NNCFLinear | weight | [576, 768] | 442368 | 163996 | 0.629277 |
|
30 |
+
| 75 | nncf_module.bert.encoder.layer.4.attention.output.dense | NNCFLinear | weight | [768, 576] | 442368 | 159335 | 0.639813 |
|
31 |
+
| 79 | nncf_module.bert.encoder.layer.4.intermediate.dense | NNCFLinear | weight | [386, 768] | 296448 | 167726 | 0.434214 |
|
32 |
+
| 81 | nncf_module.bert.encoder.layer.4.output.dense | NNCFLinear | weight | [768, 386] | 296448 | 159865 | 0.460732 |
|
33 |
+
| 85 | nncf_module.bert.encoder.layer.5.attention.self.query | NNCFLinear | weight | [384, 768] | 294912 | 114186 | 0.612813 |
|
34 |
+
| 87 | nncf_module.bert.encoder.layer.5.attention.self.key | NNCFLinear | weight | [384, 768] | 294912 | 132782 | 0.549757 |
|
35 |
+
| 89 | nncf_module.bert.encoder.layer.5.attention.self.value | NNCFLinear | weight | [384, 768] | 294912 | 134830 | 0.542813 |
|
36 |
+
| 91 | nncf_module.bert.encoder.layer.5.attention.output.dense | NNCFLinear | weight | [768, 384] | 294912 | 131941 | 0.552609 |
|
37 |
+
| 95 | nncf_module.bert.encoder.layer.5.intermediate.dense | NNCFLinear | weight | [336, 768] | 258048 | 153916 | 0.403537 |
|
38 |
+
| 97 | nncf_module.bert.encoder.layer.5.output.dense | NNCFLinear | weight | [768, 336] | 258048 | 145794 | 0.435012 |
|
39 |
+
| 101 | nncf_module.bert.encoder.layer.6.attention.self.query | NNCFLinear | weight | [448, 768] | 344064 | 131878 | 0.616705 |
|
40 |
+
| 103 | nncf_module.bert.encoder.layer.6.attention.self.key | NNCFLinear | weight | [448, 768] | 344064 | 144502 | 0.580014 |
|
41 |
+
| 105 | nncf_module.bert.encoder.layer.6.attention.self.value | NNCFLinear | weight | [448, 768] | 344064 | 130911 | 0.619516 |
|
42 |
+
| 107 | nncf_module.bert.encoder.layer.6.attention.output.dense | NNCFLinear | weight | [768, 448] | 344064 | 125928 | 0.633998 |
|
43 |
+
| 111 | nncf_module.bert.encoder.layer.6.intermediate.dense | NNCFLinear | weight | [280, 768] | 215040 | 135283 | 0.370894 |
|
44 |
+
| 113 | nncf_module.bert.encoder.layer.6.output.dense | NNCFLinear | weight | [768, 280] | 215040 | 131619 | 0.387932 |
|
45 |
+
| 117 | nncf_module.bert.encoder.layer.7.attention.self.query | NNCFLinear | weight | [448, 768] | 344064 | 132120 | 0.616002 |
|
46 |
+
| 119 | nncf_module.bert.encoder.layer.7.attention.self.key | NNCFLinear | weight | [448, 768] | 344064 | 152223 | 0.557574 |
|
47 |
+
| 121 | nncf_module.bert.encoder.layer.7.attention.self.value | NNCFLinear | weight | [448, 768] | 344064 | 141066 | 0.590001 |
|
48 |
+
| 123 | nncf_module.bert.encoder.layer.7.attention.output.dense | NNCFLinear | weight | [768, 448] | 344064 | 135662 | 0.605707 |
|
49 |
+
| 127 | nncf_module.bert.encoder.layer.7.intermediate.dense | NNCFLinear | weight | [211, 768] | 162048 | 109590 | 0.323719 |
|
50 |
+
| 129 | nncf_module.bert.encoder.layer.7.output.dense | NNCFLinear | weight | [768, 211] | 162048 | 107335 | 0.337635 |
|
51 |
+
| 133 | nncf_module.bert.encoder.layer.8.attention.self.query | NNCFLinear | weight | [448, 768] | 344064 | 129148 | 0.62464 |
|
52 |
+
| 135 | nncf_module.bert.encoder.layer.8.attention.self.key | NNCFLinear | weight | [448, 768] | 344064 | 130060 | 0.621989 |
|
53 |
+
| 137 | nncf_module.bert.encoder.layer.8.attention.self.value | NNCFLinear | weight | [448, 768] | 344064 | 108162 | 0.685634 |
|
54 |
+
| 139 | nncf_module.bert.encoder.layer.8.attention.output.dense | NNCFLinear | weight | [768, 448] | 344064 | 103447 | 0.699338 |
|
55 |
+
| 143 | nncf_module.bert.encoder.layer.8.intermediate.dense | NNCFLinear | weight | [108, 768] | 82944 | 63275 | 0.237136 |
|
56 |
+
| 145 | nncf_module.bert.encoder.layer.8.output.dense | NNCFLinear | weight | [768, 108] | 82944 | 62725 | 0.243767 |
|
57 |
+
| 149 | nncf_module.bert.encoder.layer.9.attention.self.query | NNCFLinear | weight | [320, 768] | 245760 | 107145 | 0.564026 |
|
58 |
+
| 151 | nncf_module.bert.encoder.layer.9.attention.self.key | NNCFLinear | weight | [320, 768] | 245760 | 101811 | 0.58573 |
|
59 |
+
| 153 | nncf_module.bert.encoder.layer.9.attention.self.value | NNCFLinear | weight | [320, 768] | 245760 | 52182 | 0.787671 |
|
60 |
+
| 155 | nncf_module.bert.encoder.layer.9.attention.output.dense | NNCFLinear | weight | [768, 320] | 245760 | 53210 | 0.783488 |
|
61 |
+
| 159 | nncf_module.bert.encoder.layer.9.intermediate.dense | NNCFLinear | weight | [53, 768] | 40704 | 33461 | 0.177943 |
|
62 |
+
| 161 | nncf_module.bert.encoder.layer.9.output.dense | NNCFLinear | weight | [768, 53] | 40704 | 32551 | 0.2003 |
|
63 |
+
| 165 | nncf_module.bert.encoder.layer.10.attention.self.query | NNCFLinear | weight | [384, 768] | 294912 | 112430 | 0.618768 |
|
64 |
+
| 167 | nncf_module.bert.encoder.layer.10.attention.self.key | NNCFLinear | weight | [384, 768] | 294912 | 109594 | 0.628384 |
|
65 |
+
| 169 | nncf_module.bert.encoder.layer.10.attention.self.value | NNCFLinear | weight | [384, 768] | 294912 | 61774 | 0.790534 |
|
66 |
+
| 171 | nncf_module.bert.encoder.layer.10.attention.output.dense | NNCFLinear | weight | [768, 384] | 294912 | 64183 | 0.782366 |
|
67 |
+
| 175 | nncf_module.bert.encoder.layer.10.intermediate.dense | NNCFLinear | weight | [86, 768] | 66048 | 50455 | 0.236086 |
|
68 |
+
| 177 | nncf_module.bert.encoder.layer.10.output.dense | NNCFLinear | weight | [768, 86] | 66048 | 49741 | 0.246896 |
|
69 |
+
| 181 | nncf_module.bert.encoder.layer.11.attention.self.query | NNCFLinear | weight | [384, 768] | 294912 | 88129 | 0.701168 |
|
70 |
+
| 183 | nncf_module.bert.encoder.layer.11.attention.self.key | NNCFLinear | weight | [384, 768] | 294912 | 85288 | 0.710802 |
|
71 |
+
| 185 | nncf_module.bert.encoder.layer.11.attention.self.value | NNCFLinear | weight | [384, 768] | 294912 | 47258 | 0.839756 |
|
72 |
+
| 187 | nncf_module.bert.encoder.layer.11.attention.output.dense | NNCFLinear | weight | [768, 384] | 294912 | 49311 | 0.832794 |
|
73 |
+
| 191 | nncf_module.bert.encoder.layer.11.intermediate.dense | NNCFLinear | weight | [105, 768] | 80640 | 62254 | 0.228001 |
|
74 |
+
| 193 | nncf_module.bert.encoder.layer.11.output.dense | NNCFLinear | weight | [768, 105] | 80640 | 61669 | 0.235255 |
|
all_results.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"eval_exact_match": 80.44465468306528,
|
3 |
+
"eval_f1": 87.76782349205592,
|
4 |
+
"eval_samples": 10784
|
5 |
+
}
|
bert-base-squadv1-block-pruning-hybrid-filled-lt-nncf-57.92sparse-lt.onnx
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:790862e15f5e7452d972f1a381cb03309b7f76b47b2961eb3eac3ef31776783c
|
3 |
+
size 176456442
|
checkpoint-20000/config.json
ADDED
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "/data1/vchua/tld-poc/bert-base-squadv1-local-hybrid-compiled",
|
3 |
+
"architectures": [
|
4 |
+
"NNCFNetwork"
|
5 |
+
],
|
6 |
+
"attention_probs_dropout_prob": 0.1,
|
7 |
+
"classifier_dropout": null,
|
8 |
+
"gradient_checkpointing": false,
|
9 |
+
"hidden_act": "gelu",
|
10 |
+
"hidden_dropout_prob": 0.1,
|
11 |
+
"hidden_size": 768,
|
12 |
+
"initializer_range": 0.02,
|
13 |
+
"intermediate_size": 3072,
|
14 |
+
"layer_norm_eps": 1e-12,
|
15 |
+
"max_position_embeddings": 512,
|
16 |
+
"model_type": "bert",
|
17 |
+
"num_attention_heads": 12,
|
18 |
+
"num_hidden_layers": 12,
|
19 |
+
"pad_token_id": 0,
|
20 |
+
"position_embedding_type": "absolute",
|
21 |
+
"pruned_heads": {
|
22 |
+
"0": [
|
23 |
+
0,
|
24 |
+
2,
|
25 |
+
4,
|
26 |
+
5,
|
27 |
+
6,
|
28 |
+
7,
|
29 |
+
11
|
30 |
+
],
|
31 |
+
"1": [
|
32 |
+
0,
|
33 |
+
2,
|
34 |
+
3,
|
35 |
+
5,
|
36 |
+
6,
|
37 |
+
7,
|
38 |
+
8
|
39 |
+
],
|
40 |
+
"2": [
|
41 |
+
8,
|
42 |
+
4,
|
43 |
+
7
|
44 |
+
],
|
45 |
+
"3": [
|
46 |
+
2,
|
47 |
+
4,
|
48 |
+
6
|
49 |
+
],
|
50 |
+
"4": [
|
51 |
+
1,
|
52 |
+
2,
|
53 |
+
11
|
54 |
+
],
|
55 |
+
"5": [
|
56 |
+
1,
|
57 |
+
2,
|
58 |
+
5,
|
59 |
+
6,
|
60 |
+
7,
|
61 |
+
11
|
62 |
+
],
|
63 |
+
"6": [
|
64 |
+
0,
|
65 |
+
2,
|
66 |
+
3,
|
67 |
+
7,
|
68 |
+
10
|
69 |
+
],
|
70 |
+
"7": [
|
71 |
+
1,
|
72 |
+
3,
|
73 |
+
6,
|
74 |
+
7,
|
75 |
+
11
|
76 |
+
],
|
77 |
+
"8": [
|
78 |
+
0,
|
79 |
+
3,
|
80 |
+
4,
|
81 |
+
5,
|
82 |
+
8
|
83 |
+
],
|
84 |
+
"9": [
|
85 |
+
1,
|
86 |
+
3,
|
87 |
+
4,
|
88 |
+
5,
|
89 |
+
7,
|
90 |
+
9,
|
91 |
+
10
|
92 |
+
],
|
93 |
+
"10": [
|
94 |
+
1,
|
95 |
+
4,
|
96 |
+
5,
|
97 |
+
6,
|
98 |
+
7,
|
99 |
+
8
|
100 |
+
],
|
101 |
+
"11": [
|
102 |
+
4,
|
103 |
+
5,
|
104 |
+
7,
|
105 |
+
8,
|
106 |
+
10,
|
107 |
+
11
|
108 |
+
]
|
109 |
+
},
|
110 |
+
"torch_dtype": "float32",
|
111 |
+
"transformers_version": "4.9.1",
|
112 |
+
"type_vocab_size": 2,
|
113 |
+
"use_cache": true,
|
114 |
+
"vocab_size": 30522
|
115 |
+
}
|
checkpoint-20000/optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4b399532249838321d98181e99b6b7e3fe3d092488a606d60372a9d15133919f
|
3 |
+
size 352877861
|
checkpoint-20000/pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c549becb6c19d157aa4613fd291f20d211f6e25a23fe8e971ce2660b9cd58dad
|
3 |
+
size 257253793
|
checkpoint-20000/rng_state.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:38acbf96e3d114b4e820fc825cb129fc1dda7723810be41dbe2bc35cfaf24968
|
3 |
+
size 14503
|
checkpoint-20000/scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:09e958fb5ca92f7a4b806664c6998c7aa199cf58d8b133b73b5b462f12693a94
|
3 |
+
size 623
|
checkpoint-20000/special_tokens_map.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
|
checkpoint-20000/tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
checkpoint-20000/tokenizer_config.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "/data1/vchua/tld-poc/bert-base-squadv1-local-hybrid-compiled", "tokenizer_class": "BertTokenizer"}
|
checkpoint-20000/trainer_state.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:afa70c2c5a4720fa84584001cccb6b59fdc2c8825f6fca9af18526e8de268b16
|
3 |
+
size 10483928
|
checkpoint-20000/training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a4ac8cd60142a63b26c4c08d769e2244549a2b0a8a5b18d6214f46615971dad7
|
3 |
+
size 3439
|
checkpoint-20000/vocab.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
compressed_graph.dot
ADDED
The diff for this file is too large to render.
See raw diff
|
|
config.json
ADDED
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "/data1/vchua/tld-poc/bert-base-squadv1-local-hybrid-compiled",
|
3 |
+
"architectures": [
|
4 |
+
"NNCFNetwork"
|
5 |
+
],
|
6 |
+
"attention_probs_dropout_prob": 0.1,
|
7 |
+
"classifier_dropout": null,
|
8 |
+
"gradient_checkpointing": false,
|
9 |
+
"hidden_act": "gelu",
|
10 |
+
"hidden_dropout_prob": 0.1,
|
11 |
+
"hidden_size": 768,
|
12 |
+
"initializer_range": 0.02,
|
13 |
+
"intermediate_size": 3072,
|
14 |
+
"layer_norm_eps": 1e-12,
|
15 |
+
"max_position_embeddings": 512,
|
16 |
+
"model_type": "bert",
|
17 |
+
"num_attention_heads": 12,
|
18 |
+
"num_hidden_layers": 12,
|
19 |
+
"pad_token_id": 0,
|
20 |
+
"position_embedding_type": "absolute",
|
21 |
+
"pruned_heads": {
|
22 |
+
"0": [
|
23 |
+
0,
|
24 |
+
2,
|
25 |
+
4,
|
26 |
+
5,
|
27 |
+
6,
|
28 |
+
7,
|
29 |
+
11
|
30 |
+
],
|
31 |
+
"1": [
|
32 |
+
0,
|
33 |
+
2,
|
34 |
+
3,
|
35 |
+
5,
|
36 |
+
6,
|
37 |
+
7,
|
38 |
+
8
|
39 |
+
],
|
40 |
+
"2": [
|
41 |
+
8,
|
42 |
+
4,
|
43 |
+
7
|
44 |
+
],
|
45 |
+
"3": [
|
46 |
+
2,
|
47 |
+
4,
|
48 |
+
6
|
49 |
+
],
|
50 |
+
"4": [
|
51 |
+
1,
|
52 |
+
2,
|
53 |
+
11
|
54 |
+
],
|
55 |
+
"5": [
|
56 |
+
1,
|
57 |
+
2,
|
58 |
+
5,
|
59 |
+
6,
|
60 |
+
7,
|
61 |
+
11
|
62 |
+
],
|
63 |
+
"6": [
|
64 |
+
0,
|
65 |
+
2,
|
66 |
+
3,
|
67 |
+
7,
|
68 |
+
10
|
69 |
+
],
|
70 |
+
"7": [
|
71 |
+
1,
|
72 |
+
3,
|
73 |
+
6,
|
74 |
+
7,
|
75 |
+
11
|
76 |
+
],
|
77 |
+
"8": [
|
78 |
+
0,
|
79 |
+
3,
|
80 |
+
4,
|
81 |
+
5,
|
82 |
+
8
|
83 |
+
],
|
84 |
+
"9": [
|
85 |
+
1,
|
86 |
+
3,
|
87 |
+
4,
|
88 |
+
5,
|
89 |
+
7,
|
90 |
+
9,
|
91 |
+
10
|
92 |
+
],
|
93 |
+
"10": [
|
94 |
+
1,
|
95 |
+
4,
|
96 |
+
5,
|
97 |
+
6,
|
98 |
+
7,
|
99 |
+
8
|
100 |
+
],
|
101 |
+
"11": [
|
102 |
+
4,
|
103 |
+
5,
|
104 |
+
7,
|
105 |
+
8,
|
106 |
+
10,
|
107 |
+
11
|
108 |
+
]
|
109 |
+
},
|
110 |
+
"torch_dtype": "float32",
|
111 |
+
"transformers_version": "4.9.1",
|
112 |
+
"type_vocab_size": 2,
|
113 |
+
"use_cache": true,
|
114 |
+
"vocab_size": 30522
|
115 |
+
}
|
eval_XP_results.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"eval_exact_match": 80.44465468306528,
|
3 |
+
"eval_f1": 87.76782349205592,
|
4 |
+
"eval_samples": 10784
|
5 |
+
}
|
eval_nbest_predictions.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0f9ee0a24d61f6636652cdefa82a26124098e0c0f38af53e4d3a96ae7f72356c
|
3 |
+
size 48943971
|
eval_predictions.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
nncf_bert_squad_sparsity.json
ADDED
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"input_info": [
|
3 |
+
{
|
4 |
+
"sample_size": [1, 384],
|
5 |
+
"type": "long"
|
6 |
+
},
|
7 |
+
{
|
8 |
+
"sample_size": [1, 384],
|
9 |
+
"type": "long"
|
10 |
+
},
|
11 |
+
{
|
12 |
+
"sample_size": [1, 384],
|
13 |
+
"type": "long"
|
14 |
+
}
|
15 |
+
],
|
16 |
+
"compression":
|
17 |
+
[
|
18 |
+
// {
|
19 |
+
// "algorithm": "knowledge_distillation",
|
20 |
+
// "type": "softmax"
|
21 |
+
// },
|
22 |
+
{
|
23 |
+
"algorithm": "magnitude_sparsity",
|
24 |
+
"sparsity_init": 0.579176,
|
25 |
+
"params": {
|
26 |
+
"schedule": "multistep",
|
27 |
+
"multistep_steps": [
|
28 |
+
2,
|
29 |
+
4,
|
30 |
+
6,
|
31 |
+
8
|
32 |
+
],
|
33 |
+
"multistep_sparsity_levels": [
|
34 |
+
0.579176,
|
35 |
+
0.579176,
|
36 |
+
0.579176,
|
37 |
+
0.579176,
|
38 |
+
0.579176,
|
39 |
+
]
|
40 |
+
},
|
41 |
+
"ignored_scopes": ["{re}.*NNCFEmbedding", "{re}.*qa_outputs*"]
|
42 |
+
},
|
43 |
+
// {
|
44 |
+
// "algorithm": "quantization",
|
45 |
+
// "initializer": {
|
46 |
+
// "range": {
|
47 |
+
// "num_init_samples": 32,
|
48 |
+
// "type": "percentile",
|
49 |
+
// "params":
|
50 |
+
// {
|
51 |
+
// "min_percentile": 0.01,
|
52 |
+
// "max_percentile": 99.99
|
53 |
+
// }
|
54 |
+
// },
|
55 |
+
|
56 |
+
// "batchnorm_adaptation": {
|
57 |
+
// "num_bn_adaptation_samples": 200
|
58 |
+
// }
|
59 |
+
// },
|
60 |
+
// "activations":
|
61 |
+
// {
|
62 |
+
// "mode": "symmetric"
|
63 |
+
// },
|
64 |
+
// "weights":
|
65 |
+
// {
|
66 |
+
// "mode": "symmetric",
|
67 |
+
// "signed": true,
|
68 |
+
// "per_channel": false
|
69 |
+
// }
|
70 |
+
// }
|
71 |
+
]
|
72 |
+
}
|
original_graph.dot
ADDED
The diff for this file is too large to render.
See raw diff
|
|
pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:893cbea0420d5e9b5c7eab4e10f3db7f0e3eea1bbc8dcbe5171ea62a57b8e33e
|
3 |
+
size 257253793
|
special_tokens_map.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "/data1/vchua/tld-poc/bert-base-squadv1-local-hybrid-compiled", "tokenizer_class": "BertTokenizer"}
|
train_results.json
ADDED
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"epoch": 5.0,
|
3 |
+
"train_loss": 0.2509266505930515,
|
4 |
+
"train_runtime": 38611.3829,
|
5 |
+
"train_samples": 88524,
|
6 |
+
"train_samples_per_second": 11.463,
|
7 |
+
"train_steps_per_second": 0.716
|
8 |
+
}
|
trainer_state.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c0aad6f919e9c7888c1c8dc73236192f4d3f5524276cd510280bda92290fcabd
|
3 |
+
size 14504790
|
training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a4ac8cd60142a63b26c4c08d769e2244549a2b0a8a5b18d6214f46615971dad7
|
3 |
+
size 3439
|
vocab.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|