Update readme, add collaterals and model analysis report

.gitattributes

@@ -25,3 +25,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+optimizer.pt filter=lfs diff=lfs merge=lfs -text
+nbest_predictions.json filter=lfs diff=lfs merge=lfs -text
+eval_nbest_predictions.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,61 @@
+This model is a downstream fine-tuning of [```vuiseng9/bert-base-squadv1-block-pruning-hybrid```](https://huggingface.co/vuiseng9/bert-base-squadv1-block-pruning-hybrid). "filled" means unstructured fine-grained sparsified parameters are allowed to learn during fine-tuning. "lt" means distillation of larger model as teacher, i.e. ```bert-large-uncased-whole-word-masking-finetuned-squad```  
+
+```
+  eval_exact_match = 80.3311
+  eval_f1          = 87.69
+  eval_samples     =   10784
+``` 
+This model is a replication of [block pruning paper](https://arxiv.org/abs/2109.04838) with its open-sourced codebase (forked and modified). 
+To reproduce this model, pls follow [documentation here](https://github.com/vuiseng9/nn_pruning/blob/reproduce-evaluation/reproduce-eval/readme.md) until step 3.
+
+# Eval
+The model cannot be evaluated with HF QA example out-of-the-box as the final dimension of the model architecture has been realized. Follow the custom setup below.
+
+```bash
+# OpenVINO/NNCF
+git clone https://github.com/vuiseng9/nncf && cd nncf
+git checkout tld-poc
+git reset --hard 1dec7afe7a4b567c059fcf287ea2c234980fded2
+python setup.py develop
+pip install -r examples/torch/requirements.txt
+
+# Huggingface nn_pruning
+git clone https://github.com/vuiseng9/nn_pruning && cd nn_pruning
+git checkout reproduce-evaluation
+git reset --hard 2d4e196d694c465e43e5fbce6c3836d0a60e1446
+pip install -e ".[dev]"
+
+# Huggingface Transformers
+git clone https://github.com/vuiseng9/transformers && cd transformers
+git checkout tld-poc
+git reset --hard 10a1e29d84484e48fd106f58957d9ffc89dc43c5
+pip install -e .
+head -n 1 examples/pytorch/question-answering/requirements.txt | xargs -i pip install {}
+
+```
+This repo must be cloned locally.
+```bash
+git clone https://huggingface.co/vuiseng9/bert-base-squadv1-block-pruning-hybrid-filled-lt
+```
+Add ```--optimize_model_before_eval``` and ```--optimized_checkpoint /path/to/clone``` during evaluation.
+
+```bash
+export CUDA_VISIBLE_DEVICES=0
+
+OUTDIR=eval-bert-base-squadv1-block-pruning-hybrid-filled-lt-cropped
+WORKDIR=transformers/examples/pytorch/question-answering
+cd $WORKDIR
+mkdir $OUTDIR
+
+nohup python run_qa.py  \
+    --model_name_or_path vuiseng9/bert-base-squadv1-block-pruning-hybrid  \
+    --dataset_name squad  \
+    --optimize_model_before_eval \
+    --optimized_checkpoint /path/to/clone/bert-base-squadv1-block-pruning-hybrid-filled-lt  \
+    --do_eval  \
+    --per_device_eval_batch_size 16  \
+    --max_seq_length 384  \
+    --doc_stride 128  \
+    --overwrite_output_dir \
+    --output_dir $OUTDIR 2>&1 | tee $OUTDIR/run.log &
+```

XP_layer_wise_sparsity_global_rate_0.00.csv

@@ -0,0 +1,200 @@
+,layer_id,layer_type,param_type,shape,nparam,nnz,sparsity
+0,bert.embeddings.word_embeddings,Embedding,weight,"[30522, 768]",23440896,23440896,0.0
+1,bert.embeddings.position_embeddings,Embedding,weight,"[512, 768]",393216,393216,0.0
+2,bert.embeddings.token_type_embeddings,Embedding,weight,"[2, 768]",1536,1536,0.0
+3,bert.embeddings.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+4,bert.embeddings.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+5,bert.encoder.layer.0.attention.self.query,Linear,weight,"[320, 768]",245760,245760,0.0
+6,bert.encoder.layer.0.attention.self.query,Linear,bias,[320],320,320,0.0
+7,bert.encoder.layer.0.attention.self.key,Linear,weight,"[320, 768]",245760,245760,0.0
+8,bert.encoder.layer.0.attention.self.key,Linear,bias,[320],320,320,0.0
+9,bert.encoder.layer.0.attention.self.value,Linear,weight,"[320, 768]",245760,245760,0.0
+10,bert.encoder.layer.0.attention.self.value,Linear,bias,[320],320,320,0.0
+11,bert.encoder.layer.0.attention.output.dense,Linear,weight,"[768, 320]",245760,245760,0.0
+12,bert.encoder.layer.0.attention.output.dense,Linear,bias,[768],768,768,0.0
+13,bert.encoder.layer.0.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+14,bert.encoder.layer.0.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+15,bert.encoder.layer.0.intermediate.dense,Linear,weight,"[185, 768]",142080,142080,0.0
+16,bert.encoder.layer.0.intermediate.dense,Linear,bias,[185],185,185,0.0
+17,bert.encoder.layer.0.output.dense,Linear,weight,"[768, 185]",142080,142080,0.0
+18,bert.encoder.layer.0.output.dense,Linear,bias,[768],768,768,0.0
+19,bert.encoder.layer.0.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+20,bert.encoder.layer.0.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+21,bert.encoder.layer.1.attention.self.query,Linear,weight,"[320, 768]",245760,245760,0.0
+22,bert.encoder.layer.1.attention.self.query,Linear,bias,[320],320,320,0.0
+23,bert.encoder.layer.1.attention.self.key,Linear,weight,"[320, 768]",245760,245760,0.0
+24,bert.encoder.layer.1.attention.self.key,Linear,bias,[320],320,320,0.0
+25,bert.encoder.layer.1.attention.self.value,Linear,weight,"[320, 768]",245760,245760,0.0
+26,bert.encoder.layer.1.attention.self.value,Linear,bias,[320],320,320,0.0
+27,bert.encoder.layer.1.attention.output.dense,Linear,weight,"[768, 320]",245760,245760,0.0
+28,bert.encoder.layer.1.attention.output.dense,Linear,bias,[768],768,768,0.0
+29,bert.encoder.layer.1.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+30,bert.encoder.layer.1.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+31,bert.encoder.layer.1.intermediate.dense,Linear,weight,"[315, 768]",241920,241920,0.0
+32,bert.encoder.layer.1.intermediate.dense,Linear,bias,[315],315,315,0.0
+33,bert.encoder.layer.1.output.dense,Linear,weight,"[768, 315]",241920,241920,0.0
+34,bert.encoder.layer.1.output.dense,Linear,bias,[768],768,768,0.0
+35,bert.encoder.layer.1.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+36,bert.encoder.layer.1.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+37,bert.encoder.layer.2.attention.self.query,Linear,weight,"[576, 768]",442368,442368,0.0
+38,bert.encoder.layer.2.attention.self.query,Linear,bias,[576],576,576,0.0
+39,bert.encoder.layer.2.attention.self.key,Linear,weight,"[576, 768]",442368,442368,0.0
+40,bert.encoder.layer.2.attention.self.key,Linear,bias,[576],576,576,0.0
+41,bert.encoder.layer.2.attention.self.value,Linear,weight,"[576, 768]",442368,442368,0.0
+42,bert.encoder.layer.2.attention.self.value,Linear,bias,[576],576,576,0.0
+43,bert.encoder.layer.2.attention.output.dense,Linear,weight,"[768, 576]",442368,442368,0.0
+44,bert.encoder.layer.2.attention.output.dense,Linear,bias,[768],768,768,0.0
+45,bert.encoder.layer.2.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+46,bert.encoder.layer.2.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+47,bert.encoder.layer.2.intermediate.dense,Linear,weight,"[339, 768]",260352,260352,0.0
+48,bert.encoder.layer.2.intermediate.dense,Linear,bias,[339],339,339,0.0
+49,bert.encoder.layer.2.output.dense,Linear,weight,"[768, 339]",260352,260352,0.0
+50,bert.encoder.layer.2.output.dense,Linear,bias,[768],768,768,0.0
+51,bert.encoder.layer.2.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+52,bert.encoder.layer.2.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+53,bert.encoder.layer.3.attention.self.query,Linear,weight,"[576, 768]",442368,442368,0.0
+54,bert.encoder.layer.3.attention.self.query,Linear,bias,[576],576,576,0.0
+55,bert.encoder.layer.3.attention.self.key,Linear,weight,"[576, 768]",442368,442368,0.0
+56,bert.encoder.layer.3.attention.self.key,Linear,bias,[576],576,576,0.0
+57,bert.encoder.layer.3.attention.self.value,Linear,weight,"[576, 768]",442368,442368,0.0
+58,bert.encoder.layer.3.attention.self.value,Linear,bias,[576],576,576,0.0
+59,bert.encoder.layer.3.attention.output.dense,Linear,weight,"[768, 576]",442368,442368,0.0
+60,bert.encoder.layer.3.attention.output.dense,Linear,bias,[768],768,768,0.0
+61,bert.encoder.layer.3.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+62,bert.encoder.layer.3.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+63,bert.encoder.layer.3.intermediate.dense,Linear,weight,"[368, 768]",282624,282624,0.0
+64,bert.encoder.layer.3.intermediate.dense,Linear,bias,[368],368,368,0.0
+65,bert.encoder.layer.3.output.dense,Linear,weight,"[768, 368]",282624,282624,0.0
+66,bert.encoder.layer.3.output.dense,Linear,bias,[768],768,768,0.0
+67,bert.encoder.layer.3.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+68,bert.encoder.layer.3.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+69,bert.encoder.layer.4.attention.self.query,Linear,weight,"[576, 768]",442368,442368,0.0
+70,bert.encoder.layer.4.attention.self.query,Linear,bias,[576],576,576,0.0
+71,bert.encoder.layer.4.attention.self.key,Linear,weight,"[576, 768]",442368,442368,0.0
+72,bert.encoder.layer.4.attention.self.key,Linear,bias,[576],576,576,0.0
+73,bert.encoder.layer.4.attention.self.value,Linear,weight,"[576, 768]",442368,442368,0.0
+74,bert.encoder.layer.4.attention.self.value,Linear,bias,[576],576,576,0.0
+75,bert.encoder.layer.4.attention.output.dense,Linear,weight,"[768, 576]",442368,442368,0.0
+76,bert.encoder.layer.4.attention.output.dense,Linear,bias,[768],768,768,0.0
+77,bert.encoder.layer.4.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+78,bert.encoder.layer.4.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+79,bert.encoder.layer.4.intermediate.dense,Linear,weight,"[386, 768]",296448,296448,0.0
+80,bert.encoder.layer.4.intermediate.dense,Linear,bias,[386],386,386,0.0
+81,bert.encoder.layer.4.output.dense,Linear,weight,"[768, 386]",296448,296448,0.0
+82,bert.encoder.layer.4.output.dense,Linear,bias,[768],768,768,0.0
+83,bert.encoder.layer.4.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+84,bert.encoder.layer.4.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+85,bert.encoder.layer.5.attention.self.query,Linear,weight,"[384, 768]",294912,294912,0.0
+86,bert.encoder.layer.5.attention.self.query,Linear,bias,[384],384,384,0.0
+87,bert.encoder.layer.5.attention.self.key,Linear,weight,"[384, 768]",294912,294912,0.0
+88,bert.encoder.layer.5.attention.self.key,Linear,bias,[384],384,384,0.0
+89,bert.encoder.layer.5.attention.self.value,Linear,weight,"[384, 768]",294912,294912,0.0
+90,bert.encoder.layer.5.attention.self.value,Linear,bias,[384],384,384,0.0
+91,bert.encoder.layer.5.attention.output.dense,Linear,weight,"[768, 384]",294912,294912,0.0
+92,bert.encoder.layer.5.attention.output.dense,Linear,bias,[768],768,768,0.0
+93,bert.encoder.layer.5.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+94,bert.encoder.layer.5.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+95,bert.encoder.layer.5.intermediate.dense,Linear,weight,"[336, 768]",258048,258048,0.0
+96,bert.encoder.layer.5.intermediate.dense,Linear,bias,[336],336,336,0.0
+97,bert.encoder.layer.5.output.dense,Linear,weight,"[768, 336]",258048,258048,0.0
+98,bert.encoder.layer.5.output.dense,Linear,bias,[768],768,768,0.0
+99,bert.encoder.layer.5.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+100,bert.encoder.layer.5.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+101,bert.encoder.layer.6.attention.self.query,Linear,weight,"[448, 768]",344064,344064,0.0
+102,bert.encoder.layer.6.attention.self.query,Linear,bias,[448],448,448,0.0
+103,bert.encoder.layer.6.attention.self.key,Linear,weight,"[448, 768]",344064,344064,0.0
+104,bert.encoder.layer.6.attention.self.key,Linear,bias,[448],448,448,0.0
+105,bert.encoder.layer.6.attention.self.value,Linear,weight,"[448, 768]",344064,344064,0.0
+106,bert.encoder.layer.6.attention.self.value,Linear,bias,[448],448,448,0.0
+107,bert.encoder.layer.6.attention.output.dense,Linear,weight,"[768, 448]",344064,344064,0.0
+108,bert.encoder.layer.6.attention.output.dense,Linear,bias,[768],768,768,0.0
+109,bert.encoder.layer.6.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+110,bert.encoder.layer.6.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+111,bert.encoder.layer.6.intermediate.dense,Linear,weight,"[280, 768]",215040,215040,0.0
+112,bert.encoder.layer.6.intermediate.dense,Linear,bias,[280],280,280,0.0
+113,bert.encoder.layer.6.output.dense,Linear,weight,"[768, 280]",215040,215040,0.0
+114,bert.encoder.layer.6.output.dense,Linear,bias,[768],768,768,0.0
+115,bert.encoder.layer.6.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+116,bert.encoder.layer.6.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+117,bert.encoder.layer.7.attention.self.query,Linear,weight,"[448, 768]",344064,344064,0.0
+118,bert.encoder.layer.7.attention.self.query,Linear,bias,[448],448,448,0.0
+119,bert.encoder.layer.7.attention.self.key,Linear,weight,"[448, 768]",344064,344064,0.0
+120,bert.encoder.layer.7.attention.self.key,Linear,bias,[448],448,448,0.0
+121,bert.encoder.layer.7.attention.self.value,Linear,weight,"[448, 768]",344064,344064,0.0
+122,bert.encoder.layer.7.attention.self.value,Linear,bias,[448],448,448,0.0
+123,bert.encoder.layer.7.attention.output.dense,Linear,weight,"[768, 448]",344064,344064,0.0
+124,bert.encoder.layer.7.attention.output.dense,Linear,bias,[768],768,768,0.0
+125,bert.encoder.layer.7.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+126,bert.encoder.layer.7.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+127,bert.encoder.layer.7.intermediate.dense,Linear,weight,"[211, 768]",162048,162048,0.0
+128,bert.encoder.layer.7.intermediate.dense,Linear,bias,[211],211,211,0.0
+129,bert.encoder.layer.7.output.dense,Linear,weight,"[768, 211]",162048,162048,0.0
+130,bert.encoder.layer.7.output.dense,Linear,bias,[768],768,768,0.0
+131,bert.encoder.layer.7.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+132,bert.encoder.layer.7.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+133,bert.encoder.layer.8.attention.self.query,Linear,weight,"[448, 768]",344064,344064,0.0
+134,bert.encoder.layer.8.attention.self.query,Linear,bias,[448],448,448,0.0
+135,bert.encoder.layer.8.attention.self.key,Linear,weight,"[448, 768]",344064,344064,0.0
+136,bert.encoder.layer.8.attention.self.key,Linear,bias,[448],448,448,0.0
+137,bert.encoder.layer.8.attention.self.value,Linear,weight,"[448, 768]",344064,344064,0.0
+138,bert.encoder.layer.8.attention.self.value,Linear,bias,[448],448,448,0.0
+139,bert.encoder.layer.8.attention.output.dense,Linear,weight,"[768, 448]",344064,344064,0.0
+140,bert.encoder.layer.8.attention.output.dense,Linear,bias,[768],768,768,0.0
+141,bert.encoder.layer.8.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+142,bert.encoder.layer.8.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+143,bert.encoder.layer.8.intermediate.dense,Linear,weight,"[108, 768]",82944,82944,0.0
+144,bert.encoder.layer.8.intermediate.dense,Linear,bias,[108],108,108,0.0
+145,bert.encoder.layer.8.output.dense,Linear,weight,"[768, 108]",82944,82944,0.0
+146,bert.encoder.layer.8.output.dense,Linear,bias,[768],768,768,0.0
+147,bert.encoder.layer.8.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+148,bert.encoder.layer.8.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+149,bert.encoder.layer.9.attention.self.query,Linear,weight,"[320, 768]",245760,245760,0.0
+150,bert.encoder.layer.9.attention.self.query,Linear,bias,[320],320,320,0.0
+151,bert.encoder.layer.9.attention.self.key,Linear,weight,"[320, 768]",245760,245760,0.0
+152,bert.encoder.layer.9.attention.self.key,Linear,bias,[320],320,320,0.0
+153,bert.encoder.layer.9.attention.self.value,Linear,weight,"[320, 768]",245760,245760,0.0
+154,bert.encoder.layer.9.attention.self.value,Linear,bias,[320],320,320,0.0
+155,bert.encoder.layer.9.attention.output.dense,Linear,weight,"[768, 320]",245760,245760,0.0
+156,bert.encoder.layer.9.attention.output.dense,Linear,bias,[768],768,768,0.0
+157,bert.encoder.layer.9.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+158,bert.encoder.layer.9.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+159,bert.encoder.layer.9.intermediate.dense,Linear,weight,"[53, 768]",40704,40704,5.960464477539063e-08
+160,bert.encoder.layer.9.intermediate.dense,Linear,bias,[53],53,53,0.0
+161,bert.encoder.layer.9.output.dense,Linear,weight,"[768, 53]",40704,40704,5.960464477539063e-08
+162,bert.encoder.layer.9.output.dense,Linear,bias,[768],768,768,0.0
+163,bert.encoder.layer.9.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+164,bert.encoder.layer.9.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+165,bert.encoder.layer.10.attention.self.query,Linear,weight,"[384, 768]",294912,294912,0.0
+166,bert.encoder.layer.10.attention.self.query,Linear,bias,[384],384,384,0.0
+167,bert.encoder.layer.10.attention.self.key,Linear,weight,"[384, 768]",294912,294912,0.0
+168,bert.encoder.layer.10.attention.self.key,Linear,bias,[384],384,384,0.0
+169,bert.encoder.layer.10.attention.self.value,Linear,weight,"[384, 768]",294912,294912,0.0
+170,bert.encoder.layer.10.attention.self.value,Linear,bias,[384],384,384,0.0
+171,bert.encoder.layer.10.attention.output.dense,Linear,weight,"[768, 384]",294912,294912,0.0
+172,bert.encoder.layer.10.attention.output.dense,Linear,bias,[768],768,768,0.0
+173,bert.encoder.layer.10.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+174,bert.encoder.layer.10.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+175,bert.encoder.layer.10.intermediate.dense,Linear,weight,"[86, 768]",66048,66048,0.0
+176,bert.encoder.layer.10.intermediate.dense,Linear,bias,[86],86,86,0.0
+177,bert.encoder.layer.10.output.dense,Linear,weight,"[768, 86]",66048,66048,0.0
+178,bert.encoder.layer.10.output.dense,Linear,bias,[768],768,768,0.0
+179,bert.encoder.layer.10.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+180,bert.encoder.layer.10.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+181,bert.encoder.layer.11.attention.self.query,Linear,weight,"[384, 768]",294912,294912,0.0
+182,bert.encoder.layer.11.attention.self.query,Linear,bias,[384],384,384,0.0
+183,bert.encoder.layer.11.attention.self.key,Linear,weight,"[384, 768]",294912,294912,0.0
+184,bert.encoder.layer.11.attention.self.key,Linear,bias,[384],384,384,0.0
+185,bert.encoder.layer.11.attention.self.value,Linear,weight,"[384, 768]",294912,294912,0.0
+186,bert.encoder.layer.11.attention.self.value,Linear,bias,[384],384,384,0.0
+187,bert.encoder.layer.11.attention.output.dense,Linear,weight,"[768, 384]",294912,294912,0.0
+188,bert.encoder.layer.11.attention.output.dense,Linear,bias,[768],768,768,0.0
+189,bert.encoder.layer.11.attention.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+190,bert.encoder.layer.11.attention.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+191,bert.encoder.layer.11.intermediate.dense,Linear,weight,"[105, 768]",80640,80640,0.0
+192,bert.encoder.layer.11.intermediate.dense,Linear,bias,[105],105,105,0.0
+193,bert.encoder.layer.11.output.dense,Linear,weight,"[768, 105]",80640,80640,0.0
+194,bert.encoder.layer.11.output.dense,Linear,bias,[768],768,768,0.0
+195,bert.encoder.layer.11.output.LayerNorm,LayerNorm,weight,[768],768,768,0.0
+196,bert.encoder.layer.11.output.LayerNorm,LayerNorm,bias,[768],768,768,0.0
+197,qa_outputs,Linear,weight,"[2, 768]",1536,1536,0.0
+198,qa_outputs,Linear,bias,[2],2,2,0.0

XP_layer_wise_sparsity_global_rate_0.00.md

@@ -0,0 +1,201 @@
+|     | layer_id                                         | layer_type   | param_type   | shape        |   nparam |      nnz |    sparsity |
+|----:|:-------------------------------------------------|:-------------|:-------------|:-------------|---------:|---------:|------------:|
+|   0 | bert.embeddings.word_embeddings                  | Embedding    | weight       | [30522, 768] | 23440896 | 23440896 | 0           |
+|   1 | bert.embeddings.position_embeddings              | Embedding    | weight       | [512, 768]   |   393216 |   393216 | 0           |
+|   2 | bert.embeddings.token_type_embeddings            | Embedding    | weight       | [2, 768]     |     1536 |     1536 | 0           |
+|   3 | bert.embeddings.LayerNorm                        | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|   4 | bert.embeddings.LayerNorm                        | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|   5 | bert.encoder.layer.0.attention.self.query        | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+|   6 | bert.encoder.layer.0.attention.self.query        | Linear       | bias         | [320]        |      320 |      320 | 0           |
+|   7 | bert.encoder.layer.0.attention.self.key          | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+|   8 | bert.encoder.layer.0.attention.self.key          | Linear       | bias         | [320]        |      320 |      320 | 0           |
+|   9 | bert.encoder.layer.0.attention.self.value        | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+|  10 | bert.encoder.layer.0.attention.self.value        | Linear       | bias         | [320]        |      320 |      320 | 0           |
+|  11 | bert.encoder.layer.0.attention.output.dense      | Linear       | weight       | [768, 320]   |   245760 |   245760 | 0           |
+|  12 | bert.encoder.layer.0.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  13 | bert.encoder.layer.0.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  14 | bert.encoder.layer.0.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  15 | bert.encoder.layer.0.intermediate.dense          | Linear       | weight       | [185, 768]   |   142080 |   142080 | 0           |
+|  16 | bert.encoder.layer.0.intermediate.dense          | Linear       | bias         | [185]        |      185 |      185 | 0           |
+|  17 | bert.encoder.layer.0.output.dense                | Linear       | weight       | [768, 185]   |   142080 |   142080 | 0           |
+|  18 | bert.encoder.layer.0.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  19 | bert.encoder.layer.0.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  20 | bert.encoder.layer.0.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  21 | bert.encoder.layer.1.attention.self.query        | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+|  22 | bert.encoder.layer.1.attention.self.query        | Linear       | bias         | [320]        |      320 |      320 | 0           |
+|  23 | bert.encoder.layer.1.attention.self.key          | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+|  24 | bert.encoder.layer.1.attention.self.key          | Linear       | bias         | [320]        |      320 |      320 | 0           |
+|  25 | bert.encoder.layer.1.attention.self.value        | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+|  26 | bert.encoder.layer.1.attention.self.value        | Linear       | bias         | [320]        |      320 |      320 | 0           |
+|  27 | bert.encoder.layer.1.attention.output.dense      | Linear       | weight       | [768, 320]   |   245760 |   245760 | 0           |
+|  28 | bert.encoder.layer.1.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  29 | bert.encoder.layer.1.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  30 | bert.encoder.layer.1.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  31 | bert.encoder.layer.1.intermediate.dense          | Linear       | weight       | [315, 768]   |   241920 |   241920 | 0           |
+|  32 | bert.encoder.layer.1.intermediate.dense          | Linear       | bias         | [315]        |      315 |      315 | 0           |
+|  33 | bert.encoder.layer.1.output.dense                | Linear       | weight       | [768, 315]   |   241920 |   241920 | 0           |
+|  34 | bert.encoder.layer.1.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  35 | bert.encoder.layer.1.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  36 | bert.encoder.layer.1.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  37 | bert.encoder.layer.2.attention.self.query        | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  38 | bert.encoder.layer.2.attention.self.query        | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  39 | bert.encoder.layer.2.attention.self.key          | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  40 | bert.encoder.layer.2.attention.self.key          | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  41 | bert.encoder.layer.2.attention.self.value        | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  42 | bert.encoder.layer.2.attention.self.value        | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  43 | bert.encoder.layer.2.attention.output.dense      | Linear       | weight       | [768, 576]   |   442368 |   442368 | 0           |
+|  44 | bert.encoder.layer.2.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  45 | bert.encoder.layer.2.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  46 | bert.encoder.layer.2.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  47 | bert.encoder.layer.2.intermediate.dense          | Linear       | weight       | [339, 768]   |   260352 |   260352 | 0           |
+|  48 | bert.encoder.layer.2.intermediate.dense          | Linear       | bias         | [339]        |      339 |      339 | 0           |
+|  49 | bert.encoder.layer.2.output.dense                | Linear       | weight       | [768, 339]   |   260352 |   260352 | 0           |
+|  50 | bert.encoder.layer.2.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  51 | bert.encoder.layer.2.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  52 | bert.encoder.layer.2.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  53 | bert.encoder.layer.3.attention.self.query        | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  54 | bert.encoder.layer.3.attention.self.query        | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  55 | bert.encoder.layer.3.attention.self.key          | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  56 | bert.encoder.layer.3.attention.self.key          | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  57 | bert.encoder.layer.3.attention.self.value        | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  58 | bert.encoder.layer.3.attention.self.value        | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  59 | bert.encoder.layer.3.attention.output.dense      | Linear       | weight       | [768, 576]   |   442368 |   442368 | 0           |
+|  60 | bert.encoder.layer.3.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  61 | bert.encoder.layer.3.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  62 | bert.encoder.layer.3.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  63 | bert.encoder.layer.3.intermediate.dense          | Linear       | weight       | [368, 768]   |   282624 |   282624 | 0           |
+|  64 | bert.encoder.layer.3.intermediate.dense          | Linear       | bias         | [368]        |      368 |      368 | 0           |
+|  65 | bert.encoder.layer.3.output.dense                | Linear       | weight       | [768, 368]   |   282624 |   282624 | 0           |
+|  66 | bert.encoder.layer.3.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  67 | bert.encoder.layer.3.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  68 | bert.encoder.layer.3.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  69 | bert.encoder.layer.4.attention.self.query        | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  70 | bert.encoder.layer.4.attention.self.query        | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  71 | bert.encoder.layer.4.attention.self.key          | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  72 | bert.encoder.layer.4.attention.self.key          | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  73 | bert.encoder.layer.4.attention.self.value        | Linear       | weight       | [576, 768]   |   442368 |   442368 | 0           |
+|  74 | bert.encoder.layer.4.attention.self.value        | Linear       | bias         | [576]        |      576 |      576 | 0           |
+|  75 | bert.encoder.layer.4.attention.output.dense      | Linear       | weight       | [768, 576]   |   442368 |   442368 | 0           |
+|  76 | bert.encoder.layer.4.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  77 | bert.encoder.layer.4.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  78 | bert.encoder.layer.4.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  79 | bert.encoder.layer.4.intermediate.dense          | Linear       | weight       | [386, 768]   |   296448 |   296448 | 0           |
+|  80 | bert.encoder.layer.4.intermediate.dense          | Linear       | bias         | [386]        |      386 |      386 | 0           |
+|  81 | bert.encoder.layer.4.output.dense                | Linear       | weight       | [768, 386]   |   296448 |   296448 | 0           |
+|  82 | bert.encoder.layer.4.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  83 | bert.encoder.layer.4.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  84 | bert.encoder.layer.4.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  85 | bert.encoder.layer.5.attention.self.query        | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+|  86 | bert.encoder.layer.5.attention.self.query        | Linear       | bias         | [384]        |      384 |      384 | 0           |
+|  87 | bert.encoder.layer.5.attention.self.key          | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+|  88 | bert.encoder.layer.5.attention.self.key          | Linear       | bias         | [384]        |      384 |      384 | 0           |
+|  89 | bert.encoder.layer.5.attention.self.value        | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+|  90 | bert.encoder.layer.5.attention.self.value        | Linear       | bias         | [384]        |      384 |      384 | 0           |
+|  91 | bert.encoder.layer.5.attention.output.dense      | Linear       | weight       | [768, 384]   |   294912 |   294912 | 0           |
+|  92 | bert.encoder.layer.5.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  93 | bert.encoder.layer.5.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+|  94 | bert.encoder.layer.5.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+|  95 | bert.encoder.layer.5.intermediate.dense          | Linear       | weight       | [336, 768]   |   258048 |   258048 | 0           |
+|  96 | bert.encoder.layer.5.intermediate.dense          | Linear       | bias         | [336]        |      336 |      336 | 0           |
+|  97 | bert.encoder.layer.5.output.dense                | Linear       | weight       | [768, 336]   |   258048 |   258048 | 0           |
+|  98 | bert.encoder.layer.5.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+|  99 | bert.encoder.layer.5.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 100 | bert.encoder.layer.5.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 101 | bert.encoder.layer.6.attention.self.query        | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 102 | bert.encoder.layer.6.attention.self.query        | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 103 | bert.encoder.layer.6.attention.self.key          | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 104 | bert.encoder.layer.6.attention.self.key          | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 105 | bert.encoder.layer.6.attention.self.value        | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 106 | bert.encoder.layer.6.attention.self.value        | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 107 | bert.encoder.layer.6.attention.output.dense      | Linear       | weight       | [768, 448]   |   344064 |   344064 | 0           |
+| 108 | bert.encoder.layer.6.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 109 | bert.encoder.layer.6.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 110 | bert.encoder.layer.6.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 111 | bert.encoder.layer.6.intermediate.dense          | Linear       | weight       | [280, 768]   |   215040 |   215040 | 0           |
+| 112 | bert.encoder.layer.6.intermediate.dense          | Linear       | bias         | [280]        |      280 |      280 | 0           |
+| 113 | bert.encoder.layer.6.output.dense                | Linear       | weight       | [768, 280]   |   215040 |   215040 | 0           |
+| 114 | bert.encoder.layer.6.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 115 | bert.encoder.layer.6.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 116 | bert.encoder.layer.6.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 117 | bert.encoder.layer.7.attention.self.query        | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 118 | bert.encoder.layer.7.attention.self.query        | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 119 | bert.encoder.layer.7.attention.self.key          | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 120 | bert.encoder.layer.7.attention.self.key          | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 121 | bert.encoder.layer.7.attention.self.value        | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 122 | bert.encoder.layer.7.attention.self.value        | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 123 | bert.encoder.layer.7.attention.output.dense      | Linear       | weight       | [768, 448]   |   344064 |   344064 | 0           |
+| 124 | bert.encoder.layer.7.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 125 | bert.encoder.layer.7.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 126 | bert.encoder.layer.7.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 127 | bert.encoder.layer.7.intermediate.dense          | Linear       | weight       | [211, 768]   |   162048 |   162048 | 0           |
+| 128 | bert.encoder.layer.7.intermediate.dense          | Linear       | bias         | [211]        |      211 |      211 | 0           |
+| 129 | bert.encoder.layer.7.output.dense                | Linear       | weight       | [768, 211]   |   162048 |   162048 | 0           |
+| 130 | bert.encoder.layer.7.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 131 | bert.encoder.layer.7.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 132 | bert.encoder.layer.7.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 133 | bert.encoder.layer.8.attention.self.query        | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 134 | bert.encoder.layer.8.attention.self.query        | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 135 | bert.encoder.layer.8.attention.self.key          | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 136 | bert.encoder.layer.8.attention.self.key          | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 137 | bert.encoder.layer.8.attention.self.value        | Linear       | weight       | [448, 768]   |   344064 |   344064 | 0           |
+| 138 | bert.encoder.layer.8.attention.self.value        | Linear       | bias         | [448]        |      448 |      448 | 0           |
+| 139 | bert.encoder.layer.8.attention.output.dense      | Linear       | weight       | [768, 448]   |   344064 |   344064 | 0           |
+| 140 | bert.encoder.layer.8.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 141 | bert.encoder.layer.8.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 142 | bert.encoder.layer.8.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 143 | bert.encoder.layer.8.intermediate.dense          | Linear       | weight       | [108, 768]   |    82944 |    82944 | 0           |
+| 144 | bert.encoder.layer.8.intermediate.dense          | Linear       | bias         | [108]        |      108 |      108 | 0           |
+| 145 | bert.encoder.layer.8.output.dense                | Linear       | weight       | [768, 108]   |    82944 |    82944 | 0           |
+| 146 | bert.encoder.layer.8.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 147 | bert.encoder.layer.8.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 148 | bert.encoder.layer.8.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 149 | bert.encoder.layer.9.attention.self.query        | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+| 150 | bert.encoder.layer.9.attention.self.query        | Linear       | bias         | [320]        |      320 |      320 | 0           |
+| 151 | bert.encoder.layer.9.attention.self.key          | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+| 152 | bert.encoder.layer.9.attention.self.key          | Linear       | bias         | [320]        |      320 |      320 | 0           |
+| 153 | bert.encoder.layer.9.attention.self.value        | Linear       | weight       | [320, 768]   |   245760 |   245760 | 0           |
+| 154 | bert.encoder.layer.9.attention.self.value        | Linear       | bias         | [320]        |      320 |      320 | 0           |
+| 155 | bert.encoder.layer.9.attention.output.dense      | Linear       | weight       | [768, 320]   |   245760 |   245760 | 0           |
+| 156 | bert.encoder.layer.9.attention.output.dense      | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 157 | bert.encoder.layer.9.attention.output.LayerNorm  | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 158 | bert.encoder.layer.9.attention.output.LayerNorm  | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 159 | bert.encoder.layer.9.intermediate.dense          | Linear       | weight       | [53, 768]    |    40704 |    40704 | 5.96046e-08 |
+| 160 | bert.encoder.layer.9.intermediate.dense          | Linear       | bias         | [53]         |       53 |       53 | 0           |
+| 161 | bert.encoder.layer.9.output.dense                | Linear       | weight       | [768, 53]    |    40704 |    40704 | 5.96046e-08 |
+| 162 | bert.encoder.layer.9.output.dense                | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 163 | bert.encoder.layer.9.output.LayerNorm            | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 164 | bert.encoder.layer.9.output.LayerNorm            | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 165 | bert.encoder.layer.10.attention.self.query       | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+| 166 | bert.encoder.layer.10.attention.self.query       | Linear       | bias         | [384]        |      384 |      384 | 0           |
+| 167 | bert.encoder.layer.10.attention.self.key         | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+| 168 | bert.encoder.layer.10.attention.self.key         | Linear       | bias         | [384]        |      384 |      384 | 0           |
+| 169 | bert.encoder.layer.10.attention.self.value       | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+| 170 | bert.encoder.layer.10.attention.self.value       | Linear       | bias         | [384]        |      384 |      384 | 0           |
+| 171 | bert.encoder.layer.10.attention.output.dense     | Linear       | weight       | [768, 384]   |   294912 |   294912 | 0           |
+| 172 | bert.encoder.layer.10.attention.output.dense     | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 173 | bert.encoder.layer.10.attention.output.LayerNorm | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 174 | bert.encoder.layer.10.attention.output.LayerNorm | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 175 | bert.encoder.layer.10.intermediate.dense         | Linear       | weight       | [86, 768]    |    66048 |    66048 | 0           |
+| 176 | bert.encoder.layer.10.intermediate.dense         | Linear       | bias         | [86]         |       86 |       86 | 0           |
+| 177 | bert.encoder.layer.10.output.dense               | Linear       | weight       | [768, 86]    |    66048 |    66048 | 0           |
+| 178 | bert.encoder.layer.10.output.dense               | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 179 | bert.encoder.layer.10.output.LayerNorm           | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 180 | bert.encoder.layer.10.output.LayerNorm           | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 181 | bert.encoder.layer.11.attention.self.query       | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+| 182 | bert.encoder.layer.11.attention.self.query       | Linear       | bias         | [384]        |      384 |      384 | 0           |
+| 183 | bert.encoder.layer.11.attention.self.key         | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+| 184 | bert.encoder.layer.11.attention.self.key         | Linear       | bias         | [384]        |      384 |      384 | 0           |
+| 185 | bert.encoder.layer.11.attention.self.value       | Linear       | weight       | [384, 768]   |   294912 |   294912 | 0           |
+| 186 | bert.encoder.layer.11.attention.self.value       | Linear       | bias         | [384]        |      384 |      384 | 0           |
+| 187 | bert.encoder.layer.11.attention.output.dense     | Linear       | weight       | [768, 384]   |   294912 |   294912 | 0           |
+| 188 | bert.encoder.layer.11.attention.output.dense     | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 189 | bert.encoder.layer.11.attention.output.LayerNorm | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 190 | bert.encoder.layer.11.attention.output.LayerNorm | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 191 | bert.encoder.layer.11.intermediate.dense         | Linear       | weight       | [105, 768]   |    80640 |    80640 | 0           |
+| 192 | bert.encoder.layer.11.intermediate.dense         | Linear       | bias         | [105]        |      105 |      105 | 0           |
+| 193 | bert.encoder.layer.11.output.dense               | Linear       | weight       | [768, 105]   |    80640 |    80640 | 0           |
+| 194 | bert.encoder.layer.11.output.dense               | Linear       | bias         | [768]        |      768 |      768 | 0           |
+| 195 | bert.encoder.layer.11.output.LayerNorm           | LayerNorm    | weight       | [768]        |      768 |      768 | 0           |
+| 196 | bert.encoder.layer.11.output.LayerNorm           | LayerNorm    | bias         | [768]        |      768 |      768 | 0           |
+| 197 | qa_outputs                                       | Linear       | weight       | [2, 768]     |     1536 |     1536 | 0           |
+| 198 | qa_outputs                                       | Linear       | bias         | [2]          |        2 |        2 | 0           |

XP_linear_layer_sparsity_20M_params_0.00_sparsity.csv

@@ -0,0 +1,73 @@
+,layer_id,layer_type,param_type,shape,nparam,nnz,sparsity
+5,bert.encoder.layer.0.attention.self.query,Linear,weight,"[320, 768]",245760,245760,0.0
+7,bert.encoder.layer.0.attention.self.key,Linear,weight,"[320, 768]",245760,245760,0.0
+9,bert.encoder.layer.0.attention.self.value,Linear,weight,"[320, 768]",245760,245760,0.0
+11,bert.encoder.layer.0.attention.output.dense,Linear,weight,"[768, 320]",245760,245760,0.0
+15,bert.encoder.layer.0.intermediate.dense,Linear,weight,"[185, 768]",142080,142080,0.0
+17,bert.encoder.layer.0.output.dense,Linear,weight,"[768, 185]",142080,142080,0.0
+21,bert.encoder.layer.1.attention.self.query,Linear,weight,"[320, 768]",245760,245760,0.0
+23,bert.encoder.layer.1.attention.self.key,Linear,weight,"[320, 768]",245760,245760,0.0
+25,bert.encoder.layer.1.attention.self.value,Linear,weight,"[320, 768]",245760,245760,0.0
+27,bert.encoder.layer.1.attention.output.dense,Linear,weight,"[768, 320]",245760,245760,0.0
+31,bert.encoder.layer.1.intermediate.dense,Linear,weight,"[315, 768]",241920,241920,0.0
+33,bert.encoder.layer.1.output.dense,Linear,weight,"[768, 315]",241920,241920,0.0
+37,bert.encoder.layer.2.attention.self.query,Linear,weight,"[576, 768]",442368,442368,0.0
+39,bert.encoder.layer.2.attention.self.key,Linear,weight,"[576, 768]",442368,442368,0.0
+41,bert.encoder.layer.2.attention.self.value,Linear,weight,"[576, 768]",442368,442368,0.0
+43,bert.encoder.layer.2.attention.output.dense,Linear,weight,"[768, 576]",442368,442368,0.0
+47,bert.encoder.layer.2.intermediate.dense,Linear,weight,"[339, 768]",260352,260352,0.0
+49,bert.encoder.layer.2.output.dense,Linear,weight,"[768, 339]",260352,260352,0.0
+53,bert.encoder.layer.3.attention.self.query,Linear,weight,"[576, 768]",442368,442368,0.0
+55,bert.encoder.layer.3.attention.self.key,Linear,weight,"[576, 768]",442368,442368,0.0
+57,bert.encoder.layer.3.attention.self.value,Linear,weight,"[576, 768]",442368,442368,0.0
+59,bert.encoder.layer.3.attention.output.dense,Linear,weight,"[768, 576]",442368,442368,0.0
+63,bert.encoder.layer.3.intermediate.dense,Linear,weight,"[368, 768]",282624,282624,0.0
+65,bert.encoder.layer.3.output.dense,Linear,weight,"[768, 368]",282624,282624,0.0
+69,bert.encoder.layer.4.attention.self.query,Linear,weight,"[576, 768]",442368,442368,0.0
+71,bert.encoder.layer.4.attention.self.key,Linear,weight,"[576, 768]",442368,442368,0.0
+73,bert.encoder.layer.4.attention.self.value,Linear,weight,"[576, 768]",442368,442368,0.0
+75,bert.encoder.layer.4.attention.output.dense,Linear,weight,"[768, 576]",442368,442368,0.0
+79,bert.encoder.layer.4.intermediate.dense,Linear,weight,"[386, 768]",296448,296448,0.0
+81,bert.encoder.layer.4.output.dense,Linear,weight,"[768, 386]",296448,296448,0.0
+85,bert.encoder.layer.5.attention.self.query,Linear,weight,"[384, 768]",294912,294912,0.0
+87,bert.encoder.layer.5.attention.self.key,Linear,weight,"[384, 768]",294912,294912,0.0
+89,bert.encoder.layer.5.attention.self.value,Linear,weight,"[384, 768]",294912,294912,0.0
+91,bert.encoder.layer.5.attention.output.dense,Linear,weight,"[768, 384]",294912,294912,0.0
+95,bert.encoder.layer.5.intermediate.dense,Linear,weight,"[336, 768]",258048,258048,0.0
+97,bert.encoder.layer.5.output.dense,Linear,weight,"[768, 336]",258048,258048,0.0
+101,bert.encoder.layer.6.attention.self.query,Linear,weight,"[448, 768]",344064,344064,0.0
+103,bert.encoder.layer.6.attention.self.key,Linear,weight,"[448, 768]",344064,344064,0.0
+105,bert.encoder.layer.6.attention.self.value,Linear,weight,"[448, 768]",344064,344064,0.0
+107,bert.encoder.layer.6.attention.output.dense,Linear,weight,"[768, 448]",344064,344064,0.0
+111,bert.encoder.layer.6.intermediate.dense,Linear,weight,"[280, 768]",215040,215040,0.0
+113,bert.encoder.layer.6.output.dense,Linear,weight,"[768, 280]",215040,215040,0.0
+117,bert.encoder.layer.7.attention.self.query,Linear,weight,"[448, 768]",344064,344064,0.0
+119,bert.encoder.layer.7.attention.self.key,Linear,weight,"[448, 768]",344064,344064,0.0
+121,bert.encoder.layer.7.attention.self.value,Linear,weight,"[448, 768]",344064,344064,0.0
+123,bert.encoder.layer.7.attention.output.dense,Linear,weight,"[768, 448]",344064,344064,0.0
+127,bert.encoder.layer.7.intermediate.dense,Linear,weight,"[211, 768]",162048,162048,0.0
+129,bert.encoder.layer.7.output.dense,Linear,weight,"[768, 211]",162048,162048,0.0
+133,bert.encoder.layer.8.attention.self.query,Linear,weight,"[448, 768]",344064,344064,0.0
+135,bert.encoder.layer.8.attention.self.key,Linear,weight,"[448, 768]",344064,344064,0.0
+137,bert.encoder.layer.8.attention.self.value,Linear,weight,"[448, 768]",344064,344064,0.0
+139,bert.encoder.layer.8.attention.output.dense,Linear,weight,"[768, 448]",344064,344064,0.0
+143,bert.encoder.layer.8.intermediate.dense,Linear,weight,"[108, 768]",82944,82944,0.0
+145,bert.encoder.layer.8.output.dense,Linear,weight,"[768, 108]",82944,82944,0.0
+149,bert.encoder.layer.9.attention.self.query,Linear,weight,"[320, 768]",245760,245760,0.0
+151,bert.encoder.layer.9.attention.self.key,Linear,weight,"[320, 768]",245760,245760,0.0
+153,bert.encoder.layer.9.attention.self.value,Linear,weight,"[320, 768]",245760,245760,0.0
+155,bert.encoder.layer.9.attention.output.dense,Linear,weight,"[768, 320]",245760,245760,0.0
+159,bert.encoder.layer.9.intermediate.dense,Linear,weight,"[53, 768]",40704,40704,5.960464477539063e-08
+161,bert.encoder.layer.9.output.dense,Linear,weight,"[768, 53]",40704,40704,5.960464477539063e-08
+165,bert.encoder.layer.10.attention.self.query,Linear,weight,"[384, 768]",294912,294912,0.0
+167,bert.encoder.layer.10.attention.self.key,Linear,weight,"[384, 768]",294912,294912,0.0
+169,bert.encoder.layer.10.attention.self.value,Linear,weight,"[384, 768]",294912,294912,0.0
+171,bert.encoder.layer.10.attention.output.dense,Linear,weight,"[768, 384]",294912,294912,0.0
+175,bert.encoder.layer.10.intermediate.dense,Linear,weight,"[86, 768]",66048,66048,0.0
+177,bert.encoder.layer.10.output.dense,Linear,weight,"[768, 86]",66048,66048,0.0
+181,bert.encoder.layer.11.attention.self.query,Linear,weight,"[384, 768]",294912,294912,0.0
+183,bert.encoder.layer.11.attention.self.key,Linear,weight,"[384, 768]",294912,294912,0.0
+185,bert.encoder.layer.11.attention.self.value,Linear,weight,"[384, 768]",294912,294912,0.0
+187,bert.encoder.layer.11.attention.output.dense,Linear,weight,"[768, 384]",294912,294912,0.0
+191,bert.encoder.layer.11.intermediate.dense,Linear,weight,"[105, 768]",80640,80640,0.0
+193,bert.encoder.layer.11.output.dense,Linear,weight,"[768, 105]",80640,80640,0.0

XP_linear_layer_sparsity_20M_params_0.00_sparsity.md

@@ -0,0 +1,74 @@
+|     | layer_id                                     | layer_type   | param_type   | shape      |   nparam |    nnz |    sparsity |
+|----:|:---------------------------------------------|:-------------|:-------------|:-----------|---------:|-------:|------------:|
+|   5 | bert.encoder.layer.0.attention.self.query    | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+|   7 | bert.encoder.layer.0.attention.self.key      | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+|   9 | bert.encoder.layer.0.attention.self.value    | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+|  11 | bert.encoder.layer.0.attention.output.dense  | Linear       | weight       | [768, 320] |   245760 | 245760 | 0           |
+|  15 | bert.encoder.layer.0.intermediate.dense      | Linear       | weight       | [185, 768] |   142080 | 142080 | 0           |
+|  17 | bert.encoder.layer.0.output.dense            | Linear       | weight       | [768, 185] |   142080 | 142080 | 0           |
+|  21 | bert.encoder.layer.1.attention.self.query    | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+|  23 | bert.encoder.layer.1.attention.self.key      | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+|  25 | bert.encoder.layer.1.attention.self.value    | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+|  27 | bert.encoder.layer.1.attention.output.dense  | Linear       | weight       | [768, 320] |   245760 | 245760 | 0           |
+|  31 | bert.encoder.layer.1.intermediate.dense      | Linear       | weight       | [315, 768] |   241920 | 241920 | 0           |
+|  33 | bert.encoder.layer.1.output.dense            | Linear       | weight       | [768, 315] |   241920 | 241920 | 0           |
+|  37 | bert.encoder.layer.2.attention.self.query    | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  39 | bert.encoder.layer.2.attention.self.key      | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  41 | bert.encoder.layer.2.attention.self.value    | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  43 | bert.encoder.layer.2.attention.output.dense  | Linear       | weight       | [768, 576] |   442368 | 442368 | 0           |
+|  47 | bert.encoder.layer.2.intermediate.dense      | Linear       | weight       | [339, 768] |   260352 | 260352 | 0           |
+|  49 | bert.encoder.layer.2.output.dense            | Linear       | weight       | [768, 339] |   260352 | 260352 | 0           |
+|  53 | bert.encoder.layer.3.attention.self.query    | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  55 | bert.encoder.layer.3.attention.self.key      | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  57 | bert.encoder.layer.3.attention.self.value    | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  59 | bert.encoder.layer.3.attention.output.dense  | Linear       | weight       | [768, 576] |   442368 | 442368 | 0           |
+|  63 | bert.encoder.layer.3.intermediate.dense      | Linear       | weight       | [368, 768] |   282624 | 282624 | 0           |
+|  65 | bert.encoder.layer.3.output.dense            | Linear       | weight       | [768, 368] |   282624 | 282624 | 0           |
+|  69 | bert.encoder.layer.4.attention.self.query    | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  71 | bert.encoder.layer.4.attention.self.key      | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  73 | bert.encoder.layer.4.attention.self.value    | Linear       | weight       | [576, 768] |   442368 | 442368 | 0           |
+|  75 | bert.encoder.layer.4.attention.output.dense  | Linear       | weight       | [768, 576] |   442368 | 442368 | 0           |
+|  79 | bert.encoder.layer.4.intermediate.dense      | Linear       | weight       | [386, 768] |   296448 | 296448 | 0           |
+|  81 | bert.encoder.layer.4.output.dense            | Linear       | weight       | [768, 386] |   296448 | 296448 | 0           |
+|  85 | bert.encoder.layer.5.attention.self.query    | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+|  87 | bert.encoder.layer.5.attention.self.key      | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+|  89 | bert.encoder.layer.5.attention.self.value    | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+|  91 | bert.encoder.layer.5.attention.output.dense  | Linear       | weight       | [768, 384] |   294912 | 294912 | 0           |
+|  95 | bert.encoder.layer.5.intermediate.dense      | Linear       | weight       | [336, 768] |   258048 | 258048 | 0           |
+|  97 | bert.encoder.layer.5.output.dense            | Linear       | weight       | [768, 336] |   258048 | 258048 | 0           |
+| 101 | bert.encoder.layer.6.attention.self.query    | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 103 | bert.encoder.layer.6.attention.self.key      | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 105 | bert.encoder.layer.6.attention.self.value    | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 107 | bert.encoder.layer.6.attention.output.dense  | Linear       | weight       | [768, 448] |   344064 | 344064 | 0           |
+| 111 | bert.encoder.layer.6.intermediate.dense      | Linear       | weight       | [280, 768] |   215040 | 215040 | 0           |
+| 113 | bert.encoder.layer.6.output.dense            | Linear       | weight       | [768, 280] |   215040 | 215040 | 0           |
+| 117 | bert.encoder.layer.7.attention.self.query    | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 119 | bert.encoder.layer.7.attention.self.key      | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 121 | bert.encoder.layer.7.attention.self.value    | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 123 | bert.encoder.layer.7.attention.output.dense  | Linear       | weight       | [768, 448] |   344064 | 344064 | 0           |
+| 127 | bert.encoder.layer.7.intermediate.dense      | Linear       | weight       | [211, 768] |   162048 | 162048 | 0           |
+| 129 | bert.encoder.layer.7.output.dense            | Linear       | weight       | [768, 211] |   162048 | 162048 | 0           |
+| 133 | bert.encoder.layer.8.attention.self.query    | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 135 | bert.encoder.layer.8.attention.self.key      | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 137 | bert.encoder.layer.8.attention.self.value    | Linear       | weight       | [448, 768] |   344064 | 344064 | 0           |
+| 139 | bert.encoder.layer.8.attention.output.dense  | Linear       | weight       | [768, 448] |   344064 | 344064 | 0           |
+| 143 | bert.encoder.layer.8.intermediate.dense      | Linear       | weight       | [108, 768] |    82944 |  82944 | 0           |
+| 145 | bert.encoder.layer.8.output.dense            | Linear       | weight       | [768, 108] |    82944 |  82944 | 0           |
+| 149 | bert.encoder.layer.9.attention.self.query    | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+| 151 | bert.encoder.layer.9.attention.self.key      | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+| 153 | bert.encoder.layer.9.attention.self.value    | Linear       | weight       | [320, 768] |   245760 | 245760 | 0           |
+| 155 | bert.encoder.layer.9.attention.output.dense  | Linear       | weight       | [768, 320] |   245760 | 245760 | 0           |
+| 159 | bert.encoder.layer.9.intermediate.dense      | Linear       | weight       | [53, 768]  |    40704 |  40704 | 5.96046e-08 |
+| 161 | bert.encoder.layer.9.output.dense            | Linear       | weight       | [768, 53]  |    40704 |  40704 | 5.96046e-08 |
+| 165 | bert.encoder.layer.10.attention.self.query   | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+| 167 | bert.encoder.layer.10.attention.self.key     | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+| 169 | bert.encoder.layer.10.attention.self.value   | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+| 171 | bert.encoder.layer.10.attention.output.dense | Linear       | weight       | [768, 384] |   294912 | 294912 | 0           |
+| 175 | bert.encoder.layer.10.intermediate.dense     | Linear       | weight       | [86, 768]  |    66048 |  66048 | 0           |
+| 177 | bert.encoder.layer.10.output.dense           | Linear       | weight       | [768, 86]  |    66048 |  66048 | 0           |
+| 181 | bert.encoder.layer.11.attention.self.query   | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+| 183 | bert.encoder.layer.11.attention.self.key     | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+| 185 | bert.encoder.layer.11.attention.self.value   | Linear       | weight       | [384, 768] |   294912 | 294912 | 0           |
+| 187 | bert.encoder.layer.11.attention.output.dense | Linear       | weight       | [768, 384] |   294912 | 294912 | 0           |
+| 191 | bert.encoder.layer.11.intermediate.dense     | Linear       | weight       | [105, 768] |    80640 |  80640 | 0           |
+| 193 | bert.encoder.layer.11.output.dense           | Linear       | weight       | [768, 105] |    80640 |  80640 | 0           |

all_results.json

@@ -0,0 +1,5 @@
+{
+    "eval_exact_match": 80.33112582781457,
+    "eval_f1": 87.69004456685987,
+    "eval_samples": 10784
+}

config.json

@@ -0,0 +1,115 @@
+{
+  "_name_or_path": "/home/vchua/blk-prune/nn_pruning/reproduce-eval/question_answering/tmp_finetune",
+  "architectures": [
+    "BertForQuestionAnswering"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "pruned_heads": {
+    "0": [
+      0,
+      2,
+      4,
+      5,
+      6,
+      7,
+      11
+    ],
+    "1": [
+      0,
+      2,
+      3,
+      5,
+      6,
+      7,
+      8
+    ],
+    "2": [
+      8,
+      4,
+      7
+    ],
+    "3": [
+      2,
+      4,
+      6
+    ],
+    "4": [
+      1,
+      2,
+      11
+    ],
+    "5": [
+      1,
+      2,
+      5,
+      6,
+      7,
+      11
+    ],
+    "6": [
+      0,
+      2,
+      3,
+      7,
+      10
+    ],
+    "7": [
+      1,
+      3,
+      6,
+      7,
+      11
+    ],
+    "8": [
+      0,
+      3,
+      4,
+      5,
+      8
+    ],
+    "9": [
+      1,
+      3,
+      4,
+      5,
+      7,
+      9,
+      10
+    ],
+    "10": [
+      1,
+      4,
+      5,
+      6,
+      7,
+      8
+    ],
+    "11": [
+      4,
+      5,
+      7,
+      8,
+      10,
+      11
+    ]
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.10.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

data_args.json

@@ -0,0 +1,16 @@
+{
+    "dataset_cache_dir": "dataset_cache",
+    "dataset_config_name": null,
+    "dataset_name": "squad",
+    "doc_stride": 128,
+    "max_answer_length": 30,
+    "max_seq_length": 384,
+    "n_best_size": 20,
+    "null_score_diff_threshold": 0.0,
+    "overwrite_cache": 0,
+    "pad_to_max_length": true,
+    "preprocessing_num_workers": null,
+    "train_file": null,
+    "validation_file": null,
+    "version_2_with_negative": 0
+}

eval_XP_results.json

@@ -0,0 +1,5 @@
+{
+    "eval_exact_match": 80.33112582781457,
+    "eval_f1": 87.69004456685987,
+    "eval_samples": 10784
+}

eval_metrics.json

@@ -0,0 +1,4 @@
+{
+    "exact_match": 78.32544938505204,
+    "f1": 86.24638521438392
+}

eval_nbest_predictions.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a033915d48e30fc3f40b9a7bc5da2f9fea134b7da0030e12c57b81629ca00dc
+size 48965701

evaluate_timing.json

@@ -0,0 +1 @@
+{"eval_elapsed_time": 44.88275432190858, "cuda_eval_elapsed_time": 41.537567138671875}

model_args.json

@@ -0,0 +1,7 @@
+{
+    "cache_dir": null,
+    "config_name": null,
+    "model_name_or_path": "bert-base-uncased",
+    "tokenizer_name": null,
+    "use_fast_tokenizer": true
+}

nbest_predictions.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be6e27bb645dab0baa406b39378effcee1dbf2e2d4f4f07fe42d5ecf70fc3bae
+size 48942794

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac6a31ff182b9879808f0bef0a56fb3bee8be55e15c73d36eadd865ce63c0ca1
+size 871812469

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:868e4add2cd6b65ee381e9b92f4d875db38fbb31b6fabbe1d1b775e107b10b58
+size 176464689

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38c3cad2f3a30a6ac205297d841b5188c144be26d3b7b0b4fb42c6d126a49102
+size 14567

sparse_args.json

@@ -0,0 +1,41 @@
+{
+    "ampere_pruning_method": "disabled",
+    "attention_block_cols": 32,
+    "attention_block_rows": 32,
+    "attention_lambda": 1.0,
+    "attention_output_with_dense": 0,
+    "attention_pruning_method": "sigmoied_threshold",
+    "bias_mask": true,
+    "decoder_attention_lambda": null,
+    "decoder_dense_lambda": null,
+    "dense_block_cols": 1,
+    "dense_block_rows": 1,
+    "dense_lambda": 1.0,
+    "dense_pruning_method": "sigmoied_threshold:1d_alt",
+    "distil_alpha_ce": 0.1,
+    "distil_alpha_teacher": 0.9,
+    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
+    "distil_temperature": 2.0,
+    "eval_with_current_patch_params": false,
+    "final_ampere_temperature": 20.0,
+    "final_finetune": false,
+    "final_threshold": 0.1,
+    "final_warmup": 10,
+    "gelu_patch": false,
+    "gelu_patch_steps": 50000,
+    "initial_ampere_temperature": 0.0,
+    "initial_threshold": 0,
+    "initial_warmup": 1,
+    "layer_norm_patch": false,
+    "layer_norm_patch_start_delta": 0.99,
+    "layer_norm_patch_steps": 50000,
+    "linear_min_parameters": 0,
+    "mask_init": "constant",
+    "mask_scale": 0.0,
+    "mask_scores_learning_rate": 0.01,
+    "qat": false,
+    "qconfig": "default",
+    "regularization": "l1",
+    "regularization_final_lambda": 20.0,
+    "rewind_model_name_or_path": null
+}

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "/home/vchua/blk-prune/nn_pruning/reproduce-eval/question_answering/tmp_finetune", "tokenizer_class": "BertTokenizer"}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c589bae8acd6247c9250208506ae4c47488ef5615b8596c1707e96e327bdc44
+size 2863