| { | |
| "_name_or_path": "../plms/bert-base-uncased", | |
| "architectures": [ | |
| "SparseBertMiniLM" | |
| ], | |
| "attention_head_size": 64, | |
| "attention_probs_dropout_prob": 0.1, | |
| "classifier_dropout": null, | |
| "gradient_checkpointing": false, | |
| "hidden_act": "gelu", | |
| "hidden_dropout_prob": 0.1, | |
| "hidden_size": 768, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 3072, | |
| "layer_norm_eps": 1e-12, | |
| "layer_skip": 1, | |
| "max_position_embeddings": 512, | |
| "model_type": "bert", | |
| "num_attention_heads": 12, | |
| "num_hidden_layers": 6, | |
| "num_relation_heads": 32, | |
| "pad_token_id": 0, | |
| "position_embedding_type": "absolute", | |
| "sparsity": "6,768", | |
| "sparsity_map": { | |
| "3,192": { | |
| "head": { | |
| "0": 9, | |
| "1": 9, | |
| "2": 9, | |
| "3": 12, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 576, | |
| "0": 576, | |
| "1": 576, | |
| "2": 576, | |
| "3": 576, | |
| "4": 576, | |
| "5": 576 | |
| }, | |
| "neuron": { | |
| "0": 2304, | |
| "1": 2304, | |
| "2": 2304, | |
| "3": 3072, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "3,256": { | |
| "head": { | |
| "0": 8, | |
| "1": 8, | |
| "2": 8, | |
| "3": 12, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 512, | |
| "0": 512, | |
| "1": 512, | |
| "2": 512, | |
| "3": 512, | |
| "4": 512, | |
| "5": 512 | |
| }, | |
| "neuron": { | |
| "0": 2048, | |
| "1": 2048, | |
| "2": 2048, | |
| "3": 3072, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "3,384": { | |
| "head": { | |
| "0": 6, | |
| "1": 6, | |
| "2": 6, | |
| "3": 12, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 384, | |
| "0": 384, | |
| "1": 384, | |
| "2": 384, | |
| "3": 384, | |
| "4": 384, | |
| "5": 384 | |
| }, | |
| "neuron": { | |
| "0": 1536, | |
| "1": 1536, | |
| "2": 1536, | |
| "3": 3072, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "3,512": { | |
| "head": { | |
| "0": 4, | |
| "1": 4, | |
| "2": 4, | |
| "3": 12, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 256, | |
| "0": 256, | |
| "1": 256, | |
| "2": 256, | |
| "3": 256, | |
| "4": 256, | |
| "5": 256 | |
| }, | |
| "neuron": { | |
| "0": 1024, | |
| "1": 1024, | |
| "2": 1024, | |
| "3": 3072, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "3,768": { | |
| "head": { | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 12, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 0, | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 0, | |
| "4": 0, | |
| "5": 0 | |
| }, | |
| "neuron": { | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 3072, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "4,192": { | |
| "head": { | |
| "0": 9, | |
| "1": 9, | |
| "2": 9, | |
| "3": 9, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 576, | |
| "0": 576, | |
| "1": 576, | |
| "2": 576, | |
| "3": 576, | |
| "4": 576, | |
| "5": 576 | |
| }, | |
| "neuron": { | |
| "0": 2304, | |
| "1": 2304, | |
| "2": 2304, | |
| "3": 2304, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "4,256": { | |
| "head": { | |
| "0": 8, | |
| "1": 8, | |
| "2": 8, | |
| "3": 8, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 512, | |
| "0": 512, | |
| "1": 512, | |
| "2": 512, | |
| "3": 512, | |
| "4": 512, | |
| "5": 512 | |
| }, | |
| "neuron": { | |
| "0": 2048, | |
| "1": 2048, | |
| "2": 2048, | |
| "3": 2048, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "4,384": { | |
| "head": { | |
| "0": 6, | |
| "1": 6, | |
| "2": 6, | |
| "3": 6, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 384, | |
| "0": 384, | |
| "1": 384, | |
| "2": 384, | |
| "3": 384, | |
| "4": 384, | |
| "5": 384 | |
| }, | |
| "neuron": { | |
| "0": 1536, | |
| "1": 1536, | |
| "2": 1536, | |
| "3": 1536, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "4,512": { | |
| "head": { | |
| "0": 4, | |
| "1": 4, | |
| "2": 4, | |
| "3": 4, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 256, | |
| "0": 256, | |
| "1": 256, | |
| "2": 256, | |
| "3": 256, | |
| "4": 256, | |
| "5": 256 | |
| }, | |
| "neuron": { | |
| "0": 1024, | |
| "1": 1024, | |
| "2": 1024, | |
| "3": 1024, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "4,768": { | |
| "head": { | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 0, | |
| "4": 12, | |
| "5": 12 | |
| }, | |
| "hidden": { | |
| "-1": 0, | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 0, | |
| "4": 0, | |
| "5": 0 | |
| }, | |
| "neuron": { | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 0, | |
| "4": 3072, | |
| "5": 3072 | |
| } | |
| }, | |
| "6,192": { | |
| "head": { | |
| "0": 9, | |
| "1": 9, | |
| "2": 9, | |
| "3": 9, | |
| "4": 9, | |
| "5": 9 | |
| }, | |
| "hidden": { | |
| "-1": 576, | |
| "0": 576, | |
| "1": 576, | |
| "2": 576, | |
| "3": 576, | |
| "4": 576, | |
| "5": 576 | |
| }, | |
| "neuron": { | |
| "0": 2304, | |
| "1": 2304, | |
| "2": 2304, | |
| "3": 2304, | |
| "4": 2304, | |
| "5": 2304 | |
| } | |
| }, | |
| "6,256": { | |
| "head": { | |
| "0": 8, | |
| "1": 8, | |
| "2": 8, | |
| "3": 8, | |
| "4": 8, | |
| "5": 8 | |
| }, | |
| "hidden": { | |
| "-1": 512, | |
| "0": 512, | |
| "1": 512, | |
| "2": 512, | |
| "3": 512, | |
| "4": 512, | |
| "5": 512 | |
| }, | |
| "neuron": { | |
| "0": 2048, | |
| "1": 2048, | |
| "2": 2048, | |
| "3": 2048, | |
| "4": 2048, | |
| "5": 2048 | |
| } | |
| }, | |
| "6,384": { | |
| "head": { | |
| "0": 6, | |
| "1": 6, | |
| "2": 6, | |
| "3": 6, | |
| "4": 6, | |
| "5": 6 | |
| }, | |
| "hidden": { | |
| "-1": 384, | |
| "0": 384, | |
| "1": 384, | |
| "2": 384, | |
| "3": 384, | |
| "4": 384, | |
| "5": 384 | |
| }, | |
| "neuron": { | |
| "0": 1536, | |
| "1": 1536, | |
| "2": 1536, | |
| "3": 1536, | |
| "4": 1536, | |
| "5": 1536 | |
| } | |
| }, | |
| "6,512": { | |
| "head": { | |
| "0": 4, | |
| "1": 4, | |
| "2": 4, | |
| "3": 4, | |
| "4": 4, | |
| "5": 4 | |
| }, | |
| "hidden": { | |
| "-1": 256, | |
| "0": 256, | |
| "1": 256, | |
| "2": 256, | |
| "3": 256, | |
| "4": 256, | |
| "5": 256 | |
| }, | |
| "neuron": { | |
| "0": 1024, | |
| "1": 1024, | |
| "2": 1024, | |
| "3": 1024, | |
| "4": 1024, | |
| "5": 1024 | |
| } | |
| }, | |
| "6,768": { | |
| "head": { | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 0, | |
| "4": 0, | |
| "5": 0 | |
| }, | |
| "hidden": { | |
| "-1": 0, | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 0, | |
| "4": 0, | |
| "5": 0 | |
| }, | |
| "neuron": { | |
| "0": 0, | |
| "1": 0, | |
| "2": 0, | |
| "3": 0, | |
| "4": 0, | |
| "5": 0 | |
| } | |
| } | |
| }, | |
| "torch_dtype": "float32", | |
| "transformers_version": "4.9.2", | |
| "type_vocab_size": 2, | |
| "use_cache": true, | |
| "vocab_size": 30522 | |
| } | |