{ "_name_or_path": "ckpt-last", "architectures": [ "SparseBertMiniLM" ], "attention_probs_dropout_prob": 0.1, "classifier_dropout": null, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-12, "layer_skip": 1, "max_position_embeddings": 512, "model_type": "bert", "num_attention_heads": 12, "num_hidden_layers": 12, "num_relation_heads": 32, "pad_token_id": 0, "position_embedding_type": "absolute", "sparsity": "90", "sparsity_map": { "0": { "head": {}, "hidden": { "-1": 0, "0": 0, "1": 0, "10": 0, "11": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0 }, "neuron": {} }, "10": { "head": { "1": 5, "11": 1, "3": 1 }, "hidden": { "-1": 38, "0": 38, "1": 38, "10": 38, "11": 38, "2": 38, "3": 38, "4": 38, "5": 38, "6": 38, "7": 38, "8": 38, "9": 38 }, "neuron": { "0": 190, "1": 298, "11": 1, "2": 1323, "3": 4, "4": 5, "5": 8, "6": 4, "7": 7, "8": 2, "9": 1 } }, "20": { "head": { "0": 1, "1": 7, "10": 1, "11": 4, "3": 2, "9": 1 }, "hidden": { "-1": 84, "0": 84, "1": 84, "10": 84, "11": 84, "2": 84, "3": 84, "4": 84, "5": 84, "6": 84, "7": 84, "8": 84, "9": 84 }, "neuron": { "0": 597, "1": 802, "11": 16, "2": 2294, "3": 48, "4": 78, "5": 123, "6": 22, "7": 47, "8": 23, "9": 5 } }, "30": { "head": { "0": 1, "1": 8, "10": 2, "11": 4, "2": 1, "3": 2, "6": 1, "7": 1, "8": 1, "9": 2 }, "hidden": { "-1": 123, "0": 123, "1": 123, "10": 123, "11": 123, "2": 123, "3": 123, "4": 123, "5": 123, "6": 123, "7": 123, "8": 123, "9": 123 }, "neuron": { "0": 980, "1": 1233, "10": 2, "11": 116, "2": 2512, "3": 132, "4": 233, "5": 386, "6": 61, "7": 146, "8": 73, "9": 24 } }, "40": { "head": { "0": 3, "1": 8, "10": 2, "11": 5, "2": 1, "3": 3, "4": 2, "5": 1, "6": 2, "7": 1, "8": 1, "9": 4 }, "hidden": { "-1": 177, "0": 177, "1": 177, "10": 177, "11": 177, "2": 177, "3": 177, "4": 177, "5": 177, "6": 177, "7": 177, "8": 177, "9": 177 }, "neuron": { "0": 1436, "1": 1664, "10": 6, "11": 518, "2": 2634, "3": 306, "4": 449, "5": 732, "6": 138, "7": 321, "8": 184, "9": 91 } }, "50": { "head": { "0": 5, "1": 8, "10": 3, "11": 5, "2": 2, "3": 3, "4": 3, "5": 2, "6": 3, "7": 2, "8": 2, "9": 4 }, "hidden": { "-1": 223, "0": 223, "1": 223, "10": 223, "11": 223, "2": 223, "3": 223, "4": 223, "5": 223, "6": 223, "7": 223, "8": 223, "9": 223 }, "neuron": { "0": 1749, "1": 1933, "10": 21, "11": 915, "2": 2699, "3": 519, "4": 664, "5": 976, "6": 252, "7": 469, "8": 307, "9": 187 } }, "60": { "head": { "0": 6, "1": 8, "10": 5, "11": 7, "2": 3, "3": 4, "4": 4, "5": 3, "6": 5, "7": 2, "8": 2, "9": 4 }, "hidden": { "-1": 284, "0": 284, "1": 284, "10": 284, "11": 284, "2": 284, "3": 284, "4": 284, "5": 284, "6": 284, "7": 284, "8": 284, "9": 284 }, "neuron": { "0": 2065, "1": 2200, "10": 67, "11": 1392, "2": 2762, "3": 817, "4": 991, "5": 1279, "6": 454, "7": 695, "8": 521, "9": 397 } }, "70": { "head": { "0": 6, "1": 8, "10": 6, "11": 7, "2": 4, "3": 6, "4": 4, "5": 5, "6": 6, "7": 3, "8": 5, "9": 5 }, "hidden": { "-1": 346, "0": 346, "1": 346, "10": 346, "11": 346, "2": 346, "3": 346, "4": 346, "5": 346, "6": 346, "7": 346, "8": 346, "9": 346 }, "neuron": { "0": 2266, "1": 2390, "10": 206, "11": 1745, "2": 2820, "3": 1173, "4": 1308, "5": 1537, "6": 694, "7": 951, "8": 791, "9": 708 } }, "80": { "head": { "0": 6, "1": 9, "10": 8, "11": 8, "2": 6, "3": 6, "4": 6, "5": 9, "6": 7, "7": 4, "8": 5, "9": 5 }, "hidden": { "-1": 422, "0": 422, "1": 422, "10": 422, "11": 422, "2": 422, "3": 422, "4": 422, "5": 422, "6": 422, "7": 422, "8": 422, "9": 422 }, "neuron": { "0": 2450, "1": 2541, "10": 574, "11": 2105, "2": 2863, "3": 1559, "4": 1676, "5": 1862, "6": 1071, "7": 1294, "8": 1154, "9": 1126 } }, "85": { "head": { "0": 6, "1": 9, "10": 10, "11": 8, "2": 6, "3": 6, "4": 7, "5": 9, "6": 8, "7": 7, "8": 5, "9": 7 }, "hidden": { "-1": 468, "0": 468, "1": 468, "10": 468, "11": 468, "2": 468, "3": 468, "4": 468, "5": 468, "6": 468, "7": 468, "8": 468, "9": 468 }, "neuron": { "0": 2554, "1": 2635, "10": 936, "11": 2278, "2": 2889, "3": 1780, "4": 1836, "5": 2037, "6": 1292, "7": 1489, "8": 1360, "9": 1401 } }, "90": { "head": { "0": 6, "1": 9, "10": 11, "11": 8, "2": 7, "3": 7, "4": 9, "5": 9, "6": 9, "7": 9, "8": 6, "9": 8 }, "hidden": { "-1": 522, "0": 522, "1": 522, "10": 522, "11": 522, "2": 522, "3": 522, "4": 522, "5": 522, "6": 522, "7": 522, "8": 522, "9": 522 }, "neuron": { "0": 2653, "1": 2713, "10": 1390, "11": 2433, "2": 2913, "3": 2013, "4": 2064, "5": 2243, "6": 1569, "7": 1741, "8": 1648, "9": 1688 } }, "95": { "head": { "0": 8, "1": 10, "10": 11, "11": 9, "2": 9, "3": 9, "4": 9, "5": 9, "6": 11, "7": 9, "8": 9, "9": 9 }, "hidden": { "-1": 599, "0": 599, "1": 599, "10": 599, "11": 599, "2": 599, "3": 599, "4": 599, "5": 599, "6": 599, "7": 599, "8": 599, "9": 599 }, "neuron": { "0": 2753, "1": 2799, "10": 2001, "11": 2654, "2": 2948, "3": 2337, "4": 2381, "5": 2519, "6": 1992, "7": 2148, "8": 2061, "9": 2161 } }, "96": { "head": { "0": 9, "1": 10, "10": 11, "11": 10, "2": 9, "3": 9, "4": 9, "5": 10, "6": 11, "7": 9, "8": 9, "9": 9 }, "hidden": { "-1": 614, "0": 614, "1": 614, "10": 614, "11": 614, "2": 614, "3": 614, "4": 614, "5": 614, "6": 614, "7": 614, "8": 614, "9": 614 }, "neuron": { "0": 2768, "1": 2822, "10": 2133, "11": 2689, "2": 2955, "3": 2392, "4": 2439, "5": 2579, "6": 2077, "7": 2223, "8": 2148, "9": 2266 } }, "97": { "head": { "0": 10, "1": 10, "10": 11, "11": 10, "2": 9, "3": 9, "4": 9, "5": 10, "6": 11, "7": 10, "8": 11, "9": 10 }, "hidden": { "-1": 637, "0": 637, "1": 637, "10": 637, "11": 637, "2": 637, "3": 637, "4": 637, "5": 637, "6": 637, "7": 637, "8": 637, "9": 637 }, "neuron": { "0": 2799, "1": 2846, "10": 2294, "11": 2746, "2": 2970, "3": 2487, "4": 2537, "5": 2656, "6": 2207, "7": 2355, "8": 2291, "9": 2409 } }, "98": { "head": { "0": 11, "1": 10, "10": 11, "11": 10, "2": 10, "3": 10, "4": 10, "5": 10, "6": 11, "7": 10, "8": 11, "9": 10 }, "hidden": { "-1": 660, "0": 660, "1": 660, "10": 660, "11": 660, "2": 660, "3": 660, "4": 660, "5": 660, "6": 660, "7": 660, "8": 660, "9": 660 }, "neuron": { "0": 2826, "1": 2870, "10": 2455, "11": 2794, "2": 2984, "3": 2588, "4": 2634, "5": 2726, "6": 2350, "7": 2501, "8": 2440, "9": 2535 } }, "99": { "head": { "0": 11, "1": 10, "10": 11, "11": 11, "2": 11, "3": 11, "4": 11, "5": 11, "6": 11, "7": 11, "8": 11, "9": 10 }, "hidden": { "-1": 691, "0": 691, "1": 691, "10": 691, "11": 691, "2": 691, "3": 691, "4": 691, "5": 691, "6": 691, "7": 691, "8": 691, "9": 691 }, "neuron": { "0": 2861, "1": 2915, "10": 2683, "11": 2862, "2": 2998, "3": 2709, "4": 2746, "5": 2845, "6": 2551, "7": 2678, "8": 2635, "9": 2695 } } }, "torch_dtype": "float32", "transformers_version": "4.9.2", "type_vocab_size": 2, "use_cache": true, "vocab_size": 30522 }