File size: 7,824 Bytes
2e144f7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 |
{
"_name_or_path": "checkpoints/microsoft/phi-1_5",
"anyprec": {
"arch_config": {
"layers_name": "layers",
"model_name": "model",
"module_names": [
"self_attn.q_proj",
"self_attn.k_proj",
"self_attn.v_proj",
"self_attn.dense",
"mlp.fc1",
"mlp.fc2"
]
},
"group_count": 1,
"parent_precision": 4,
"seed_precision": 2,
"sparse_numvals": {
"model.layers.0.mlp.fc1": 1279820,
"model.layers.0.mlp.fc2": 1236307,
"model.layers.0.self_attn.dense": 189187,
"model.layers.0.self_attn.k_proj": 318137,
"model.layers.0.self_attn.q_proj": 285336,
"model.layers.0.self_attn.v_proj": 212019,
"model.layers.1.mlp.fc1": 539655,
"model.layers.1.mlp.fc2": 766840,
"model.layers.1.self_attn.dense": 176195,
"model.layers.1.self_attn.k_proj": 224990,
"model.layers.1.self_attn.q_proj": 219788,
"model.layers.1.self_attn.v_proj": 201422,
"model.layers.10.mlp.fc1": 710241,
"model.layers.10.mlp.fc2": 754886,
"model.layers.10.self_attn.dense": 180978,
"model.layers.10.self_attn.k_proj": 221678,
"model.layers.10.self_attn.q_proj": 212231,
"model.layers.10.self_attn.v_proj": 200830,
"model.layers.11.mlp.fc1": 710220,
"model.layers.11.mlp.fc2": 741443,
"model.layers.11.self_attn.dense": 178022,
"model.layers.11.self_attn.k_proj": 215089,
"model.layers.11.self_attn.q_proj": 207761,
"model.layers.11.self_attn.v_proj": 194583,
"model.layers.12.mlp.fc1": 697047,
"model.layers.12.mlp.fc2": 752645,
"model.layers.12.self_attn.dense": 176274,
"model.layers.12.self_attn.k_proj": 220310,
"model.layers.12.self_attn.q_proj": 213155,
"model.layers.12.self_attn.v_proj": 198321,
"model.layers.13.mlp.fc1": 687659,
"model.layers.13.mlp.fc2": 738003,
"model.layers.13.self_attn.dense": 178018,
"model.layers.13.self_attn.k_proj": 221049,
"model.layers.13.self_attn.q_proj": 210113,
"model.layers.13.self_attn.v_proj": 195362,
"model.layers.14.mlp.fc1": 682287,
"model.layers.14.mlp.fc2": 781981,
"model.layers.14.self_attn.dense": 184895,
"model.layers.14.self_attn.k_proj": 217243,
"model.layers.14.self_attn.q_proj": 213786,
"model.layers.14.self_attn.v_proj": 198728,
"model.layers.15.mlp.fc1": 666136,
"model.layers.15.mlp.fc2": 761731,
"model.layers.15.self_attn.dense": 177136,
"model.layers.15.self_attn.k_proj": 217895,
"model.layers.15.self_attn.q_proj": 232288,
"model.layers.15.self_attn.v_proj": 191081,
"model.layers.16.mlp.fc1": 656390,
"model.layers.16.mlp.fc2": 799995,
"model.layers.16.self_attn.dense": 174656,
"model.layers.16.self_attn.k_proj": 217135,
"model.layers.16.self_attn.q_proj": 219926,
"model.layers.16.self_attn.v_proj": 185798,
"model.layers.17.mlp.fc1": 639288,
"model.layers.17.mlp.fc2": 775904,
"model.layers.17.self_attn.dense": 173271,
"model.layers.17.self_attn.k_proj": 206996,
"model.layers.17.self_attn.q_proj": 205270,
"model.layers.17.self_attn.v_proj": 184931,
"model.layers.18.mlp.fc1": 632334,
"model.layers.18.mlp.fc2": 768287,
"model.layers.18.self_attn.dense": 189412,
"model.layers.18.self_attn.k_proj": 215687,
"model.layers.18.self_attn.q_proj": 242190,
"model.layers.18.self_attn.v_proj": 200154,
"model.layers.19.mlp.fc1": 625021,
"model.layers.19.mlp.fc2": 738002,
"model.layers.19.self_attn.dense": 186977,
"model.layers.19.self_attn.k_proj": 216466,
"model.layers.19.self_attn.q_proj": 240694,
"model.layers.19.self_attn.v_proj": 197648,
"model.layers.2.mlp.fc1": 621667,
"model.layers.2.mlp.fc2": 757420,
"model.layers.2.self_attn.dense": 170986,
"model.layers.2.self_attn.k_proj": 225618,
"model.layers.2.self_attn.q_proj": 217741,
"model.layers.2.self_attn.v_proj": 200228,
"model.layers.20.mlp.fc1": 614692,
"model.layers.20.mlp.fc2": 727978,
"model.layers.20.self_attn.dense": 175731,
"model.layers.20.self_attn.k_proj": 213423,
"model.layers.20.self_attn.q_proj": 236043,
"model.layers.20.self_attn.v_proj": 183771,
"model.layers.21.mlp.fc1": 618662,
"model.layers.21.mlp.fc2": 738785,
"model.layers.21.self_attn.dense": 177493,
"model.layers.21.self_attn.k_proj": 208350,
"model.layers.21.self_attn.q_proj": 237646,
"model.layers.21.self_attn.v_proj": 187251,
"model.layers.22.mlp.fc1": 629352,
"model.layers.22.mlp.fc2": 818793,
"model.layers.22.self_attn.dense": 175140,
"model.layers.22.self_attn.k_proj": 202527,
"model.layers.22.self_attn.q_proj": 284459,
"model.layers.22.self_attn.v_proj": 180999,
"model.layers.23.mlp.fc1": 711633,
"model.layers.23.mlp.fc2": 1103566,
"model.layers.23.self_attn.dense": 219201,
"model.layers.23.self_attn.k_proj": 224644,
"model.layers.23.self_attn.q_proj": 397194,
"model.layers.23.self_attn.v_proj": 230928,
"model.layers.3.mlp.fc1": 663185,
"model.layers.3.mlp.fc2": 761065,
"model.layers.3.self_attn.dense": 185269,
"model.layers.3.self_attn.k_proj": 240041,
"model.layers.3.self_attn.q_proj": 232277,
"model.layers.3.self_attn.v_proj": 214858,
"model.layers.4.mlp.fc1": 716587,
"model.layers.4.mlp.fc2": 767640,
"model.layers.4.self_attn.dense": 179773,
"model.layers.4.self_attn.k_proj": 227913,
"model.layers.4.self_attn.q_proj": 220527,
"model.layers.4.self_attn.v_proj": 211685,
"model.layers.5.mlp.fc1": 707590,
"model.layers.5.mlp.fc2": 780274,
"model.layers.5.self_attn.dense": 178504,
"model.layers.5.self_attn.k_proj": 247977,
"model.layers.5.self_attn.q_proj": 243896,
"model.layers.5.self_attn.v_proj": 207831,
"model.layers.6.mlp.fc1": 710038,
"model.layers.6.mlp.fc2": 763787,
"model.layers.6.self_attn.dense": 190308,
"model.layers.6.self_attn.k_proj": 224045,
"model.layers.6.self_attn.q_proj": 220275,
"model.layers.6.self_attn.v_proj": 212698,
"model.layers.7.mlp.fc1": 715221,
"model.layers.7.mlp.fc2": 758669,
"model.layers.7.self_attn.dense": 175635,
"model.layers.7.self_attn.k_proj": 231017,
"model.layers.7.self_attn.q_proj": 224708,
"model.layers.7.self_attn.v_proj": 200219,
"model.layers.8.mlp.fc1": 722869,
"model.layers.8.mlp.fc2": 747381,
"model.layers.8.self_attn.dense": 184555,
"model.layers.8.self_attn.k_proj": 230928,
"model.layers.8.self_attn.q_proj": 224025,
"model.layers.8.self_attn.v_proj": 206979,
"model.layers.9.mlp.fc1": 719199,
"model.layers.9.mlp.fc2": 748623,
"model.layers.9.self_attn.dense": 174700,
"model.layers.9.self_attn.k_proj": 228878,
"model.layers.9.self_attn.q_proj": 222182,
"model.layers.9.self_attn.v_proj": 199200
}
},
"architectures": [
"PhiForCausalLM"
],
"attention_dropout": 0.0,
"bos_token_id": null,
"embd_pdrop": 0.0,
"eos_token_id": null,
"hidden_act": "gelu_new",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 8192,
"layer_norm_eps": 1e-05,
"max_position_embeddings": 2048,
"model_type": "phi",
"num_attention_heads": 32,
"num_hidden_layers": 24,
"num_key_value_heads": 32,
"partial_rotary_factor": 0.5,
"qk_layernorm": false,
"resid_pdrop": 0.0,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.39.3",
"use_cache": true,
"vocab_size": 51200
}
|