diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,7073 @@ +{ + "metadata": { + "ParamSize": 515, + "ParamBytes": 5704949760.0, + "BitsPerParam": 15.99998851243995 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 262144000, + "records": [ + { + "name": "lm_head.weight", + "shape": [ + 32000, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144000, + "byteOffset": 0 + } + ], + "md5sum": "d6b7f632d45a6671e86edd191d6060ea" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.29.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "a21d5e099a8d3bc4945299ab09fbb72c" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.29.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "7ec36871dd1e19c8056ce5902094cf8f" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.29.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "37c724116c889c79e233fa5df10145fe" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.29.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.29.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.29.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.29.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.29.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "f192a4d1da64fd7400f8cbb44ceced97" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.30.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "9fb4499e1af81fcbdc951a6ba9de71ba" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.30.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "4fdc541177a504e05f6fd97fa322fa79" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.30.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "f94535f6c9fecee18662810e66180fd9" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.29.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.29.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.30.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 13418496 + }, + { + "name": "model.layers.30.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 13426688 + }, + { + "name": "model.layers.30.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 23207936 + } + ], + "md5sum": "6c10c4785fd479bbbc0d83d423c37d4e" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.30.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.30.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.30.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.30.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.30.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "9a6aadb91672096e83c1080776390ce5" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.30.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.30.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.30.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.30.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.30.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "09a98930771991bb1e4311ddbc110ddb" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.31.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "b00cfc9f71c513f2db21ae659bacf8b3" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.31.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "8ae3e46d7e119314f496319d701e2626" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.31.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "e12ebf0954ad0b0ade20ca84693d44b6" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.31.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.31.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.31.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.31.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.31.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "35f633e37b95ab1c71dc7bd83b5f222e" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.31.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.31.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.31.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.31.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.31.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "3d67f8f3f8272a39fa1b1e233860c626" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 262144000, + "records": [ + { + "name": "model.embed_tokens.weight", + "shape": [ + 32000, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144000, + "byteOffset": 0 + } + ], + "md5sum": "f1f601338a702fe42b2da79aa982284a" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.0.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "770b9294731b9ab8a2e3104ec1155b6e" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.0.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "f6be3849bd949ce506c5b46e3e0526ad" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 29925376, + "records": [ + { + "name": "model.layers.31.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.31.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.31.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.norm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20135936 + }, + { + "name": "model.layers.0.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20144128 + } + ], + "md5sum": "aad14f44f073a5a03f8e70954aad715b" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.0.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "92b4deeef18b3f62cf5327eab5d78b7f" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.0.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.0.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.0.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.0.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "d9d2910fde6473baff0424d83ffa572b" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.0.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.0.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.0.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.0.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.0.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "03c6787507f895e4a0c4099d89b37575" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.1.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "483fcfaf03bbf9021e73a8292ad8f737" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.1.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "54252e3a3f3df609fbac50f32a8532a9" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.1.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "674aa0c7f34b6ea7fd5d6a4b7d8aa00a" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.0.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.1.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.1.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "e28db7e4ec9ac06686dc17524f02e7e5" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.1.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.1.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.1.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.1.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "7e4c9b640a5cb550e25383f3c16f5f1c" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.1.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.1.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.1.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.1.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.1.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "559a5c617c99fe0ac65fc22418e804a4" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.10.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "6a6267c20a69ffe5fb9c7f051d0dba7e" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.10.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "07c2a321f7288abdb497d06cf5ac269d" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.10.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "a333accdad4245508d4c129c62717ac6" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.10.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.10.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.10.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "f8e1c811e2b3e3764bcd5ffab57a1c38" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.10.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.10.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.10.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.10.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.10.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "08140804cf8be3b947d07ce5c7f0e292" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.11.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "ae33b2569205a1b1154ec51cd74867b7" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.11.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "f887e7ae16cc277e0257183111cd63af" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.10.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.10.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.10.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.11.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "2c9af521d34224b6eab859f0ca0ca071" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.11.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "14ad495d6586db8a5042e556004de833" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.11.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.11.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.11.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.11.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "f0529f4c5436c4fda6b9165c1380c413" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.11.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.11.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.11.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.11.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.11.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "63452eeeebfb133cfcc9926f29fa56db" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.12.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "b2c9411ad4f669918b615b56963caa08" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.12.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "9b3a5c06ebed489b8009d8c01113c9c3" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.12.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "94668294082edda2222fe76b77d63cec" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.11.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.12.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.12.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "e6a4a62393366d6fd46825ca3d9df949" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.12.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.12.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.12.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.12.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "19f071ffebdc2e8096f4a67f976d97f5" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.12.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.12.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.12.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.12.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.12.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "ad5fe71d40e1b8c0b2e6f39d749d59b8" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.13.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "0261decbebdb1ac8cf76a479d708b245" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.13.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "ff782019dddcb3261e86b3a7c18c5b08" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.13.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "cc740b9b500838e88016f4729b962c95" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.13.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.13.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.13.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "5271f77d56a3feac0d30065867374ebf" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.13.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.13.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.13.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.13.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.13.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "488ae71d99b3af90ea9a2cdb8345b4bd" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.14.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "50c9e33af6170666eecec68840147af7" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.14.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "6eb8bb5f3167b0ec7d56dd31c77efe33" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.13.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.13.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.13.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.14.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "1b8487b4f29fa759a954976475631c14" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.14.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "53bf85dd756b90fedccbe2968a537bae" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.14.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.14.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.14.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.14.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "8d4bb12722293e6747add5bdd569fbb8" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.14.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.14.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.14.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.14.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.14.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "cc1b43a2e1bfbccebb99808abf1d5836" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.15.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "8b2d94a6e702b46a47a62609f2bd5599" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.15.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "07c21b784843b1a0a86630f63c4e3c48" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.15.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "223fe843f01eaa9ea6afe5a02e328f1e" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.14.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.15.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.15.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "6468d91659d67e9542dd4252817a8ba0" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.15.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.15.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.15.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.15.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "ceb671283dd3cc5cfb6cd8436b05fcab" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.15.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.15.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.15.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.15.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.15.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "dc0be584c948f9a720bd8a00183634f0" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.16.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "ed8f30a5fe08c3513c24a499a1ae6fd2" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.16.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "0b781c2ea966642a3f0a5922936d4e33" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.16.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "45a6e38d6b0ac4248198e66d298bc340" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.16.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.16.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.16.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "acb988d9faf11d82a05976a4c4cc1740" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.16.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.16.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.16.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.16.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.16.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "72cbf8f0ef2d0d2dc53c9f3b5b927805" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.17.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "d6eed0d1085bcdf827bf1cd16bb456ec" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.17.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "f964585379574bda44a7b054d807fb27" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.16.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.16.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.16.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.17.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "59cf684fb920c4a384348817e8a4ea52" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.17.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "6662f273ffd4672c7512a448d0a35712" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.17.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.17.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.17.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.17.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "81a81cbffd26d250bfc0225782b6d5a3" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.17.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.17.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.17.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.17.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.17.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "c31fe1ca0044f3348504fe1fd4d25f42" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.18.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "132338359df5e164d968e679ef9306ec" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.18.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "88656ebc3d78bdbdbecea35a57d916b1" + }, + { + "dataPath": "params_shard_76.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.18.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "84ef14fbb4987c28edbaafe09abbe88a" + }, + { + "dataPath": "params_shard_77.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.17.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.18.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.18.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "59f4eeba9ee828f2a95c55d78144b79e" + }, + { + "dataPath": "params_shard_78.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.18.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.18.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.18.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.18.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "8e7c488a93b0dead89c7268127623f55" + }, + { + "dataPath": "params_shard_79.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.18.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.18.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.18.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.18.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.18.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "0eb2f67f67024b91e9d39d37d3ea491a" + }, + { + "dataPath": "params_shard_80.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.19.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "50c6448600f8042583da77547f884097" + }, + { + "dataPath": "params_shard_81.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.19.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "5de4122ef49b16bff4e9df6cdf2a59c8" + }, + { + "dataPath": "params_shard_82.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.19.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "d507f0b31040ffc10f4f7308b8bc8522" + }, + { + "dataPath": "params_shard_83.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.19.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.19.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.19.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "f0c7ba66fe73d5035af9345af07018e1" + }, + { + "dataPath": "params_shard_84.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.19.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.19.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.19.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.19.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.19.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "47547d0391ac44efc1f68d1ca42e4c71" + }, + { + "dataPath": "params_shard_85.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.2.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "bc700dba33f22272b3bd4575181db358" + }, + { + "dataPath": "params_shard_86.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.2.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "d0d519c736a0d3c641377ad2fa521074" + }, + { + "dataPath": "params_shard_87.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.19.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.19.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.19.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.2.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "7f7293e032a6f5f87271bef5c2177220" + }, + { + "dataPath": "params_shard_88.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.2.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "f64df0dfafd35e90a07e41c7886f72c8" + }, + { + "dataPath": "params_shard_89.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.2.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.2.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.2.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.2.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "b82e53767368a843a1ac2c432bdfd287" + }, + { + "dataPath": "params_shard_90.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.2.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.2.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.2.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.2.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.2.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "b8a4bc7e88592dfc5a8d09127c5f974d" + }, + { + "dataPath": "params_shard_91.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.20.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "5cddc28349210b89bc63c63099ce9bcf" + }, + { + "dataPath": "params_shard_92.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.20.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "a88392ba45f25bc8c3458217e291e0fe" + }, + { + "dataPath": "params_shard_93.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.20.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "95ab42c2ae2ba0e0cfc64926577ae282" + }, + { + "dataPath": "params_shard_94.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.2.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.20.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.20.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "328d304305d42dbc2e20df3350692548" + }, + { + "dataPath": "params_shard_95.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.20.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.20.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.20.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.20.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "22a5edec155ec9b038e07c9d5038f826" + }, + { + "dataPath": "params_shard_96.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.20.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.20.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.20.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.20.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.20.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "3c7faf551a9898371d21689fe5b45bc7" + }, + { + "dataPath": "params_shard_97.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.21.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "336ad3776d0382f863315a6602dfeab8" + }, + { + "dataPath": "params_shard_98.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.21.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "259c64ece4bb483e68cc26e521d857d8" + }, + { + "dataPath": "params_shard_99.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.21.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "0bbc45df49c9c6e5588125e2c0008902" + }, + { + "dataPath": "params_shard_100.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.21.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.21.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.21.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "292a3ca520bd736021f4bd4e0afc50df" + }, + { + "dataPath": "params_shard_101.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.21.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.21.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.21.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.21.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.21.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "e3a0a31058f1d383347a67537e41175d" + }, + { + "dataPath": "params_shard_102.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.22.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "b986c586af66027e04c1517c73a90c56" + }, + { + "dataPath": "params_shard_103.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.22.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "25a895efdfe5100056533d9881d300a5" + }, + { + "dataPath": "params_shard_104.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.21.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.21.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.21.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.22.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "f3d8b421ac999dc7dd6ab68d8a0ffabf" + }, + { + "dataPath": "params_shard_105.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.22.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "e959b055b08257fc4657bfa0342eaea0" + }, + { + "dataPath": "params_shard_106.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.22.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.22.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.22.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.22.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "cbbffcfaa807c130546697ebb8feceae" + }, + { + "dataPath": "params_shard_107.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.22.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.22.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.22.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.22.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.22.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "2963cc60fb8750eb6642eed6cbad9916" + }, + { + "dataPath": "params_shard_108.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.23.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "f25d36e964380404a07e5e2e204ebafb" + }, + { + "dataPath": "params_shard_109.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.23.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "7be10f15db97e3d904e24934581d5255" + }, + { + "dataPath": "params_shard_110.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.23.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "d82eb8512368768fc7ed92533b3bb914" + }, + { + "dataPath": "params_shard_111.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.22.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.23.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.23.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "449ec1c33caf57448729a50628b609cb" + }, + { + "dataPath": "params_shard_112.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.23.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.23.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.23.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.23.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "966cf1ee4092d0da9c800281ea41645b" + }, + { + "dataPath": "params_shard_113.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.23.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.23.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.23.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.23.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.23.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "4186b4b52f8b368b5ed39829c565fe91" + }, + { + "dataPath": "params_shard_114.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.24.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "e8db855a3fa72fccc24ae48f9bbe4869" + }, + { + "dataPath": "params_shard_115.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.24.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "066cefe914d97b6ea392de5e80cbe22d" + }, + { + "dataPath": "params_shard_116.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.24.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "f03ca1164924d6f11862d1ac5698d1d2" + }, + { + "dataPath": "params_shard_117.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.24.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.24.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.24.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "e1397b5c368f98ffde1ddcba21473abe" + }, + { + "dataPath": "params_shard_118.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.24.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.24.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.24.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.24.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.24.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "cf229f9f1ca16f6a79a74509f8477151" + }, + { + "dataPath": "params_shard_119.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.25.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "9e46b981da80acf63585e27d10a2b7ae" + }, + { + "dataPath": "params_shard_120.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.25.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "0c9b2c341e7f1132781947f4e3e48fa8" + }, + { + "dataPath": "params_shard_121.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.24.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.24.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.24.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.25.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "7ebe7ab3767ca9e235ca49d296415da7" + }, + { + "dataPath": "params_shard_122.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.25.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "2acebbe4682bd7fcd31a3f2426ca3a9f" + }, + { + "dataPath": "params_shard_123.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.25.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.25.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.25.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.25.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "54949e79ecea1f7ea88f20c6a0749e48" + }, + { + "dataPath": "params_shard_124.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.25.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.25.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.25.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.25.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.25.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "2c8ab0b963f11b6b68792cac7825c24d" + }, + { + "dataPath": "params_shard_125.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.26.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "03be4cbcf6058fd1153c02f42a53714a" + }, + { + "dataPath": "params_shard_126.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.26.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "67f48f7759b042036da33fb3c0fa6a65" + }, + { + "dataPath": "params_shard_127.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.26.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "7bafb6f1e9fb7f7f8d3552cfa4c4998c" + }, + { + "dataPath": "params_shard_128.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.25.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.26.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.26.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "40c21b8070a65921bbf48e606347ce5e" + }, + { + "dataPath": "params_shard_129.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.26.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.26.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.26.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.26.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "60de4a0ad55cab957095838f40e1addd" + }, + { + "dataPath": "params_shard_130.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.26.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.26.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.26.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.26.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.26.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "97f4924ee88af6df03ae7de33cc78f39" + }, + { + "dataPath": "params_shard_131.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.27.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "81f9c3c5fadf31c39189e96fcd4e8c8e" + }, + { + "dataPath": "params_shard_132.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.27.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "34008bb9f0860ec255d1742a4643e7a8" + }, + { + "dataPath": "params_shard_133.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.27.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "54990635b2ee72dfab4d48e494019bf2" + }, + { + "dataPath": "params_shard_134.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.27.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.27.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.27.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "bbe31486b2592451ce9277a50a4370f4" + }, + { + "dataPath": "params_shard_135.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.27.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.27.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.27.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.27.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.27.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "47694ed46f0300a38505fad0d11c7d84" + }, + { + "dataPath": "params_shard_136.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.28.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "41213206665dd316534e7ec2c862362e" + }, + { + "dataPath": "params_shard_137.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.28.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "8bfb51fc81aff79fcf8a9fdb42304ab9" + }, + { + "dataPath": "params_shard_138.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.27.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.27.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.27.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.28.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.28.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "5adf386066a1a01f755dae8e3f29443a" + }, + { + "dataPath": "params_shard_139.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.28.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "13b7db88973c467915b8892d63eb2508" + }, + { + "dataPath": "params_shard_140.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.28.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.28.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.28.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.28.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.28.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "01b923a2143d56c6132569f77546997c" + }, + { + "dataPath": "params_shard_141.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.28.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.28.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.28.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.28.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.28.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "b61b51edc8d1f3e5842e7c851b834c6f" + }, + { + "dataPath": "params_shard_142.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.28.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.29.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.29.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.29.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.29.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "45b5c6f53d66c1bb192ea1bfd3f62927" + }, + { + "dataPath": "params_shard_143.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.3.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "c02585fcf4aa304f3c2c6887a080c874" + }, + { + "dataPath": "params_shard_144.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.3.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "e6061da8377b7e1ed4a14221d7d1c597" + }, + { + "dataPath": "params_shard_145.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.3.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "dc97fa7733bdd106826b1231532a9aa2" + }, + { + "dataPath": "params_shard_146.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.29.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.29.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 13418496 + }, + { + "name": "model.layers.3.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 13426688 + }, + { + "name": "model.layers.3.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 23207936 + } + ], + "md5sum": "b723503e4cf9b2fe002cf3cb1cbd7dc6" + }, + { + "dataPath": "params_shard_147.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.3.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.3.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.3.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.3.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "015cfbb26f3c4a072673127b743e2894" + }, + { + "dataPath": "params_shard_148.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.3.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.3.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.3.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.3.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.3.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "4b025478d6337e9353b549fee15dbb22" + }, + { + "dataPath": "params_shard_149.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.4.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "0ffe97287723d184f19416fbfdcb659c" + }, + { + "dataPath": "params_shard_150.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.4.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "cb36b5c6c569e57373a37b13d3039fc8" + }, + { + "dataPath": "params_shard_151.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.4.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "d43d6d136af85e037f9568352a52de66" + }, + { + "dataPath": "params_shard_152.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.4.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.4.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.4.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "fec85e6a287a40d836f69f963fde1565" + }, + { + "dataPath": "params_shard_153.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.4.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.4.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.4.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.4.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.4.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "284b2073bfdf3c7da622008a24b067f7" + }, + { + "dataPath": "params_shard_154.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.5.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "ea5b9ac113a5bdff2ae6086c8d98a45a" + }, + { + "dataPath": "params_shard_155.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.5.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "64683f92566cca3d208ce36237e4783a" + }, + { + "dataPath": "params_shard_156.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.4.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.4.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.4.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.5.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "a5d06fb517a7ab7283232233a4e21ac6" + }, + { + "dataPath": "params_shard_157.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.5.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "a84c920af4ac65196a4dce438796e095" + }, + { + "dataPath": "params_shard_158.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.5.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.5.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.5.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.5.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "a68fae5e49bc7438aa67ff423197285d" + }, + { + "dataPath": "params_shard_159.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.5.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.5.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.5.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.5.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.5.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "cfa2ef7e8f20b3715245b90e86cedb4e" + }, + { + "dataPath": "params_shard_160.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.6.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "98d8a7dc6527d048adf80411c49114dd" + }, + { + "dataPath": "params_shard_161.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.6.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "c12109887fd1f590287ed95c4975438b" + }, + { + "dataPath": "params_shard_162.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.6.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "f6e7bb408a45b98e69b110fc10e0c872" + }, + { + "dataPath": "params_shard_163.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.5.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.6.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.6.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "08a779b05cae43a1d885ba862ca1800f" + }, + { + "dataPath": "params_shard_164.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.6.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.6.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.6.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.6.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "e5998fdac09257ead3ef6cf9118b13e3" + }, + { + "dataPath": "params_shard_165.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.6.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.6.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.6.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.6.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.6.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "55fe1d4614d78d69ddda96b227eedbfd" + }, + { + "dataPath": "params_shard_166.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.7.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "1d22d2d42ff9a705b48d06020ad40795" + }, + { + "dataPath": "params_shard_167.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.7.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "200beeef4614b45ad7194dcbe6c9acea" + }, + { + "dataPath": "params_shard_168.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.7.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "aa15a629855302c5a404f905c50ae5f8" + }, + { + "dataPath": "params_shard_169.bin", + "format": "raw-shard", + "nbytes": 29360128, + "records": [ + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.7.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 8192 + }, + { + "name": "model.layers.7.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.7.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 29351936 + } + ], + "md5sum": "1db53bc518ff737796e3e3bb98a4173c" + }, + { + "dataPath": "params_shard_170.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.7.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.7.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.7.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.7.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.7.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "c80bd23c3282368481e569d85ec70262" + }, + { + "dataPath": "params_shard_171.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.8.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "e54145ab303c3b917c06181f4dd0aabf" + }, + { + "dataPath": "params_shard_172.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.8.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "8094b90d0e894c0b2dc0e0e4d14e078f" + }, + { + "dataPath": "params_shard_173.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.7.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.7.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.7.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 20127744 + }, + { + "name": "model.layers.8.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 20135936 + } + ], + "md5sum": "50ca96af51f584cb80005c4adc8a4628" + }, + { + "dataPath": "params_shard_174.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.8.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "830f7bea17a6cd9cea8cbba362b2b7d7" + }, + { + "dataPath": "params_shard_175.bin", + "format": "raw-shard", + "nbytes": 32989184, + "records": [ + { + "name": "model.layers.8.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.8.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 9781248 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19562496 + }, + { + "name": "model.layers.8.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 19570688 + }, + { + "name": "model.layers.8.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26279936 + } + ], + "md5sum": "f78f5b53ad707cdb8d39a5e29e85ce75" + }, + { + "dataPath": "params_shard_176.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.8.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.8.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.8.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.8.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.8.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "313138351edb5e54a5f22710c94e2389" + }, + { + "dataPath": "params_shard_177.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.9.mlp.down_v_proj.weight", + "shape": [ + 1194, + 11008 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "27b74d3f73ec21d6d8459aaca8e02ced" + }, + { + "dataPath": "params_shard_178.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.9.mlp.gate_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "e3f20fa6c31fb031b705d990e8df38e2" + }, + { + "dataPath": "params_shard_179.bin", + "format": "raw-shard", + "nbytes": 26287104, + "records": [ + { + "name": "model.layers.9.mlp.up_u_proj.weight", + "shape": [ + 11008, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 26287104, + "byteOffset": 0 + } + ], + "md5sum": "dc0f1f1b2323d74961839a085e516850" + }, + { + "dataPath": "params_shard_180.bin", + "format": "raw-shard", + "nbytes": 26279936, + "records": [ + { + "name": "model.layers.8.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6709248 + }, + { + "name": "model.layers.9.mlp.down_u_proj.weight", + "shape": [ + 4096, + 1194 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 6717440 + }, + { + "name": "model.layers.9.mlp.gate_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 16498688 + } + ], + "md5sum": "71481d112d1d94492941a60befdd2b1f" + }, + { + "dataPath": "params_shard_181.bin", + "format": "raw-shard", + "nbytes": 29917184, + "records": [ + { + "name": "model.layers.9.mlp.up_v_proj.weight", + "shape": [ + 1194, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9781248, + "byteOffset": 0 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9781248 + }, + { + "name": "model.layers.9.self_attn.k_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 9789440 + }, + { + "name": "model.layers.9.self_attn.k_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 16498688 + }, + { + "name": "model.layers.9.self_attn.o_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 23207936 + } + ], + "md5sum": "9b883c2be96e7ca0843008b66b5a5439" + }, + { + "dataPath": "params_shard_182.bin", + "format": "raw-shard", + "nbytes": 33546240, + "records": [ + { + "name": "model.layers.9.self_attn.o_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 0 + }, + { + "name": "model.layers.9.self_attn.q_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 6709248 + }, + { + "name": "model.layers.9.self_attn.q_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 13418496 + }, + { + "name": "model.layers.9.self_attn.v_u_proj.weight", + "shape": [ + 4096, + 819 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 20127744 + }, + { + "name": "model.layers.9.self_attn.v_v_proj.weight", + "shape": [ + 819, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6709248, + "byteOffset": 26836992 + } + ], + "md5sum": "46f98ae74caec9a5e618f7d8112a024d" + } + ] +} \ No newline at end of file