diff --git "a/ndarray-cache-b16.json" "b/ndarray-cache-b16.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache-b16.json" @@ -0,0 +1,4025 @@ +{ + "metadata": { + "ParamSize": 323, + "ParamBytes": 2398187520.0, + "BitsPerParam": 5.0014054560524706 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 307298304, + "records": [ + { + "name": "transformer.embd.q_weight", + "shape": [ + 200064, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 307298304, + "byteOffset": 0 + } + ], + "md5sum": "82c7a74b6693e6518c3ef6d610a95678" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 38412288, + "records": [ + { + "name": "transformer.embd.q_scale", + "shape": [ + 200064, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 38412288, + "byteOffset": 0 + } + ], + "md5sum": "a255a3f1ec9e1768ed2f07e306044d43" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.0.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5a5953e1f5b2d984ec15affbc39a2cc7" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 31475712, + "records": [ + { + "name": "transformer.h.0.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 0 + }, + { + "name": "transformer.h.0.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 6144 + }, + { + "name": "transformer.h.0.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12589056 + }, + { + "name": "transformer.h.0.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.0.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.0.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17313792 + }, + { + "name": "transformer.h.0.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22032384 + }, + { + "name": "transformer.h.0.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22622208 + }, + { + "name": "transformer.h.0.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30486528 + }, + { + "name": "transformer.h.1.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31469568 + } + ], + "md5sum": "cc3c26fd04b584d3e5f89081646355df" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.1.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "612abf0e6666784f107b4d785c3ff2fc" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.1.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.1.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.1.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.1.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.1.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.1.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.1.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.1.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.10.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "c2504d630a67dbbfd838c6ca2ad8043f" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.10.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "ab5f28025dee361e9a8ac728cbf65bd2" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.10.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.10.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.10.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.10.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.10.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.10.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.10.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.10.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.11.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "8e1e5c80991638f5559315ed2aa1a7f9" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.11.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "4fb985985c379505f6fa18205de2a264" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.11.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.11.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.11.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.11.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.11.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.11.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.11.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.11.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.12.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "aefae0fc8bd49a07288ee36097d66fc6" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.12.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "0bfc040c21da5a6095524d52573ef225" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.12.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.12.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.12.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.12.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.12.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.12.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.12.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.12.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.13.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "6c21e15f146bb8da69755bc270dc1584" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.13.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "dea1339c2ba2a691a3e40f5cfe223eba" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.13.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.13.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.13.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.13.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.13.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.13.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.13.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.13.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.14.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "e9925d0850ff319b1869dfc5a7348088" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.14.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "29b8e125d3a80ec9f62e78f9dea9488c" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.14.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.14.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.14.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.14.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.14.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.14.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.14.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.14.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.15.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "5b7b7908e78607ba89a42b4f589aa354" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.15.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "7f10246e31892bed3709ceeee35674e9" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.15.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.15.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.15.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.15.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.15.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.15.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.15.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.15.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.16.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "39a86b8b9d2d89fec7cde46c6a1604de" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.16.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "579275b9bd82d855c6c8c6aff6d0c482" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.16.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.16.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.16.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.16.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.16.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.16.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.16.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.16.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.17.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "4e54bb3939b411945f3ff782c848ea90" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.17.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "f6f2e05c32fdd5ae44b2837a3d96db71" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 31463424, + "records": [ + { + "name": "transformer.h.17.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.17.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.17.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.17.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.17.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.17.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.17.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.17.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + } + ], + "md5sum": "6b2dcb4b1c317976150ed9e8679f4909" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.2.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "d66711671b3398ae45fc1c8f364bcfe2" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.18.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.18.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.18.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.18.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.2.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.2.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.2.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.2.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.2.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "dc83c9be3a257bf5daf287a565f1a691" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.3.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5f2aecd1b35108bfe60a2deb4b7abb82" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.2.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.2.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.2.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.2.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.3.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.3.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.3.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.3.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.3.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "a1b3b865387affdca6cc2f6ba5ecde21" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.4.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "168bdefe007a38c35a3f627c5b86d13f" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.3.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.3.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.3.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.3.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.4.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.4.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.4.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.4.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.4.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "4cd0686089ad87795135a877eb41d8b6" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.5.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "541cd1129f4575d14e4eadfc7496ef4a" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.4.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.4.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.4.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.4.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.5.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.5.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.5.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.5.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.5.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "7048834958650a81b9e91fe5b1e99a99" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.6.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "6fe7f966f2ec2fc821ea3984d94799c7" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.5.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.5.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.5.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.5.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.6.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.6.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.6.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.6.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.6.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "5965053dbc8b0bc06c4295db6d3efd63" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.7.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "78ee3edcc0d64d6ed4850eb2ee49cf83" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.6.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.6.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.6.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.6.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.7.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.7.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.7.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.7.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.7.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "0ed2a206126fd93beb780120145c19c8" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.8.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2d7b190d765b191c0c172676aebcc82e" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.7.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.7.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.7.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.7.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.8.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.8.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.8.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.8.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.8.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "1e33074838acba10035869751ed75490" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.9.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1b573999d380fb827a621b2a2e7fddbb" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.8.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.8.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.8.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.8.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.9.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.9.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.9.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.9.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.9.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "d13900dcfe7f2f4c274b1cf18b5f014b" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.18.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "b54907f0fcd1197cff8eac08ebad4a26" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 31475712, + "records": [ + { + "name": "transformer.h.9.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.9.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.9.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.9.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 13172736 + }, + { + "name": "transformer.h.18.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.18.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 14161920 + }, + { + "name": "transformer.h.18.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 26744832 + }, + { + "name": "transformer.h.18.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 28317696 + }, + { + "name": "transformer.h.18.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + }, + { + "name": "transformer.h.19.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31469568 + } + ], + "md5sum": "57d3a895e53246c7337d0eee4d985f6c" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.19.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "cefd6cc86a6b35c649d904489aefeee3" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.19.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.19.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.19.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.19.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.19.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.19.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.19.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.19.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.20.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "7224be0f5e07fc9da7cc57b369cd09d8" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.20.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "bbe82e456e726886e1dc70a44997cd8b" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.20.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.20.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.20.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.20.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.20.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.20.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.20.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.20.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.21.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "d6c7356baf8df02c6ff37958032e04ca" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.21.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "6e0646643aadcd3423104c63353386ac" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.21.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.21.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.21.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.21.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.21.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.21.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.21.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.21.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.22.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "26167288574ef541032be1fd42419771" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.22.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5ee6349d2cd39521a53581110ba0b5b8" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.22.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.22.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.22.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.22.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.22.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.22.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.22.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.22.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.23.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "d04e49417b25160ec31e5b51598dd825" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.23.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2e75644202d9833a5e6162235e6a0506" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.23.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.23.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.23.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.23.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.23.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.23.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.23.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.23.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.24.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "b671d9596dacbcc26710d1ee215347d7" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.24.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "f239234a5bd359875a652d992cc4e81b" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.24.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.24.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.24.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.24.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.24.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.24.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.24.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.24.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.25.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "f5e17f3013b99dd1c88aadb461d54e9a" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.25.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "0ef3fe2305bb2f74f188016fa6bf3bc5" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.25.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.25.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.25.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.25.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.25.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.25.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.25.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.25.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.26.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "a6cdd09ef7cc9e352f9847ba5a274e16" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.26.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "7a9bde39459c097f36f21588eff0d6fd" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.26.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.26.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.26.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.26.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.26.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.26.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.26.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.26.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.27.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "2493e574d82e2eff5b5f15f5012e898b" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.27.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "222a2523a58b0a06938a46ff68991c55" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.27.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.27.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.27.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.27.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.27.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.27.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.27.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.27.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.28.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "98e4e06d53d7fb583c66f01b6cecef0e" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.28.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2378a2fe4ba51bb98ce9951aa5f04f53" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.28.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.28.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.28.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.28.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.28.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.28.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.28.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.28.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.29.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "42244917a39b85a95fcde7b3643bf459" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.29.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "425842dace8f8219831e213fd3326b74" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.29.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.29.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.29.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.29.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.29.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.29.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.29.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.29.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.30.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "b8caac7f8ae7a531bcbceba43fd1e433" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.30.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "16799cd2b411baa28e35fdcf4da3bb27" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.30.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.30.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.30.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.30.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.30.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.30.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.30.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.30.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.h.31.ln.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "6770a369914f6c8f35d52343031fbaec" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.31.mlp.gate_up_proj.q_weight", + "shape": [ + 16384, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "6d9d814263103a86ee07e5fd4b00a6f3" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 31469568, + "records": [ + { + "name": "transformer.h.31.mlp.down_proj.q_weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.31.mlp.down_proj.q_scale", + "shape": [ + 3072, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.31.mlp.gate_up_proj.q_scale", + "shape": [ + 16384, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.31.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.31.mixer.out_proj.q_weight", + "shape": [ + 3072, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.31.mixer.out_proj.q_scale", + "shape": [ + 3072, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 589824, + "byteOffset": 22026240 + }, + { + "name": "transformer.h.31.mixer.qkv_proj.q_weight", + "shape": [ + 5120, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 7864320, + "byteOffset": 22616064 + }, + { + "name": "transformer.h.31.mixer.qkv_proj.q_scale", + "shape": [ + 5120, + 96 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 983040, + "byteOffset": 30480384 + }, + { + "name": "transformer.norm.weight", + "shape": [ + 3072 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6144, + "byteOffset": 31463424 + } + ], + "md5sum": "97c63133467fbb61b004784823182224" + } + ] +} \ No newline at end of file