| { | |
| "metadata": { | |
| "ParamSize": 235, | |
| "ParamBytes": 4143283200.0, | |
| "BitsPerParam": 16.0 | |
| }, | |
| "records": [ | |
| { | |
| "dataPath": "params_shard_0.bin", | |
| "format": "raw-shard", | |
| "nbytes": 262144000, | |
| "records": [ | |
| { | |
| "name": "lm_head.linear.weight", | |
| "shape": [ | |
| 51200, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144000, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "d975f6e26151b33f352a07426b0e0b04" | |
| }, | |
| { | |
| "dataPath": "params_shard_1.bin", | |
| "format": "raw-shard", | |
| "nbytes": 262144000, | |
| "records": [ | |
| { | |
| "name": "transformer.embd.weight", | |
| "shape": [ | |
| 51200, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144000, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "92e1da58eb645448e09ca03417db18f3" | |
| }, | |
| { | |
| "dataPath": "params_shard_2.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0bdc7b67ee83655337c030646e83d88e" | |
| }, | |
| { | |
| "dataPath": "params_shard_3.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "5d10e5a3d6384a7fc31ec1f207b379ec" | |
| }, | |
| { | |
| "dataPath": "params_shard_4.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "fe6498a99c51e2973b01337d7ef56239" | |
| }, | |
| { | |
| "dataPath": "params_shard_5.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.1.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "4d3b3ee8deab0e85807b3a83c1eed0e1" | |
| }, | |
| { | |
| "dataPath": "params_shard_6.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.1.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "48483266ee7a4c7ff9dfcb118ebf3d96" | |
| }, | |
| { | |
| "dataPath": "params_shard_7.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.1.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "7bd23caad9e0fd9f4527ca5c25cf3d8f" | |
| }, | |
| { | |
| "dataPath": "params_shard_8.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8daf4bf12359c5718d276c5f0797e7a6" | |
| }, | |
| { | |
| "dataPath": "params_shard_9.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "cba797485e7e94226fcf8fde61e5a98a" | |
| }, | |
| { | |
| "dataPath": "params_shard_10.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26480640, | |
| "records": [ | |
| { | |
| "name": "lm_head.linear.bias", | |
| "shape": [ | |
| 51200 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 102400, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "lm_head.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 102400 | |
| }, | |
| { | |
| "name": "lm_head.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 107520 | |
| }, | |
| { | |
| "name": "transformer.h.0.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 112640 | |
| }, | |
| { | |
| "name": "transformer.h.0.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 117760 | |
| }, | |
| { | |
| "name": "transformer.h.0.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 122880 | |
| }, | |
| { | |
| "name": "transformer.h.0.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 143360 | |
| }, | |
| { | |
| "name": "transformer.h.0.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 148480 | |
| }, | |
| { | |
| "name": "transformer.h.0.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 153600 | |
| }, | |
| { | |
| "name": "transformer.h.0.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13260800 | |
| }, | |
| { | |
| "name": "transformer.h.1.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13276160 | |
| }, | |
| { | |
| "name": "transformer.h.1.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13281280 | |
| }, | |
| { | |
| "name": "transformer.h.1.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13286400 | |
| }, | |
| { | |
| "name": "transformer.h.1.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13306880 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13312000 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13317120 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26424320 | |
| }, | |
| { | |
| "name": "transformer.h.10.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26439680 | |
| }, | |
| { | |
| "name": "transformer.h.10.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26444800 | |
| }, | |
| { | |
| "name": "transformer.h.10.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26449920 | |
| }, | |
| { | |
| "name": "transformer.h.10.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26470400 | |
| }, | |
| { | |
| "name": "transformer.h.10.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26475520 | |
| } | |
| ], | |
| "md5sum": "fb9268641560780ae50adafce6bf0d95" | |
| }, | |
| { | |
| "dataPath": "params_shard_11.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "4497a13726c83e9eee58a31065a7ee06" | |
| }, | |
| { | |
| "dataPath": "params_shard_12.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.11.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "c228e3e0c39f47e17b58c8bc814d99a0" | |
| }, | |
| { | |
| "dataPath": "params_shard_13.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.11.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "d6165059c3d6fa2335880fc490746341" | |
| }, | |
| { | |
| "dataPath": "params_shard_14.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.11.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3ccb1f0cfc3d89d2865fbd8f31f9489f" | |
| }, | |
| { | |
| "dataPath": "params_shard_15.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "685f4e48bb24fbd4706a1f953959a4de" | |
| }, | |
| { | |
| "dataPath": "params_shard_16.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "18dd01bcd1fc6c15b2ad3834494b2e10" | |
| }, | |
| { | |
| "dataPath": "params_shard_17.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.10.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.11.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.11.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.11.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.11.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.12.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.12.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.12.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.12.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.12.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "ebdd34f965ad45585cb1d413ff32c87a" | |
| }, | |
| { | |
| "dataPath": "params_shard_18.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "b9ca5680d399eac0fe69b94c9139a762" | |
| }, | |
| { | |
| "dataPath": "params_shard_19.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.13.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "e760d6f347a139bcfa275427e68014d0" | |
| }, | |
| { | |
| "dataPath": "params_shard_20.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.13.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "c77575b7c79c77d4e1c96c59db2ffad9" | |
| }, | |
| { | |
| "dataPath": "params_shard_21.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.13.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "bef6be3d5cbd91e76dd9e96c6947ab2b" | |
| }, | |
| { | |
| "dataPath": "params_shard_22.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "552173b4d3bc1a941ae9e875c1b9a3c9" | |
| }, | |
| { | |
| "dataPath": "params_shard_23.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "02d10cf80c1621361f3b8b95d6523237" | |
| }, | |
| { | |
| "dataPath": "params_shard_24.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.12.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.13.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.13.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.13.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.13.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.14.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.14.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.14.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.14.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.14.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "7470d38c13da58b4e16b68d8b53a4f7d" | |
| }, | |
| { | |
| "dataPath": "params_shard_25.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "915efda74d5e6cdb38577723576c9e25" | |
| }, | |
| { | |
| "dataPath": "params_shard_26.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.15.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "04e3311d15e6d46a564ca36e1b38d285" | |
| }, | |
| { | |
| "dataPath": "params_shard_27.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.15.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "d9cacec0b90fda6bbdaad37787c7dd6c" | |
| }, | |
| { | |
| "dataPath": "params_shard_28.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.15.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "61af82e7e0e6f1257ca91e1631437aa9" | |
| }, | |
| { | |
| "dataPath": "params_shard_29.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0409b2be922e8c2d9a5dd2b5b355a24f" | |
| }, | |
| { | |
| "dataPath": "params_shard_30.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "09fcf89c90d088cf29f2f592c4b13399" | |
| }, | |
| { | |
| "dataPath": "params_shard_31.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.14.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.15.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.15.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.15.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.15.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.16.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.16.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.16.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.16.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.16.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "49dde728544c69a8567da515c4be4ce8" | |
| }, | |
| { | |
| "dataPath": "params_shard_32.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "db25133baad6ed95cb35462d0829659f" | |
| }, | |
| { | |
| "dataPath": "params_shard_33.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.17.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "e8f7342ea1f419bc859466bcc47878ee" | |
| }, | |
| { | |
| "dataPath": "params_shard_34.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.17.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3f904362ca645a27908a5ddb3af6fe8e" | |
| }, | |
| { | |
| "dataPath": "params_shard_35.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.17.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "9327030a82979c6f0b956c2004bf55ef" | |
| }, | |
| { | |
| "dataPath": "params_shard_36.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0136610c46ce7572c18a329905b38cb3" | |
| }, | |
| { | |
| "dataPath": "params_shard_37.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "b2d14b4910f2925ce449539beb4276d8" | |
| }, | |
| { | |
| "dataPath": "params_shard_38.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.16.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.17.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.17.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.17.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.17.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.18.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.18.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.18.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.18.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.18.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "a720c26c8caab158ca8d44fd540ef473" | |
| }, | |
| { | |
| "dataPath": "params_shard_39.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3e03fe3d8585a1eb94fcfa924b0e1eb6" | |
| }, | |
| { | |
| "dataPath": "params_shard_40.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.19.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "abff0df61e86f9521dbec4e50ac77038" | |
| }, | |
| { | |
| "dataPath": "params_shard_41.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.19.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "25c3a2dff2a3ca9f8ee1ea8dddc8c3f1" | |
| }, | |
| { | |
| "dataPath": "params_shard_42.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.19.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "07962eb90630eaec197d525039e2e9df" | |
| }, | |
| { | |
| "dataPath": "params_shard_43.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3cea15f82e21560e1cce5d8063947824" | |
| }, | |
| { | |
| "dataPath": "params_shard_44.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8365db0f576a14c95219fc9dda08e260" | |
| }, | |
| { | |
| "dataPath": "params_shard_45.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.18.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.19.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.19.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.19.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.19.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.2.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.2.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.2.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.2.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.2.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "9bc1a5fd7432ffaa65afc68a8c3a6e74" | |
| }, | |
| { | |
| "dataPath": "params_shard_46.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3a49f142bf7930c0ee5a94e20f010f13" | |
| }, | |
| { | |
| "dataPath": "params_shard_47.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.20.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8dcced55b9eaec4b718b9a56f78e4d49" | |
| }, | |
| { | |
| "dataPath": "params_shard_48.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.20.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "5a75fd46e30ccf641daa0636b5dd0264" | |
| }, | |
| { | |
| "dataPath": "params_shard_49.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.20.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "5eedc47ccdcff35f60877503bf6e66b9" | |
| }, | |
| { | |
| "dataPath": "params_shard_50.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.21.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "c0b683977d7a2edc44653b064306150d" | |
| }, | |
| { | |
| "dataPath": "params_shard_51.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.21.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "f125a8a4b3545863244f18623cc512da" | |
| }, | |
| { | |
| "dataPath": "params_shard_52.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.2.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.20.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.20.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.20.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.20.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.20.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.20.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.20.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.21.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.21.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.21.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.21.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.21.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "6947b187e1ca2190ed6311f3915e4ef8" | |
| }, | |
| { | |
| "dataPath": "params_shard_53.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.21.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "b899b0dbcb031aeacc451306c8f1ae25" | |
| }, | |
| { | |
| "dataPath": "params_shard_54.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.22.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "bd8ac225ee2c277df3c93b493d2d8933" | |
| }, | |
| { | |
| "dataPath": "params_shard_55.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.22.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "037c2f1c81ba45bedc504c8856457dde" | |
| }, | |
| { | |
| "dataPath": "params_shard_56.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.22.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8404f1afc77cc40a7178a0df1c7c9f39" | |
| }, | |
| { | |
| "dataPath": "params_shard_57.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "c1bb8512e839bf599a9ac98892009925" | |
| }, | |
| { | |
| "dataPath": "params_shard_58.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "b96e8aafaac424695560941d5e50ad30" | |
| }, | |
| { | |
| "dataPath": "params_shard_59.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.21.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.21.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.22.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.22.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.22.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.22.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.22.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.22.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.22.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.3.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.3.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.3.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.3.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.3.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "af858ce782d911fb513eee844e091a40" | |
| }, | |
| { | |
| "dataPath": "params_shard_60.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "26a07301f01ffa3e7a3a6149f65e405e" | |
| }, | |
| { | |
| "dataPath": "params_shard_61.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "1f28448279bf1ba57250ee22add97da3" | |
| }, | |
| { | |
| "dataPath": "params_shard_62.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "fd1c56b1710979e1d82ec440fdd94df0" | |
| }, | |
| { | |
| "dataPath": "params_shard_63.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "928dfd41b96d24a6ee5830f614e90a45" | |
| }, | |
| { | |
| "dataPath": "params_shard_64.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3fa5a9f5f7bc1f159a4ebda20fb47362" | |
| }, | |
| { | |
| "dataPath": "params_shard_65.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "64d5b6885efe51ccd3d7103ea0761603" | |
| }, | |
| { | |
| "dataPath": "params_shard_66.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.3.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.4.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.4.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.4.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.4.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.4.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.4.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.4.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.5.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.5.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.5.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.5.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.5.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "3e0eeb97a79924e2e4783a809fe0eaef" | |
| }, | |
| { | |
| "dataPath": "params_shard_67.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "17167b52036123547f13af01f8b693f4" | |
| }, | |
| { | |
| "dataPath": "params_shard_68.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "5d6e54c3dc08202399c9d31999f533d5" | |
| }, | |
| { | |
| "dataPath": "params_shard_69.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "977cac1bc6c99b131e2ce1a4af98704a" | |
| }, | |
| { | |
| "dataPath": "params_shard_70.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8448e6e8d38c3886c23714baeacd9c37" | |
| }, | |
| { | |
| "dataPath": "params_shard_71.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "c980565fe25899f39a70dd6b47eb58e6" | |
| }, | |
| { | |
| "dataPath": "params_shard_72.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "30caa3ec65e21a25c14b8c8a49a80553" | |
| }, | |
| { | |
| "dataPath": "params_shard_73.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.5.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.6.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.6.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.6.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.6.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.6.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.6.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.6.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.7.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.7.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.7.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.7.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.7.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "699afebf9e308fc569b732d547a30299" | |
| }, | |
| { | |
| "dataPath": "params_shard_74.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "36abbb0380d0d56a8713550c5e6734ff" | |
| }, | |
| { | |
| "dataPath": "params_shard_75.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "fcbe8f8702945e3109b6471d11bb78f0" | |
| }, | |
| { | |
| "dataPath": "params_shard_76.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "86725f22c1a86fab7db93b993e01afcd" | |
| }, | |
| { | |
| "dataPath": "params_shard_77.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "e6c2f54635dd7446a39479f3211d1e1a" | |
| }, | |
| { | |
| "dataPath": "params_shard_78.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "eb041af6660abcd5f0304fd440ef766a" | |
| }, | |
| { | |
| "dataPath": "params_shard_79.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "ac71cfcc030d327a40baf3f6569f7715" | |
| }, | |
| { | |
| "dataPath": "params_shard_80.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.7.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.8.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.8.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.8.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.8.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.8.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.8.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.8.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.9.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.9.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.9.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.9.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.9.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "83756665eb8eaa3a960f154ddd8314cd" | |
| }, | |
| { | |
| "dataPath": "params_shard_81.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "1f1fe500f7ea4efcf70db80a565ea111" | |
| }, | |
| { | |
| "dataPath": "params_shard_82.bin", | |
| "format": "raw-shard", | |
| "nbytes": 13122560, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.9.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13107200 | |
| } | |
| ], | |
| "md5sum": "e7df98843394b6d06546d18800e65587" | |
| } | |
| ] | |
| } |