Hugging Face's logo

mlc-ai
/
RedPajama-INCITE-Chat-3B-v1-q4f32_1-MLC

MLC-LLM
web-llm
Model card Files Community
RedPajama-INCITE-Chat-3B-v1-q4f32_1-MLC / ndarray-cache.json
CharlieFRuan's picture
CharlieFRuan
Initial commit
7352892
raw
history blame
203 kB
{
"metadata": {
"ParamSize": 518,
"ParamBytes": 1738526720.0,
"BitsPerParam": 5.010408347335939
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 64552960,
"records": [
{
"name": "gpt_neox.embed_in.q_weight",
"shape": [
50432,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 64552960,
"byteOffset": 0
}
],
"md5sum": "e65e8796e5e8cce8ac7ea3dfbf8a288a"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 32604160,
"records": [
{
"name": "gpt_neox.embed_in.q_scale",
"shape": [
50432,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 16138240,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16138240
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16148480
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16158720
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16168960
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16179200
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26009600
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28467200
},
{
"name": "gpt_neox.layers.0.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28497920
},
{
"name": "gpt_neox.layers.0.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 31774720
},
{
"name": "gpt_neox.layers.0.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32593920
}
],
"md5sum": "506e9427406c74b0bf5b4b24fb00cbae"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.1.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.1.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "6185ffeb1b33da59d73ae2967c6bb442"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.1.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.1.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.1.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "8f749f91ddbab931dd212fa1b4df0d51"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.2.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.2.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.2.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.2.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.2.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "6e3e620ee925777df20f56f5b88e2b35"
},
{
"dataPath": "params_shard_5.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.3.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.3.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "6799407e1a1eb4c5f3417c9c5709f642"
},
{
"dataPath": "params_shard_6.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.3.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.3.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.3.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "d6fe5397b8cc5e0ad3a66b7c7a3536bf"
},
{
"dataPath": "params_shard_7.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.4.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.4.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.4.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.4.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.4.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "b47ab02c5b6de722a6ea74fab12f2137"
},
{
"dataPath": "params_shard_8.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.5.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.5.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "9100970503defeb117f58f11f03ea8de"
},
{
"dataPath": "params_shard_9.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.5.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.5.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.5.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "dab7777ac759ba43db020d9c61ca24e9"
},
{
"dataPath": "params_shard_10.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.6.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.6.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.6.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.6.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.6.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "5e9b5beac432bd7587c2f9abc49b4b3b"
},
{
"dataPath": "params_shard_11.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.7.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.7.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "a71f63374fe69d389a9cf465ee611aca"
},
{
"dataPath": "params_shard_12.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.7.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.7.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.7.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "e85556f704c2f3bb85c69e8ce71c2b2e"
},
{
"dataPath": "params_shard_13.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.8.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.8.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.8.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.8.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.8.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "f22b0bd4f27faf2427f206899591181d"
},
{
"dataPath": "params_shard_14.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.9.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.9.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "2438429194fbbac9c078460ca03c507c"
},
{
"dataPath": "params_shard_15.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.9.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.9.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.9.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "ca6811a361c3c1067ca595cc5d6ce65f"
},
{
"dataPath": "params_shard_16.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.10.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.10.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.10.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.10.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.10.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "6b5b31948d57cbf3553792e07dab0ad2"
},
{
"dataPath": "params_shard_17.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.11.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.11.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "820ad1a5ec9ec66e960a27f157f80648"
},
{
"dataPath": "params_shard_18.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.11.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.11.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.11.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "00acc9917321b899ff99ae3750804355"
},
{
"dataPath": "params_shard_19.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.12.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.12.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.12.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.12.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.12.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "af475bbe569923eb2a2356049bf8ca83"
},
{
"dataPath": "params_shard_20.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.13.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.13.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "c368499847e95159e151f2fb82ee34ca"
},
{
"dataPath": "params_shard_21.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.13.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.13.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.13.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.13.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "d9655a3d9c6e7ea22f16eba46828c3ba"
},
{
"dataPath": "params_shard_22.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.14.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.14.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.14.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.14.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.14.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "c1c1c1dca1a0fd1d23ac2743c94df013"
},
{
"dataPath": "params_shard_23.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.15.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.15.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "5a574a2d682d4459daec407348aa7a43"
},
{
"dataPath": "params_shard_24.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.15.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.15.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.15.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.15.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "754acaa5c6b0dc8316a437e14ea2f430"
},
{
"dataPath": "params_shard_25.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.16.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.16.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.16.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.16.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.16.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.16.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.16.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "b26084a5f4defe2db8644ac3c8dc1d44"
},
{
"dataPath": "params_shard_26.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.17.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.17.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.17.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.17.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "7525b918517a80e5d548f614d2fcd2e0"
},
{
"dataPath": "params_shard_27.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.17.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.17.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.17.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.17.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "1ae1c39414fc4a973c739c12433a6443"
},
{
"dataPath": "params_shard_28.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.18.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.18.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.18.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.18.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.18.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.18.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.18.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "dd741f555678b27d5707a01d9be859a2"
},
{
"dataPath": "params_shard_29.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.19.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.19.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.19.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.19.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "88fbad4e8f625461904603962dd4e3df"
},
{
"dataPath": "params_shard_30.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.19.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.19.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.19.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.19.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "347678378af964ef64328217a72063bf"
},
{
"dataPath": "params_shard_31.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.20.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.20.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.20.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.20.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.20.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.20.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.20.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "e3d095771bb04ff0ec6a2ebed2d72652"
},
{
"dataPath": "params_shard_32.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.21.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.21.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.21.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.21.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "a08359c8c11cda410d439aec53a7c687"
},
{
"dataPath": "params_shard_33.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.21.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.21.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.21.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.21.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "4db4063ca67dd7b6d3437df6d0598228"
},
{
"dataPath": "params_shard_34.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.22.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.22.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.22.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.22.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.22.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.22.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.22.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "eba53144a64724705d075065a345a4f9"
},
{
"dataPath": "params_shard_35.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.23.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.23.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.23.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.23.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "fd7850af14a6a8d87cf20d105acfa0a3"
},
{
"dataPath": "params_shard_36.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.23.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.23.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.23.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.23.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "2508be19561d35b5e2c2ff8a53e9a269"
},
{
"dataPath": "params_shard_37.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.24.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.24.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.24.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.24.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.24.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.24.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.24.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.24.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.24.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.24.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "aa9d9236ec75307d77a3b57db9bc72ba"
},
{
"dataPath": "params_shard_38.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.24.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.24.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.24.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.24.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.24.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.24.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.25.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.25.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.25.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.25.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "b250bb0f6165f4317a256902e5093657"
},
{
"dataPath": "params_shard_39.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.25.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.25.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.25.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.25.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.25.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.25.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.25.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.25.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.25.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "fbe1041cc8d2b2e1829bb9d6dba62028"
},
{
"dataPath": "params_shard_40.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.25.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.25.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.25.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.26.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.26.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.26.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.26.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.26.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.26.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.26.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.26.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.26.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.26.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "8f928520e3d689e7726d18548206366d"
},
{
"dataPath": "params_shard_41.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.26.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.26.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.26.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.26.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.26.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.26.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.27.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.27.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.27.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.27.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "409b12f1645ef4fdd9a61bd0da8e83fe"
},
{
"dataPath": "params_shard_42.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.27.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.27.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.27.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.27.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.27.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.27.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.27.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.27.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.27.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "33c3d76b713c519d7fe90bfd80e93def"
},
{
"dataPath": "params_shard_43.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.27.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.27.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.27.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.28.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.28.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.28.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.28.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.28.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.28.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.28.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.28.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.28.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.28.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "87cf7e48f7d6cc520be0fae00115ff57"
},
{
"dataPath": "params_shard_44.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.28.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.28.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.28.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.28.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.28.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.28.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.29.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.29.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.29.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.29.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "1667fb8df1a66ee054283caea6113da9"
},
{
"dataPath": "params_shard_45.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.29.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.29.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.29.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.29.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.29.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.29.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.29.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.29.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.29.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "f448efc3ae9e949d9343581e78fa64ef"
},
{
"dataPath": "params_shard_46.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.29.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.29.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.29.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.30.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.layers.30.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "gpt_neox.layers.30.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.30.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.30.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 16435200
},
{
"name": "gpt_neox.layers.30.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 26265600
},
{
"name": "gpt_neox.layers.30.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 28723200
},
{
"name": "gpt_neox.layers.30.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 28753920
},
{
"name": "gpt_neox.layers.30.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 32030720
},
{
"name": "gpt_neox.layers.30.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "37b201b978536e7abf8694075a364488"
},
{
"dataPath": "params_shard_47.bin",
"format": "raw-shard",
"nbytes": 32860160,
"records": [
{
"name": "gpt_neox.layers.30.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.30.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.30.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 16384000
},
{
"name": "gpt_neox.layers.30.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.30.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.30.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32808960
},
{
"name": "gpt_neox.layers.31.input_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32819200
},
{
"name": "gpt_neox.layers.31.input_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32829440
},
{
"name": "gpt_neox.layers.31.post_attention_layernorm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32839680
},
{
"name": "gpt_neox.layers.31.post_attention_layernorm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 32849920
}
],
"md5sum": "77bf1ed46cfaf71cf88bbf05609cb3fc"
},
{
"dataPath": "params_shard_48.bin",
"format": "raw-shard",
"nbytes": 32849920,
"records": [
{
"name": "gpt_neox.layers.31.attention.query_key_value.q_weight",
"shape": [
7680,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 9830400,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.31.attention.query_key_value.q_scale",
"shape": [
7680,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 2457600,
"byteOffset": 9830400
},
{
"name": "gpt_neox.layers.31.attention.query_key_value.bias",
"shape": [
7680
],
"dtype": "float32",
"format": "raw",
"nbytes": 30720,
"byteOffset": 12288000
},
{
"name": "gpt_neox.layers.31.attention.dense.q_weight",
"shape": [
2560,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 12318720
},
{
"name": "gpt_neox.layers.31.attention.dense.q_scale",
"shape": [
2560,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 819200,
"byteOffset": 15595520
},
{
"name": "gpt_neox.layers.31.attention.dense.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16414720
},
{
"name": "gpt_neox.layers.31.mlp.dense_h_to_4h.q_weight",
"shape": [
10240,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 16424960
},
{
"name": "gpt_neox.layers.31.mlp.dense_h_to_4h.q_scale",
"shape": [
10240,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 29532160
},
{
"name": "gpt_neox.layers.31.mlp.dense_h_to_4h.bias",
"shape": [
10240
],
"dtype": "float32",
"format": "raw",
"nbytes": 40960,
"byteOffset": 32808960
}
],
"md5sum": "3b34c10fdd5704db70771615e5e5ab6d"
},
{
"dataPath": "params_shard_49.bin",
"format": "raw-shard",
"nbytes": 64552960,
"records": [
{
"name": "embed_out.q_weight",
"shape": [
50432,
320
],
"dtype": "uint32",
"format": "raw",
"nbytes": 64552960,
"byteOffset": 0
}
],
"md5sum": "25ea3e71fa498e95a5f1849d36ab9f50"
},
{
"dataPath": "params_shard_50.bin",
"format": "raw-shard",
"nbytes": 32552960,
"records": [
{
"name": "gpt_neox.layers.31.mlp.dense_4h_to_h.q_weight",
"shape": [
2560,
1280
],
"dtype": "uint32",
"format": "raw",
"nbytes": 13107200,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.31.mlp.dense_4h_to_h.q_scale",
"shape": [
2560,
320
],
"dtype": "float32",
"format": "raw",
"nbytes": 3276800,
"byteOffset": 13107200
},
{
"name": "gpt_neox.layers.31.mlp.dense_4h_to_h.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16384000
},
{
"name": "gpt_neox.final_layer_norm.weight",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16394240
},
{
"name": "gpt_neox.final_layer_norm.bias",
"shape": [
2560
],
"dtype": "float32",
"format": "raw",
"nbytes": 10240,
"byteOffset": 16404480
},
{
"name": "embed_out.q_scale",
"shape": [
50432,
80
],
"dtype": "float32",
"format": "raw",
"nbytes": 16138240,
"byteOffset": 16414720
}
],
"md5sum": "3d49453ac0ed0334cf213967b00a0e7b"
}
]
}