{
"measurement": {
"model.layers.0.self_attn": [
{
"accuracy": 0.9686112372498763,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.971794909552524,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9756869595301779,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822374550919784,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825488438731745,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845718145370483,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834211076560774,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855356890904275,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878572719661813,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883109958548295,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919881946162173,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932752259467778,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924710408637398,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937774633106432,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951705156188262,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967665358593589,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952879618657263,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978175653438819,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986901419531358,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.0.mlp": [
{
"accuracy": 0.9756452943149366,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773805533584795,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9782680037774538,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811145192698428,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897316655046061,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905060835574803,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992095128486031,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928499648445531,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935189014986942,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945449119335726,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950070663502342,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966300157339949,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968791166810613,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971661991194675,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980436355659836,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998352359588209,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998414587621626,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.self_attn": [
{
"accuracy": 0.9455061335312693,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9496199394527235,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9585136520235162,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9702272164194208,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9705966491448251,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9712596786649603,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9748516255303433,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755643010139465,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9746062379134329,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9748611512937044,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829107475908179,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841665358919847,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833361255495172,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846546744045458,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899474186332602,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914672178657431,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907307248366507,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969221545677436,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973794547350783,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.mlp": [
{
"accuracy": 0.9625875949859619,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9635381698608398,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742246144696286,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9779468570884905,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825188282289004,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839296246829786,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988656538097482,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.99077848933245,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918668913213831,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911660371642363,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923083343003926,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954525357798526,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961347317225054,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975049530989245,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976748430023068,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987509068297712,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992298022598812,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.self_attn": [
{
"accuracy": 0.9618843291935167,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9658066379396539,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9702556321495458,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798572079131478,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804119640275052,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806513425550962,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837499919690584,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840678541283858,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852429738170222,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856039379772387,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897997418516561,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902921867998022,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903069687517065,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990882155926604,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944449506307903,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952622761851863,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951256948866343,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977873077518061,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987045322593889,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.mlp": [
{
"accuracy": 0.9862513338264666,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876837118675834,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885524017246146,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889313955056039,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899125459947085,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994016518718318,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943534367178616,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949118769482562,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936869858126891,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975180543567005,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928554523932306,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983484303872836,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998309447969261,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984482306202775,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983382172098285,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998364545502945,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984575306114397,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.self_attn": [
{
"accuracy": 0.9980073122209624,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982002344178525,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984323067688629,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990024142163364,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991372581571341,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991467611766175,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994543610924953,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994699555988374,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995216598342124,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995461873416054,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995668991713932,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995896916983551,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996121379203702,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996443039884693,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997880655389867,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998202637132061,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998580013940993,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998988057241628,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999529099931304,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.mlp": [
{
"accuracy": 0.9968132231580583,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968790331561315,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975432881007069,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977448674801149,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998414892311159,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985298378098952,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988065507066878,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991132656210348,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991888300956864,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991280901196756,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992168824629564,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995860522612929,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996408816721094,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997602200233623,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997936974426633,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998526461054816,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999275608478408,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.self_attn": [
{
"accuracy": 0.9977589365290968,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979614333102578,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982419792366656,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989032593408698,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990082120424822,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990347620884055,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993865490940056,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994314814378557,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994694717522514,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994963890333709,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995078550357568,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999540279031192,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995682264040959,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996013371273875,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997595760873274,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997986015545106,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998399118255628,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998873840453789,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999468153477401,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.mlp": [
{
"accuracy": 0.9966510228420559,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967206506744811,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974138540656943,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976367479876468,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998372619873599,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984980802983046,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988048347203355,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.999158501821129,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.999243600097926,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991772864994249,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999279950059166,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995804284454176,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996409781620299,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997748476453125,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997859987381258,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998480528347978,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999323858646676,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.self_attn": [
{
"accuracy": 0.997586330693019,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977949363620657,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980961609827844,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987467120548612,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989693853023806,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998987142094656,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993782189527625,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994044672991884,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994440317448032,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994729292628012,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994848624950177,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995164488883395,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995440878464203,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995771006337906,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997463776533934,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997865989019996,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998330429784561,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998801817357736,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999447814679068,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.mlp": [
{
"accuracy": 0.9959781307138895,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960649409576466,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968830423527643,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997149219442355,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980385964246172,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981843760531199,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985447797532144,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989537990799076,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.999050787208896,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990040948124308,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990985960160431,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994927915303331,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995427603196156,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997201466834859,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997343342065027,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999807851462576,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999027946890381,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.self_attn": [
{
"accuracy": 0.997588281568728,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977666008237162,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981271793183527,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998762318276261,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989541195529071,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989680467467559,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993846449314764,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994115306946793,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994550391443466,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994819478965119,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994767183732045,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995104022520153,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995537386520913,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995860748581196,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997532739500073,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997913164429759,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998360128681126,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998820275313368,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999445991615128,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.mlp": [
{
"accuracy": 0.9957130123910151,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958045827715021,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966449717942037,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969135095414362,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979121232111203,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980752403406721,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984506582351107,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989200703015453,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990264057720962,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998942159606438,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990743198677113,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994600700018438,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995380214562541,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997097626083383,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997241760996219,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998010701842999,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999132915645054,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.self_attn": [
{
"accuracy": 0.9971448212469879,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997336252543487,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977381129406,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985238534251326,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987095333635807,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987388643387117,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992837784321684,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992992308873095,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993485309869835,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993808874860406,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993596426456383,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994026559748148,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999459797908601,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994994208314701,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996986114419997,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999745906477696,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998035185823315,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998635557039004,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999361640749205,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.mlp": [
{
"accuracy": 0.9949821618042494,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950889239185735,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960478022694588,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963471575787193,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975343056415257,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977274590024823,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981484822928905,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987327163352778,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988534036827715,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998752074120076,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989079796758137,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993638399694311,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994554527005867,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996597443364168,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996753456071019,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997578928933332,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998959514287937,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.self_attn": [
{
"accuracy": 0.996560080663154,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968025249085928,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972552106176552,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981577549325792,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984261103366551,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984458152596888,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99905755214001,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990826529499731,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991475142734615,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991951198072025,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999213415345079,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992602864084276,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993384723110419,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993889781793481,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996343769966379,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99968974718726,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997487989450363,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999815572004177,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9999133879073748,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.mlp": [
{
"accuracy": 0.9942348132792272,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943539535528735,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954330725105185,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957683639306771,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971674498366682,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973905374737162,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978630954497739,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985451559095007,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986861213845643,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985653275721952,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987450563593915,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992683602889118,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993741397598856,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996091614134217,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996265223690946,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997199828442382,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998836251539424,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.self_attn": [
{
"accuracy": 0.9958042518088692,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960420496369663,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965162947773933,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977874436268681,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980520402130327,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980762873433138,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988941340462157,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989209131974923,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998988451240094,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990415310389117,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990235494547769,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990850688007317,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999170683324337,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992311688042,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995476387343124,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996139476271836,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997099982084412,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997834756872371,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998998023005888,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.mlp": [
{
"accuracy": 0.9935971600444693,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.993729083161605,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949304622254873,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953010948865038,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968589999173817,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971003542213064,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997626291097779,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983857650505868,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985381001116413,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984096636701572,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986047832001197,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991890217030519,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993045050650835,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995663501322269,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995858765354282,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996906761570197,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998725817391747,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.self_attn": [
{
"accuracy": 0.9952479859716014,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954862908313149,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962403158608236,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975251975028139,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997679959002294,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977519339636752,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987164961272165,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987622712199625,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988490337211835,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988995584610262,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988524850065771,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989379652236637,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990931189569988,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991588065106618,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994989834039619,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995758290353575,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996614147614884,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997658821705141,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998874705603444,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.mlp": [
{
"accuracy": 0.9935801197823725,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937087642519098,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949276219857367,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953125872110066,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968365324955237,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970833998369543,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997621122159456,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983683371622312,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985274660744166,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983990963940558,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985972129200634,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991832792170738,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999300449155271,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995626697904969,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995835839133513,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996904477133954,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998622874998929,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.self_attn": [
{
"accuracy": 0.9954094718161383,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957549054371683,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996224702580979,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973333640710303,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978803756989931,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979125382868868,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987601000619563,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998777810288103,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988713147804925,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989269471874362,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989494527445028,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990124409332087,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991256150564081,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991866209495225,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999516754313127,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995877506995672,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996679972023949,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997622800108633,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998915482214407,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.mlp": [
{
"accuracy": 0.993170509997167,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933079536023893,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946141807656539,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950312156426279,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966140989410249,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968855373169246,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974633469001243,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982518706666795,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984269000001644,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982892131726993,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985024065367485,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991285103912416,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992530056226411,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995331995581326,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995550167697825,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996686474616198,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998554057368126,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.self_attn": [
{
"accuracy": 0.9956285969207161,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.995874800180134,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963486633802715,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997692136388076,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979426156925527,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979817253586493,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988545369552938,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988924947224165,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989599911986213,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990130173728654,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989778009292326,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990406628502043,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991324442977968,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991938470813789,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995267534334409,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995951322642597,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997018254852217,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997758301416118,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998923265252655,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.mlp": [
{
"accuracy": 0.9926817511257372,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928357891346279,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942288210517481,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946711212396622,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963763701288324,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966568864489856,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997271040160405,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981303785584474,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983099108855975,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981636857908023,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983911006466338,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990649636050588,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991981363120047,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994992126073492,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995222695563969,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996409589473746,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998424870305156,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.self_attn": [
{
"accuracy": 0.995741953975276,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960334116691038,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965343836106753,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976221813183082,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980953771032786,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981246047506207,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988950913477885,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989286080787056,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990032038798458,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990553377490294,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990421220856277,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991061413954747,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991892242902204,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992479201602308,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995567487846864,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996211448145148,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997110515832901,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997871692145341,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999901101072809,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.mlp": [
{
"accuracy": 0.9921892481414896,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923547443590666,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938169921699324,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942788928747177,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.996127897579419,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964302723345003,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970755271221462,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980108465411162,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982010106507101,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980435304735836,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982849539894807,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990058044265759,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991459169081951,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994682404457739,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.999492824616793,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.99961642644025,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998342378530651,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.self_attn": [
{
"accuracy": 0.994979187454048,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952567753038908,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959768080397656,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971938360678522,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974546422691721,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975412666405502,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985981197341492,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986430479115561,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998734403793749,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987851406790709,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998740205737321,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988262348465229,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990218175869239,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991055563288299,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994673596106862,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995486674536216,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996289992704988,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997528455661315,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998806828118273,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.mlp": [
{
"accuracy": 0.9918802678585052,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920496070071271,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935618854667011,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940472021698952,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959707146412448,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962811826875335,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969497385777926,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979250864370873,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981221854686737,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979633992831958,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982144822807688,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989649175029052,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991108543778721,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999445989531906,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994730930775404,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996047119953131,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998231776254741,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.self_attn": [
{
"accuracy": 0.9949167386481637,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.995171394003065,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959633224888852,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972144904497423,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974337467237523,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975249094790534,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985228928884393,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985852427780628,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986739035107588,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987230224436835,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986950255146152,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988006585718769,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990218995433104,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999092763976047,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99946736862981,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995467867212076,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996208983571514,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997429538420156,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998789690312391,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.mlp": [
{
"accuracy": 0.9907711234531904,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909857495834953,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926284598676782,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931477904319763,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954544556768317,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958149469212482,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965401327139453,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976120193145777,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978459736234263,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976787537728485,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979333010943312,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988269814731259,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989658050043019,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993652769511467,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993986321221057,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995402481014791,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997926245835659,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.self_attn": [
{
"accuracy": 0.9942090715232649,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945659974687978,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954169427877978,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968359295866991,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971434509283618,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972500993232978,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983431332205471,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998389188984507,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985158119938875,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985864992792669,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985752892925551,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986848137096355,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988844898578367,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989718955598379,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993890116089269,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994812207786661,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995693782050359,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996930779399056,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998547452452936,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.mlp": [
{
"accuracy": 0.9894104678379861,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896353216547715,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.991569951960915,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921815505153254,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947418488169971,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951533922239354,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959953697888475,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972947602601427,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975499318618524,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973400788087594,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976689219474792,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986474075795788,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988391217040388,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992764858332904,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993112749469123,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994807078650123,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997728895886164,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.self_attn": [
{
"accuracy": 0.9941051782746064,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994355261717972,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952801175807652,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966144475497698,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968775487259814,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969954351453405,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983077886465349,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983246479379503,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984673560646019,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985166701831316,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984260304389816,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985325664870048,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988366181128904,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989087813974995,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993707326878059,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994608732430559,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995490983618718,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999720359784796,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998468780939124,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.mlp": [
{
"accuracy": 0.9882936250222357,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885503328160236,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906127586176521,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991252097644304,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994206535973047,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946584689774012,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995549142752823,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970218427479267,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997292331763004,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970681222253724,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974291530486784,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985077284080418,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987199904495164,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992005428985545,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992391439644914,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994145361216444,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997509317472577,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.self_attn": [
{
"accuracy": 0.9936430179759076,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939522264819396,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948800041487342,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964734907997282,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968936133541559,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969762526452541,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982244540986261,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982948171857157,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984109263475004,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984877103645551,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984522664821461,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985475016659812,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987467713654041,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988423738824693,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993030947976207,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994141618280035,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995301037438606,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996741250353424,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998460795583302,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.mlp": [
{
"accuracy": 0.9867961375336898,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870931631640384,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893966309334102,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900958789022345,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934569573716113,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939670068653006,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949462751024648,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966515773221066,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969553259250364,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966893078465211,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970946809963176,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983171908871123,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985541495445528,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991009134407106,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991419335925266,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993358636274934,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997171966947223,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.self_attn": [
{
"accuracy": 0.9941833529033159,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994436592255768,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956454462126682,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968505882118878,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971891955325478,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972580957569575,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984294634900595,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984774234655657,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985863818345886,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986432813304035,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986411427196703,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987291907401461,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989490445311132,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990295072723376,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994212769364056,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999508474120184,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995830448736486,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997133822670501,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998632391032419,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.mlp": [
{
"accuracy": 0.9858320920090926,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861442509450411,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885558674209997,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892837311092176,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929553991869876,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935119697137883,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945374723327788,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963971835217977,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967259927407691,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.996433353737781,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968767091631889,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981867869041468,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984451507855403,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990330846293977,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.999074832781365,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992777173848528,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996991791321259,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.self_attn": [
{
"accuracy": 0.9930108777786556,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934402727767041,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945706838839933,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996196952305342,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965360662654826,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965996875574714,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979902784291067,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980720165920886,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998208868464357,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998280473837727,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982442510755438,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983621076925805,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986543237770859,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987471314441216,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992629601865223,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993768204493743,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994772988322534,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996457682236245,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998362177713332,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.mlp": [
{
"accuracy": 0.9846734279080441,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850160008982608,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876318871974945,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884080282951656,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923915078765467,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929880766492141,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940928256041125,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961229468646803,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964718089291924,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961541596996156,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966285216965174,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980479684707365,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983233015396094,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989629740777769,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990071190619155,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999221649413046,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999662878114338,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.self_attn": [
{
"accuracy": 0.9919458047339791,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922779116191363,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936409631842061,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955245377986055,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959419001089899,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996055922225902,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976935453320804,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977927870656315,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979693746488345,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980544492994484,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979657448436084,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981125540247089,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984351158338157,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985464737799606,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991342288098837,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992686256668285,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994028599344587,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996023499406874,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998230006129137,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.mlp": [
{
"accuracy": 0.9834267622546146,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837999312501204,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865659067505285,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873826323371184,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917935509430734,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924438925165879,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936145363669646,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958119651204661,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961945328273272,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958539899242552,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996354862655464,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978912147252184,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981882787848774,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988720218994116,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998908202506994,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999123537520829,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995857448268094,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.self_attn": [
{
"accuracy": 0.991864413022995,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992324843218452,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938130237554249,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953963705583623,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959745752184015,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961032208643461,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975154192039841,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976562481177481,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978183907897848,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979087516273323,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979617948594847,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980967866354867,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998438648879528,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985634002246355,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991485298072037,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992712514573022,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993674535500375,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99957931326016,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998039427262387,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.mlp": [
{
"accuracy": 0.9821474850177765,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825561485792461,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855361910242784,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864091183009901,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911551914717022,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918435380647057,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930932215954128,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955021288049849,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958986592920203,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955308782426935,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960787210025286,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977325304950538,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980504695130022,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987962071440721,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988447360106205,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990811836170522,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996138204771438,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.self_attn": [
{
"accuracy": 0.9905074552485817,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908358870368255,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925143154043901,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947110339999199,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995261873853834,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954250996050081,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973048150147262,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973144654772783,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975693112141207,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976737154157538,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975399823957368,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977870814894375,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981503929746779,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982991508747402,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989761694481498,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991474276114451,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999281487280601,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999535915539845,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997909374191964,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.mlp": [
{
"accuracy": 0.9807807646299663,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.981220576323961,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844186588337547,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853518605232239,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904862203096089,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912313858145162,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925706810072848,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951629293592352,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955899256624674,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951920121123916,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957846971718889,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975630199830783,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979043702938055,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987045471605501,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987575516692901,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990111712954546,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995776082045937,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.self_attn": [
{
"accuracy": 0.9898300343438199,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990377246549255,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920749287856253,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941852688789368,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947503052259746,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949336032334127,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969021715223789,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971267837834986,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973001136983696,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973981639272288,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973182870369208,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997538020344157,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980156790269049,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981821565643737,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989189076188364,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990928292666611,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992163219048005,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995118711928004,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997755105999347,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.mlp": [
{
"accuracy": 0.9791859137384515,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796656276050367,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831490500977165,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841585065189161,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897337093165046,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905300414875934,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919768548325488,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994780093039337,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952340212307478,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948132394960052,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954481838565123,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973700321034381,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997736076971418,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985988499890817,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986584819853306,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989313586173874,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995440985321214,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.self_attn": [
{
"accuracy": 0.9879401388921236,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880624413490295,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898578603016702,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924437882084596,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934509184799696,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937083599598784,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963395399482626,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964545745598642,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966748047031855,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968050418323592,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967015867954806,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969328824234637,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975048585941917,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997699582459111,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986198646457571,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988636038217106,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990249371254131,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993983258150125,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997157642715856,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.mlp": [
{
"accuracy": 0.9775604558618445,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9780812185061606,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819403729940716,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830473677108162,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889298034341711,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897860578800503,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913758070845353,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943798258900642,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948691121841732,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944116469276579,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950938409096316,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971659708964197,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975613465434626,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984970593726948,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985582253062412,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988557192447939,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995106491505316,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.self_attn": [
{
"accuracy": 0.9865093027290545,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871394014672229,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890923915725005,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918992715446573,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929729315795397,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931388160115794,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995959900711712,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959997511223743,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963712139349234,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965185468134127,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963461309671402,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966829387765181,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972809517854139,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974931600062471,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984848791439283,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987345311398569,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989151588003886,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999333191417942,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996602265164256,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.mlp": [
{
"accuracy": 0.9761340241683157,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.976696922590858,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807651513501218,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819339937285373,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882212845902694,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891382926388791,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908051804492348,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940229317075328,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945359257491011,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940497059571115,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947805808562982,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969846808204526,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974062046722362,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983958505878323,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984636102852068,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987719913846568,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994811256857294,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.self_attn": [
{
"accuracy": 0.9878712116103423,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882460848281258,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898476639860555,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99268460430597,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937842217715163,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938825819837419,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964543471210882,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965651419601942,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967836147468341,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996939758329015,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968199753447583,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970416089421824,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974875763842934,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976746275236732,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985965336427877,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988292964469445,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990447885111758,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993514632806182,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996787132940402,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.mlp": [
{
"accuracy": 0.9744201490753576,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750375292803112,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794641639056959,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807345835786117,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874105218209719,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883850985451749,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901934124921498,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936107636282319,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941565166962775,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936442477138419,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944208315328548,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967818005304587,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997226950761519,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982892161137179,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983602249504704,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986920196955141,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994229101820996,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.self_attn": [
{
"accuracy": 0.9856438134845934,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862781825818514,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882366469031886,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912729678969634,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924984440991753,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925732918475804,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957092984726554,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958918478928114,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961729653571781,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963403304940776,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962148356594538,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964690228041849,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970886928862647,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973098212166837,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983870189048742,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986577359469313,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988587867272528,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992717245691701,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996731115848219,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.mlp": [
{
"accuracy": 0.9726940801269129,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9733491163504752,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781758471539146,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.979566111376411,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865733541940388,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876134450498381,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895838875519601,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931688740065223,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937540896629032,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932194744285784,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940483860279384,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965629216871763,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970404260644787,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981670561981829,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982499186145631,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986172668439778,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993960753965535,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.self_attn": [
{
"accuracy": 0.979611580309115,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805067291385249,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834466607947099,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874825171734157,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897874499622145,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901611491253501,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994197309801453,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944611423109707,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949554353952408,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951587617397308,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950093744616759,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995301648190147,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960699038285958,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963486147554297,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978246973141244,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981894971508729,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985177417922961,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990279407877671,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995518584588641,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.mlp": [
{
"accuracy": 0.9715469008997867,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9722146235014263,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774369079815713,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9789643036691766,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859727652449357,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870577329083493,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892085069104245,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928343405849055,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934637040684098,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992907201773242,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937758959437671,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963998735735291,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969017935034475,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980772453311243,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981677828258589,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985793873079514,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993758981949404,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.self_attn": [
{
"accuracy": 0.9766579646813242,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773845358898765,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800629239333304,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858587635190863,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883406272059992,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884097058522073,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99355732531924,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993627441948966,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941248254556405,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944335912403307,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994212801911329,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945917941237751,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951043544631255,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954943237336058,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973310810562811,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977644866234378,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998286209804447,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987586978822947,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994550130182975,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.mlp": [
{
"accuracy": 0.9690493658969277,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9697631566148055,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755354100152066,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9772207454631203,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847197767935301,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859017133712769,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882812711753344,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921902543620059,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928789060366782,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922631939775065,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932087957859039,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960679651090973,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966183904754488,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979038201272488,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979960410610625,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984526074442425,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99933998272019,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.self_attn": [
{
"accuracy": 0.9767356169851202,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774166502450642,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794091064678995,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848108213198813,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879189235599417,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.988138875678966,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933879422514063,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935303887254313,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941062895875228,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944709547256169,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993908624507879,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944249971916801,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949982256481522,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954747451763404,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973148978070209,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977271752922159,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982927565119768,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987903962794104,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994433670745868,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.mlp": [
{
"accuracy": 0.9669025634464464,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9676867252901981,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738576380830062,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9756402012548948,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836597285772625,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849321904935335,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874574095010757,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916534055220453,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923873771178094,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917224496603012,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927385645477396,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957904705875799,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963823929429054,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977540379684222,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978458336309383,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983363741714704,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992993656839979,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.32.self_attn": [
{
"accuracy": 0.9733894498724687,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9740624270941082,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9768056226404089,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834723645134976,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865951098893818,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867110142582342,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926763796492627,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928462175946486,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933310166785592,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935685913813742,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993288276226897,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937043362542203,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944117426087982,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948023973326934,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969783115543818,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974097419334086,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981410813174749,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986288355369317,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993958699173833,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.32.mlp": [
{
"accuracy": 0.9652219948015714,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.966031290982899,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9725615037114996,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9744476817156139,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827821442955419,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841229899933464,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868038453553852,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911943979953465,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919700387277102,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912701680472022,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923314546283922,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955594002416259,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961791638480989,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976319237367103,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977374745434836,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982593028168929,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992879678642279,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.33.self_attn": [
{
"accuracy": 0.97215084339443,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9748005663093767,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773762775094885,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843705908248299,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.987096914335301,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871334072790647,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922490653238798,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922699951811841,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992772879569154,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931837124259848,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935687658818144,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939749138919931,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943730678213271,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947456548872747,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969253124375093,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973737746477127,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980150643540057,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986214242679509,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994254561239168,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.33.mlp": [
{
"accuracy": 0.9628788326915941,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9638048033965261,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9707444655267816,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9726766253772535,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817342648380681,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831588974124507,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859292162092108,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906854213852632,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914813904385817,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907324047465074,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918648929972398,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952823609897965,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959449226918974,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974831685815987,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975873192674235,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981382059815683,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992595896811077,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.34.self_attn": [
{
"accuracy": 0.9734924749324196,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9743499222554659,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764338371000791,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823645588598753,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868769677061784,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869918768343172,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922130578442624,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922894168841211,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926210673231828,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993204077607707,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932795645375001,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936975840675203,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939842600571481,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944126562852609,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967085872041551,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972038172969693,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997849201685504,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985099365248492,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994080049034796,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.34.mlp": [
{
"accuracy": 0.9606064495287443,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9616273892553229,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9688912441855982,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9708973075214186,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806634595519618,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9821681066563255,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850358021886725,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901447633379384,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909643899453314,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901887139207438,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913796751122725,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995006442854279,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957042438419241,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973341804978094,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974473291321805,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980086174450422,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999186950245578,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.35.self_attn": [
{
"accuracy": 0.9713474167020697,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9722266071721127,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750017476709265,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814711652303997,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985838941837612,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985910492508035,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922613336851722,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992374767598353,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929443080174295,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933803097197884,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930025323441154,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935325250813836,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940995558312065,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945772850984022,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967580014153531,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997287396733698,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997995212478073,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984852526532976,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993747898720597,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.35.mlp": [
{
"accuracy": 0.9595794269913122,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.960643730665508,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9680738386354948,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9701378533714696,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801445587685234,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816910749987552,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846374047429938,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898729191014641,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907130567651046,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899228693623292,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991142000806959,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994865970784112,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955847298628405,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972599450694887,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973699656364164,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979518271590534,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991540882344309,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.36.self_attn": [
{
"accuracy": 0.9687091457216364,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9710902603049028,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738204839982485,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817511533436022,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853365766374689,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855695868793287,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991449258829418,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917181840068415,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992302246783909,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927633483158914,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992695122172958,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931602030992508,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936330941946883,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941209660548913,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996463374087685,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970021769404411,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977435636285105,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982880455882925,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991822282930738,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.36.mlp": [
{
"accuracy": 0.9580440866319757,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9591460133853712,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9669514016101235,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9691294086606879,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793878605491236,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810050020092412,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840797013358066,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894625717087796,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903353491896077,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895281454450205,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907977032034021,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946556754018131,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954082107073382,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997138318654738,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997265098126311,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978865657589937,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991366487780684,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.37.self_attn": [
{
"accuracy": 0.9697085117038927,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9706328730834158,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9730559618849504,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805716103629062,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846965121595483,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848399868136958,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908891508453771,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.991027790464853,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916258869986785,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921260060448396,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922362424825367,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927757915697599,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931640162279731,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936901503487637,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962285874705565,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968286207632014,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975897120802026,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982975976247537,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993460461203205,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.37.mlp": [
{
"accuracy": 0.9569805578181618,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9581804589221352,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9659490083393297,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.968091933350814,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9790382244085011,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.980663454846332,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836637111086595,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893191425423873,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901730786812933,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893617237869062,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906507667742277,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945786113017484,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953398692764734,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970941208302975,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972267046962914,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978353457623407,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990984022029137,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.38.self_attn": [
{
"accuracy": 0.9734144336298892,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.974829292611072,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774050116539001,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839586097943155,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867534370798814,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868747780197545,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923802504414007,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925973823196009,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930964206394396,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993473010627847,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932975361221715,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937652694551569,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944699328196677,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948673224762866,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969852996107779,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974342698329374,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980420938840038,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986126866975897,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993863573886062,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.38.mlp": [
{
"accuracy": 0.9589921016442149,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9600806016671031,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9675494118740684,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9696322867744848,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799241847113559,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814814313461906,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844023491206922,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.989748583028191,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905873105714196,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898028844281247,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910402376400796,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948014522853651,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955306680578935,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972143898669042,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973317588630476,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979247203782985,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990730779735666,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.39.self_attn": [
{
"accuracy": 0.9755538796123705,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766967924017655,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788361976021215,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850570590872514,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878464364691785,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.988037897568,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928429134582218,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930869074244248,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934771511899797,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938701806884063,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938537105917931,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942322985122078,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945934148211228,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950038124071924,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997013839844026,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974708494387174,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980484247207642,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986931188522201,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993965237058307,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.39.mlp": [
{
"accuracy": 0.9610545572481657,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9620394330275686,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690124423880326,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9710044860839844,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9808238760421151,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823103954917506,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850955025145882,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901856599669707,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910009519049996,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902626903433549,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914442002773285,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950343586112324,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957318580464313,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973431394288415,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974566605922423,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998024635409054,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991487233262313,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.40.self_attn": [
{
"accuracy": 0.9759947092909562,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9765293707973078,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793274716327065,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843808992912895,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878816596771541,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.987991850627096,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933141948361146,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935611980526071,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939959139416092,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943285313876051,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939067685290387,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943101241400367,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995069430846917,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995466962848839,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973181366528335,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977517951475946,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998238024154776,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987979885190725,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993936337628647,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.40.mlp": [
{
"accuracy": 0.9632799594025863,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9641819251211066,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9706869690041793,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9725444128638819,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.981893915879099,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832901327233565,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859133221601185,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907320816265909,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991518044158032,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908088124112079,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919172917541704,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953166799325692,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959699805629881,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974961810206112,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976121055844583,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998136376275828,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992173090086955,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.41.self_attn": [
{
"accuracy": 0.9778106855718713,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9780343567070208,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825682593019385,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871476308295601,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891814315005353,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892836111156564,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937379713121214,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939651763752887,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942810088396072,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994507155528194,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944074742103878,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994707099701229,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956171191052386,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959562181642181,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975539285101389,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979625771704473,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982596873060653,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988591750397494,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994812875397896,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.41.mlp": [
{
"accuracy": 0.9650331861094424,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.965844032011534,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718270929236161,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735582627748188,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826588724788866,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840083373220343,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.986458337620685,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911259138270428,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918963854250155,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911931196325704,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922678643151334,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995514349325707,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961435018401397,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976029205871256,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977058617300109,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981944578650751,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992528249460616,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.42.self_attn": [
{
"accuracy": 0.9857549149739114,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863843290429366,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885265866392537,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912976104962198,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927874042799598,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931400449652421,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958762463770414,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959834751329923,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962877266501126,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964357653730794,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962194781554373,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996470318813073,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971030144707153,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972805818052668,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983907198827517,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986302454612757,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988498659314293,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999254744009752,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996567560653937,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.42.mlp": [
{
"accuracy": 0.9663912685293901,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9671489094433031,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.972727076003426,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9743507990711614,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832746308100851,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845766189851259,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868924884419692,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991455647506212,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921913562636626,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915131526558023,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992542648001721,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956790428412589,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962808529013082,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976866364871201,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977962637418195,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982421531489021,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992741478214923,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.43.self_attn": [
{
"accuracy": 0.9840574499807859,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845482236460635,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.986923914990927,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902518924913908,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920861509285475,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920898205355594,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954149577962724,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955819472670555,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958357116893718,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960190704778621,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957350268959999,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960934711914313,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996643396976747,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968934990465641,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981161467730999,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984201113448331,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986913105178821,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999141926141946,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996170027328557,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.43.mlp": [
{
"accuracy": 0.9672750360087344,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9680193411676508,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732013062426919,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747228747919986,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836520129128506,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.984946420318202,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.98712247531665,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916653382150751,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923868634198841,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917081821905939,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927292811243158,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957781073294187,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963762101374174,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977476420371156,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978370711599526,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982408890990835,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992530855203146,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.44.self_attn": [
{
"accuracy": 0.9892324938585884,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896466441844639,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909621767307583,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931632343091463,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944048402340788,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945422363908667,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967537406635912,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969367757439613,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971164570827233,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972366992580263,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971395699601424,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973698146641254,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977672484360243,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979241890342612,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987606026820446,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989529926525919,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991412968619874,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994164182755508,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997417706457016,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.44.mlp": [
{
"accuracy": 0.967724012701135,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9684311935776159,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735015567980314,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749801974547537,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838539691347825,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851177601437819,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872465737556156,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917642976108351,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924725145101547,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918069392442703,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928017250801388,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958299504298913,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964112361010752,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977806790878898,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978700074318209,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982672351363459,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992801748020085,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.45.self_attn": [
{
"accuracy": 0.9896068808279539,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901080672678194,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914103189581319,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934548100358561,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947954255499338,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949156634117428,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970706734610232,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972262376625287,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973890634351655,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975131061908445,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997453107841705,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976282911865335,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979523043883475,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980924635341293,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988604024248687,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990410042044363,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992282179821479,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994637374521086,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997584074735641,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.45.mlp": [
{
"accuracy": 0.9678127138238204,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685046484595851,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9734483737694589,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.974905567733865,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838524554905138,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851146732505999,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872191218953383,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917693145965275,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924870807873575,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918181739355388,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928137691397416,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958388821074837,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964188596135691,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977862054580137,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978767266791118,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982698508783391,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992831068995752,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.46.self_attn": [
{
"accuracy": 0.987699609838034,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882726755581404,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895845055580139,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99222484623131,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939275591781265,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940199173594776,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964952982569996,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965744300892478,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967758818284461,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969329163432121,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968300293150701,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970526989353331,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973885926761126,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975791267658535,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985372386872768,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987656925069658,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999025482390272,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993163782024854,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996910867477325,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.46.mlp": [
{
"accuracy": 0.9678293999872709,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685309623417101,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9734219657747369,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9748527250791851,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838727273439106,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851311837372027,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872045234629983,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991792829413163,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925007349566409,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918265405454134,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928190049372221,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958462323013105,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996423280160678,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977921535702128,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978731486358141,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982549406980213,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992950298872433,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.47.self_attn": [
{
"accuracy": 0.9871744709579569,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987665120708315,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890033398803911,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920135054149126,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936292402838406,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937837966962865,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963782131671906,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965228469748246,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967406584244025,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996901641550817,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967662186214798,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969692877248714,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973012061887666,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974887231855016,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985217435383483,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987408311939553,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990550165897921,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993000462357151,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997168310525778,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.47.mlp": [
{
"accuracy": 0.9676313933573271,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9683283441945126,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732016858301664,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9746281990879461,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837394733177988,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850176839452041,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870893955230713,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917224378962266,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924373077718835,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917558870817486,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927609555031124,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958069046076975,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963903685933665,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977649069930378,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978400102179301,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998220610187242,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992457262093299,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.48.self_attn": [
{
"accuracy": 0.9893137216567993,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900703124309841,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991379945685989,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935702603114279,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948807362663118,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950287036205593,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968851518474127,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970952978259638,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972799469373728,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974085563107541,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974518373216453,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976078932614703,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979128108212822,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980558398130693,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988414954982305,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990241878518933,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991815285757184,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994256064099701,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997236243890304,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.48.mlp": [
{
"accuracy": 0.9676272304434526,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9683261137259633,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9731240209780241,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.974530251402604,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837289389811064,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850057379195565,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870552255919105,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917154876809371,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924353965018925,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917575999310142,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927577486163691,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958073983066961,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963876267797068,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977632904131162,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978614506360731,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982392497752842,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992852566861793,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.49.self_attn": [
{
"accuracy": 0.9901050062556016,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906202640972639,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918317355607685,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937691872841433,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995212160050869,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953464571582643,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997124146866171,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973173372839627,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997482596651504,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976101019664815,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976221773969499,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977811620031533,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980405092631516,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981815015014849,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989069789451989,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990839859176623,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992439669900035,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994719027002391,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997511863414394,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.49.mlp": [
{
"accuracy": 0.967418112252888,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.968117052002957,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729039323957342,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9743082335120753,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836293568736628,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849091846691934,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869521699453655,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916727237011257,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923901691248542,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917063924827074,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927136301994324,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957868147053217,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963697210738534,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977589794679692,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978427765400786,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982179913081621,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992421828210354,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.50.self_attn": [
{
"accuracy": 0.9894277727917621,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899551601786363,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99111681235464,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935790645448785,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948996776028683,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949973143245044,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970120852322955,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971464605707872,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973132780900127,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974498848773932,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974104515031764,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975671672115201,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978245510474631,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979797435042105,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987952463132771,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989804178476334,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991974140468397,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99942581237931,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997404255463105,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.50.mlp": [
{
"accuracy": 0.9670769980079249,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9677955570973849,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.972612086095308,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9740159731162222,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834503751052054,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847539161380968,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.98680180311203,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991578976574697,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923094009098253,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916042536497116,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926334235228991,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957307557526388,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963286111229345,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977308109794792,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997809751449447,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981786879269701,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992329288077982,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.51.self_attn": [
{
"accuracy": 0.990923534882696,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913867915931501,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924545037119012,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944805851892421,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957350790500641,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958552871095506,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975148355098147,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976825314132791,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978298250781862,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979358556631365,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99790845438838,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998036234786636,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982275243260359,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983575187231365,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990154561635695,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991700206147996,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993383408475079,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995322247084818,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997675202933973,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.51.mlp": [
{
"accuracy": 0.966716207956013,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9674434128560518,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9722871090236463,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737040353448767,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832763938527358,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845909586078242,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866518174347124,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914989126356024,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922299785049338,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991528612218405,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925594847453268,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956936553904885,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962947117654901,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997714800662116,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978011911244768,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981796878733133,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992649194931513,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.52.self_attn": [
{
"accuracy": 0.9914266059273168,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991769687909829,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928734224093588,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946331499438537,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959947823693878,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960957734208358,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977568052709103,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979270388813395,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980638678136625,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981565265671203,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998043483613353,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981618305962336,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983687896869684,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984847796394637,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990853752548757,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992357987518373,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993987729968993,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995935906312967,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997841836884618,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.52.mlp": [
{
"accuracy": 0.9660127006079021,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9667601083454332,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716455873690153,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9730734762392546,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829082457642806,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.984269347630049,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863489034928774,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913110693818644,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920608322871359,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913372303310194,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923981843810332,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955918036009136,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962134506357344,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976554699242115,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997744803758044,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981215821677133,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992272911691352,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.53.self_attn": [
{
"accuracy": 0.9893312971842917,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898229280584737,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990987471059749,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932858732185865,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947505585457149,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948306099364632,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968776240160591,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970486646420077,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972018072087514,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973421828135064,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973059049562404,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974713901940145,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997746785220347,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979129808121606,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987407011028967,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989383199105137,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991414842048758,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994080967122787,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997208910553079,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.53.mlp": [
{
"accuracy": 0.9656882913489091,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9664504747641713,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9713773978383917,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.972820056112189,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827431173701036,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841098157983077,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862038367672971,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912246481368416,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991978548859295,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912605866005546,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923209140175268,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955546914746887,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961707846898782,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976291232987454,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977292643958017,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981112803675627,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992298585313716,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.54.self_attn": [
{
"accuracy": 0.9891174466986405,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896556327217504,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990935418950884,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932385377193752,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947102163967333,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994844698984372,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968536421656609,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970095775237209,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971902666515425,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973279712231535,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973442432911772,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975118452781125,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997804772696997,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979683945053502,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987888018551626,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989717980160525,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991664944314643,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994113062459388,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997317135873202,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.54.mlp": [
{
"accuracy": 0.9652949822576422,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9660537587968927,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9710261978601155,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9724846545018648,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825388146074194,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.983917833943116,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860336545266604,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911125819934042,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918867239826604,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911554079306754,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922359719088203,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955042561418131,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961346125132159,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976097661021509,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977033710793445,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980899209254667,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992384074844027,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.55.self_attn": [
{
"accuracy": 0.9920275376031273,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923852744855379,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934812697925066,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949613864484587,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962023822100539,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996296010127193,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977670874642698,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979195434011912,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980454993875403,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981366588096869,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981294995860049,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982447230110043,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984495286878786,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985656055964922,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991237048647905,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992648215572301,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993901117949894,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995922622712035,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999782022431885,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.55.mlp": [
{
"accuracy": 0.9645950951074299,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9653862808880053,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9704143875523618,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718914251578482,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.982191663039358,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835963060981349,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857372719990579,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909361930269944,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917245264116087,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909796675569132,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920838279159445,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954206320016008,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960566061107736,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975628954799551,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976557838289362,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980420209467411,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991929808533505,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.56.self_attn": [
{
"accuracy": 0.9866432610311007,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873686793603396,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888909851249895,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920755350276044,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935152173826569,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936291175453287,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961986035892838,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963550759773505,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996598267633664,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967676254087373,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967096918507626,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969185845632302,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972658902406693,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974592241801714,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998499128085218,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987205848293869,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998996836574454,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992863729988274,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999685130645766,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.56.mlp": [
{
"accuracy": 0.9637815544479772,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9645899534225464,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9697078999720121,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9712233104203877,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817829539901332,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832131815584082,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853995097310919,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907246781022925,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915219553207096,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907743750434173,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918929683534723,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953149000280782,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959624511630911,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975096579445036,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976066905809077,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980060360149333,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991851482834471,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.57.self_attn": [
{
"accuracy": 0.9847988251008486,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855260158839979,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871953576803207,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909848915903192,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925021411556947,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926312597174394,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956344146477548,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957709877114547,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960303137961187,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962394206147445,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961732084813871,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963854621899756,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967582561075687,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970072325515119,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981870172839415,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984652567654848,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987814655821574,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991264608933738,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995925035327673,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.57.mlp": [
{
"accuracy": 0.9634681814595273,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9642937685313978,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.969466065105639,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9709909401441875,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816219132197531,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830635252751803,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852635327138399,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906378560944608,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991438101781042,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906861444837168,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918130479360882,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995259619072864,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959180272723499,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974722564220428,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975763861286012,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979758144993531,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991478839595067,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.58.self_attn": [
{
"accuracy": 0.98940231062864,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901714787671441,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915658282606226,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936025938705394,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949544691725781,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950866322768362,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968667508740174,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997017529057829,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972001410236484,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973348532068101,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974738724137607,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976284437273678,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979586577729175,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981070058910471,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998868246415728,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990447593950912,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991658112328303,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994176699730911,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997154485042158,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.58.mlp": [
{
"accuracy": 0.9628856056614926,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9637210620077032,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9689320451334903,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9704636492227253,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813062281985032,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827930848849448,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850063527885237,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904825358014357,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913030666740317,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905215768437636,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916826729711733,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951811679884007,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958540522738507,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974293712722627,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975390218590435,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979416502541617,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991784269774431,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.59.self_attn": [
{
"accuracy": 0.9872493179220903,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882695478828329,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899725600292808,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992630638574299,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938936527622374,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940608371245233,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963016796268915,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964599240767328,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966938283882643,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968493643559908,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968782918233621,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971016957179496,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974814220086524,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976829511946753,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986069594558916,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988204476080442,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990076056044352,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993203843600655,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996754156278544,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.59.mlp": [
{
"accuracy": 0.9621143905740035,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.96297582199699,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9682576907308478,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698117218519512,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9809240024340781,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9824295357653969,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846750686043187,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990277656599095,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911107146426251,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903235694295481,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991505210336886,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950819960550258,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957669667507473,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973778618793738,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997482099423283,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978905527999526,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991469115607048,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.60.self_attn": [
{
"accuracy": 0.9874467159572401,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879701882600784,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891967436200694,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991797462105751,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938382522055977,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939602644819963,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964651096808282,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966049719797937,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968068340891286,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969631447211692,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996897950376335,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970853485559162,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973766517482305,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975651735930067,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985552497796322,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998773114167546,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990536634271082,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993280106665272,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997000362184879,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.60.mlp": [
{
"accuracy": 0.961676719941591,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9625543136345713,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9678703358298854,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9694259543167917,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.980701838669024,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822382832828321,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844965809269955,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901823283810365,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910216739303187,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902134678865734,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914155069150423,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995025915142737,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957221095499239,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973511239023585,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974485931819991,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978519889869188,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991260098391458,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.61.self_attn": [
{
"accuracy": 0.987026331456084,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878255960188413,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894854834205226,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922242972411608,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935849826586874,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993717571622447,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963220895120972,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964280900986571,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966814957166973,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968254228955821,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967519323292532,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969515925959537,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974096421348421,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975968501285503,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985756258431234,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998783506845173,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990193708554694,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993383619248083,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996785721741617,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.61.mlp": [
{
"accuracy": 0.9612649428217035,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9621469692180031,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9675153211543435,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690908130846525,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.980497774324919,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820411675854733,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843152996740843,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900693556195811,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909174207009768,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901158174401835,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913188807274166,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949749457208734,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956757739970559,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973224401473999,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997431969368144,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978460602854428,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991328599617669,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.62.self_attn": [
{
"accuracy": 0.9866813327136793,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872585919342542,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888769050962046,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917348631118473,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934384132686415,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935655123309085,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996251368993207,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964018958179575,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966089380414862,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967740274181491,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966988708627852,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996899086589876,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973036062560583,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974974397765962,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985188310475726,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987370213003535,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990105974046808,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993062765386543,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999672528625907,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.62.mlp": [
{
"accuracy": 0.9604506963177731,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9613657154534992,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668189757748654,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.968417901741831,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800852427357122,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816673805839137,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839808282099272,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898631290385598,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907280183152148,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899048734652368,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911426431254337,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948674244316,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955842145963719,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972646042312446,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973659458521166,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977860678183404,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990795189024586,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.63.self_attn": [
{
"accuracy": 0.9864221638754794,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868711091970143,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884171376102849,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911627840054663,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930634529967057,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932284002241335,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959898998862818,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961691730116543,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963861860727009,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965390672809199,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964616012416387,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966472650044843,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971337553701902,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973419091027034,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984340991236662,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986751367779154,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989347925507709,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992860881611705,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996517364818015,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.63.mlp": [
{
"accuracy": 0.9597777065477873,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9607193062179967,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9662285949054518,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.967837067026841,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9797310860533464,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813558271056727,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836877866795188,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896885674250754,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905699067994168,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897284695976659,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909873973382147,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947795664009295,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955112600797101,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972180918251213,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973228660069013,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997739739912121,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990369833417629,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.64.self_attn": [
{
"accuracy": 0.9823271020462638,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837614893913269,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867932420027884,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895921318154586,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915389959749422,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917205517229281,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950051041025865,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953506871273643,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955994251527285,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957652409609995,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956993037148526,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99598055528967,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966882663337808,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969129546692497,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998151942695442,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984306343095867,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998658148473815,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991665155087647,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995963130832503,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.64.mlp": [
{
"accuracy": 0.9594787704317194,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9604310330591703,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9659313183081777,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9675433855307729,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795911892464286,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812143994005103,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.983539951475043,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896054260040584,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904930560212386,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896531183468668,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909175838294783,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994741744116733,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954737664053315,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971988116832156,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973166051663851,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977373263161433,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990747442567035,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.65.self_attn": [
{
"accuracy": 0.9872334834776426,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880820599041487,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896373474284222,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922101983898565,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993707496084665,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938621822940675,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962490895076802,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996421029693202,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966586577264887,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968211007745642,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968199404446703,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970277953696879,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974362001215157,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976260509145888,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985697910581764,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988005676944005,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989959747579537,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993131185244573,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996406749430063,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.65.mlp": [
{
"accuracy": 0.9588666401411358,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9598375903932672,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9653789871617368,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9670027996364393,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792921621548502,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9809249529713079,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832769616654045,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894564206662931,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990354124652712,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895088037377909,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907855901278948,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946748728030607,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954120465799382,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971655352335227,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972771492443586,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977051331416557,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990527634753993,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.66.self_attn": [
{
"accuracy": 0.988014900370648,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887436400902899,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903376165189242,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927116640304264,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940301238706237,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941722601652145,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964077531507141,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965648109975614,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968031209550405,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99695753815927,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969447272780695,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971350043227798,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975587409970007,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977556010218043,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986430107566872,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988628193540009,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990249712412295,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999362649298028,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996652900192299,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.66.mlp": [
{
"accuracy": 0.958159559651425,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9591496837766547,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9647219024206463,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9663499185913488,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788922165569506,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805844488896822,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829505948643935,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892523531851015,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901716230731261,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892766263924146,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.990617067406052,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945476976664442,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953227839187572,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970917917395893,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972110024016154,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976322178385759,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990259469731858,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.67.self_attn": [
{
"accuracy": 0.9907362351292058,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991322406812718,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925682019246252,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944121614098549,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955546816712931,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957067495898196,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973585138979711,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975717038308319,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977423948676962,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978422099038174,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978349871541324,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979736789276725,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982254656129762,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983579678166854,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990082977241591,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991659895565949,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993059745940723,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995224516544687,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997464075047326,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.67.mlp": [
{
"accuracy": 0.9573406890818947,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9583669116622523,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9640340710941114,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9656871839573509,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785002485701912,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9802094854806599,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826042385477769,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890485060842413,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899763684523734,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890981167554855,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904330255169618,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944620226558886,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952342318076837,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997052337778242,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971688872105197,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976012073457241,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990358817342081,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.68.self_attn": [
{
"accuracy": 0.9883625076005333,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887921182732833,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904162640634336,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929032404171793,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994079410245544,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942248722440318,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965715106380614,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967061422373119,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996906898327564,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970326647162437,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969277074070353,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971227800767672,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976063805975413,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977732622309735,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986840408099326,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988750100724006,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990772220649218,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993963089624518,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996658192555371,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.68.mlp": [
{
"accuracy": 0.9559552481299952,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9570106111074749,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.96281267467298,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9645061273323863,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777920904912447,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795600081744947,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820188785854139,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886870635183234,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896463720422042,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887300219975019,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901151939442283,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942654696734328,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950664651237036,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996934720560124,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970576500422076,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974894178541083,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989140250376964,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.69.self_attn": [
{
"accuracy": 0.9856752298380199,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857848537595648,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896874459166276,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916078079687921,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925067832595423,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929211257319701,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958774314114922,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960923139986239,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963517683117014,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964513076763404,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962892642146662,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965017586946487,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997407812233034,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976707173413352,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985733977273891,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987975029568923,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988682715124205,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993866497258607,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996454113310105,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.69.mlp": [
{
"accuracy": 0.9546166281951101,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9557156311838251,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9616691909338299,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9634162156205428,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771350386895632,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9789336684503054,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814572177435222,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883550389816886,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893322634069543,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884082102461865,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.989819085127429,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941137166399705,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949268196758471,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996864339042651,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969816800010832,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974261469354755,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989293842135292,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.70.self_attn": [
{
"accuracy": 0.988490544651684,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889976209715793,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909045986439052,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934620543530113,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943270667603141,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944674156998333,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966997453256657,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968671994773965,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970899473287558,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972201732820586,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972123762494639,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973896002691043,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977821211673712,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979508695633788,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987691674185427,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989632166137821,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991206931636522,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994001987047101,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996971363495839,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.70.mlp": [
{
"accuracy": 0.9530627319687291,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9542241535688701,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9603106567734166,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9621065102125469,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763384194750535,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9782070188145888,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807926526195124,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987936446541234,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889520767487978,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879998478450274,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894615027465319,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939001692753089,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994747026970512,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967404086338846,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968596298835779,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973207565122529,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988543734346566,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.71.self_attn": [
{
"accuracy": 0.9758692148484682,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773481488227844,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.984094312316493,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871710429066106,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880814434666383,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886704651932967,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933441438172993,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934988802200869,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940412260199848,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941885792895367,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941268782866629,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942038396471425,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959211416150394,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962757186670053,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977596057088751,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998084225544804,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998198674893693,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990585390478373,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994659862039905,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.71.mlp": [
{
"accuracy": 0.951265457429384,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9524764042151601,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9587428507051969,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9606105779346666,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9754288290676317,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773710479861811,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800407808077963,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874626319659384,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885110776675375,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875500115909075,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890613155929666,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936708027594968,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945474345433084,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966168897716623,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967539518287307,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972226855001951,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987756113864874,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.72.self_attn": [
{
"accuracy": 0.9814798502545607,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825086326975572,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865029360118666,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893006337316412,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909715942646328,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991200820395821,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947221675201466,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951011926720017,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954814248179135,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956537089065501,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955310076475143,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957916928749335,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966672447166944,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969049007876923,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981400958801571,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984366673191911,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986219848261068,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991885482970821,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995421187363958,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.72.mlp": [
{
"accuracy": 0.9496994834197194,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9509754118166471,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9573853172753987,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9593019642327961,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9746707724897485,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766599143806257,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793859359465147,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987065096434794,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881386913751301,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871636142856196,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887171842549977,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934761304604379,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943747798863211,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965093892655874,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966510091173021,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971306518112358,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987810290369549,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.73.self_attn": [
{
"accuracy": 0.9826242829623976,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836739552648444,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860303699970245,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899400684394335,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915261911718469,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917065077706387,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950776900115766,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952952014772516,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955909377650211,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957938774635917,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958097981779199,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960694187565854,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965761249002657,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968215089879537,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981123485455388,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998397497656314,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987047650900326,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990882277488708,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995898704632724,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.73.mlp": [
{
"accuracy": 0.9480381168817219,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9493688439068041,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9559022752862227,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.957898381509279,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738564114821585,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758981874114588,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787005640958485,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866122634787309,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877178151356546,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867474546557978,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883455469420082,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932625779980108,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941896204101411,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963798162184263,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965425894448632,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997040587428369,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998752669774388,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.74.self_attn": [
{
"accuracy": 0.9730241518271597,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9746017832505075,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818353527470639,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849174697148172,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866534110746885,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870677511943015,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923835067372573,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929314263557133,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935927159692112,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937491914943645,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931494028944718,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937916111789251,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954985615454222,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959169881124246,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975765233761386,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979593728325868,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980673025313177,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989944676819601,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999419806770196,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.74.mlp": [
{
"accuracy": 0.9461296796798706,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9475297049472207,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9542829708049172,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9563724680950767,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729432432275069,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750359623055709,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9779417514801025,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861242629979786,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872685145390662,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862834199478752,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.987929147325064,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930313002122076,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939798736258557,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996252104640007,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964462404972628,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969727353830087,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987788221945888,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.75.self_attn": [
{
"accuracy": 0.9752051359728763,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771613855110971,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794702075029674,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843198735462991,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865395222839556,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867094363036909,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912564527047308,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916447259877857,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924277145611612,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992619127035141,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924372701268447,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929994634891811,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944429762269321,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948640630433434,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996951383979697,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974112771451473,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976421147584915,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987299993242088,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993261883133336,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.75.mlp": [
{
"accuracy": 0.9437795312781083,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9452712535858154,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9523003289574071,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9545384676832902,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718023193509955,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9739948291527597,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9770350095472837,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854822598005596,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986687205339733,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857121078591597,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874323943727895,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927436607448679,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937359674980766,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960877758107687,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963034089458617,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968577878255593,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986929292545507,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.76.self_attn": [
{
"accuracy": 0.9654860967083981,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9663970627282795,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9724674820899963,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.977067348204161,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9824414708112416,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833954431508717,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896730866871382,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904189290184724,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914464762336329,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918699523336009,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916485242153469,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924563091052206,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935898659260649,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940245437778925,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965002415211577,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970192332801066,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974588212606154,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984507699938197,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991809744878035,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.76.mlp": [
{
"accuracy": 0.9439941017251265,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9454274867710314,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9526413083076477,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.954931030147954,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9717670334012884,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9739654299459959,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771034686188949,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854684393656882,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866930999253926,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856823431818109,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.987409676376142,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927306935975426,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937296670518423,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960894176834508,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962912041105723,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968606525737989,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987263420694753,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.77.self_attn": [
{
"accuracy": 0.9659542692335028,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668795999727751,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9725420380893507,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783258218514291,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829787204140111,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830291914312463,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903075365643752,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906987320435675,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913504578565296,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917523688391635,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914033067853827,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921737274056986,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934143000527432,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938217911281084,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963465050647133,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969245595367331,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974394779848424,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984044244414881,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992660811955207,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.77.mlp": [
{
"accuracy": 0.9429416656494141,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9444261914805362,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9517634793331748,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.954082915657445,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9712519363353127,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735031065187956,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766730556362554,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852005274672258,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864355843318137,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854100048542023,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871747823137986,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925843743901503,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936107824507513,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960089094544712,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962155089566582,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967928207234332,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987282521630588,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.78.self_attn": [
{
"accuracy": 0.9692305765653911,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9705981518092909,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742380819822613,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812375510993757,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843660373436777,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847575128078461,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899751818493793,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904707160435224,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910761149306047,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914328424554122,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922165329519071,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927366822957993,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993286268491494,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993784982122873,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963438216792909,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969196882294981,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997411921620369,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983901754021645,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992670335953957,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.78.mlp": [
{
"accuracy": 0.941464624906841,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.942998114385103,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9506259654697619,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9530412648853502,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9705306887626648,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9728262581323323,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.976107269525528,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848115287329021,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860864234598059,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850543941322126,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868554292540801,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924037966289019,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934501526387114,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959045923069904,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961188427711788,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.99672348169904,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986985230720357,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.79.self_attn": [
{
"accuracy": 0.9650834359620747,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9664635532780698,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698836332873294,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9767262527817174,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817179284597698,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820772472180819,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896565213015205,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900090655213908,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907050917023107,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911527045463261,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908277823736793,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914819527613489,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926899099036267,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932696301686136,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960766822883957,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996642181747838,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997344502100819,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982449988785543,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992513284087181,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.79.mlp": [
{
"accuracy": 0.9399976667604948,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9415651685313174,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9494047227658724,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9519204842416864,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9696960919781735,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9721032632024664,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9754828340128848,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843105234597859,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856289672224146,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845570482705769,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864429925617418,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921176104169143,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932282002348649,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995724959985206,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959640648019942,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965734223001882,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986711713044267,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.80.self_attn": [
{
"accuracy": 0.9702632427215576,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9712134756539997,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751233740856773,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804401021254691,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848425670673973,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847339234854046,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907816279875604,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911644709737677,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918827594895112,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921967167603342,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920290630114706,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926205755848634,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993792863660737,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943325213695827,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966656244114825,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971555789050303,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975482911655777,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985783804992312,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992567903215164,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.80.mlp": [
{
"accuracy": 0.9376562645560816,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9392837097770289,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9475488505865398,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.95018091954683,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685348366436205,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9710288643836975,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9745898121281674,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837450479206286,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851201804060685,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839887556276823,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859468748694972,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918464367326937,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992989287564629,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955993828020597,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958322903043345,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964744840797625,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986636484728048,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.81.self_attn": [
{
"accuracy": 0.9661412615525096,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9673445789437545,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9712291108934503,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784512017902575,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828720986843109,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831980341359189,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892840118784654,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896770417690277,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903717542949476,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908999283062784,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914278191955466,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919766485691071,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925922759269413,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931321348014631,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995880201458931,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965214348937336,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970628956430837,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982235733615724,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999167283408736,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.81.mlp": [
{
"accuracy": 0.9354908466339111,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9371878975316098,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9457255601882935,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9484717877287614,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.967417974221079,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.970015171327089,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737100005149841,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830802459465829,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845314402329294,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833851723294509,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854176797364887,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915178206406141,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927156700899726,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953997264567175,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956497295122397,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963129139260242,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985541629752046,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.82.self_attn": [
{
"accuracy": 0.9638578232965971,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.965449006933915,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9695192606825578,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9769923090934753,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9815195485165245,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819116984543047,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878967772973212,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883037068341908,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895559580702531,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900516036309694,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908173162686197,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915531246285689,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920195529335424,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927301006881815,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956651247645679,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963837360080919,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968600149609541,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981357323888101,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991226238443663,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.82.mlp": [
{
"accuracy": 0.9327557965328819,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.934528576700311,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9434900283813477,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9464135985625417,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9660937158684981,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9687820265167638,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9726797216816953,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823837578296661,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839062596622267,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827316732783067,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848440650262331,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911938019488987,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924381296885642,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952333549919882,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.995507148535628,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962276220321655,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985830499545524,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.83.self_attn": [
{
"accuracy": 0.9599329358653018,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9615296376378912,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9663974485899273,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9744200784909097,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804966198770624,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.98081363659156,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877438066821349,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880474986214387,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887726612781224,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893828504963925,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902620158697429,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909474826172778,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915174881094381,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921661246764032,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995370195100182,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996086800961118,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996770386829188,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997970823002489,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991145779035593,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.83.mlp": [
{
"accuracy": 0.9288912258650127,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9307171043596769,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9402115658709878,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9433364679938868,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9641712684380381,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9669911108518902,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9711510131233617,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813176095485687,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829050459359822,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817257837245339,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839118860269848,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906691344160783,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919700614715877,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949342600609127,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952351290144419,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960213374150427,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985305878676867,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.84.self_attn": [
{
"accuracy": 0.9561178903830678,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9588352755496377,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9634908437728882,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716214098428425,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9770793177579579,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777838619131791,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876686067957627,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879464036539981,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884983465859765,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890808439568469,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886872588019622,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989295872418504,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990997955987328,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919409634251344,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953096293305096,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959771750788939,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967735815597208,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979327517120462,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990839035691399,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.84.mlp": [
{
"accuracy": 0.9258293666337666,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9277360439300537,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9378319915972257,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9412708972629747,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9623770462839227,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9653674144493907,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698820427844399,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9802234439473403,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819494454484237,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807519567640204,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.983058142034631,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901328110381177,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991514979224456,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945984957249541,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949378151642648,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958108973346258,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983885437250137,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.85.self_attn": [
{
"accuracy": 0.9459732959144994,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9480861519512377,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9554729900862041,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9671305700352317,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9733356206040633,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735814772154155,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843784633435702,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845264287371385,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859977835103085,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867327793648368,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865996021973459,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876178503036499,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887961431553489,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989815468066617,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939560164746485,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949286599692545,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958816358917638,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974587732239774,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988876893920334,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.85.mlp": [
{
"accuracy": 0.9135036154797203,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.915624135418942,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9277268585405851,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9319266331823248,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9569625823121322,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9603468305186221,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9658699976770502,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9772875857980627,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794401542136544,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9780296604884299,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807498988352323,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887609873947344,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903679129324461,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938283656772814,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942778274416924,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954242627871664,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998242273730667,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.86.self_attn": [
{
"accuracy": 0.9465035011893824,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9483125523517006,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9584646538684243,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9694596779973883,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.973570914644944,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9741022775047704,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853000452643946,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858534540000715,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.986769996191326,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873491180570502,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869297970282404,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.987793925561403,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898842282985386,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905860314243718,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944769681284302,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953136761721811,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996223483038576,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976455122232437,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989374804457551,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.86.mlp": [
{
"accuracy": 0.9148707389831543,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.916399453815661,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9282098757593256,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9331886705599333,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9600381788454557,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.963532290960613,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9697161505096837,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.978235832954708,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807768432717574,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798957015338697,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825174526164406,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896064181076852,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911442116687172,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940198016794104,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994591058476975,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959712777482835,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978346512898019,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.87.self_attn": [
{
"accuracy": 0.9811902924587852,
"total_bits": 693407744,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820571447673597,
"total_bits": 712282112,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985370358354167,
"total_bits": 722632192,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897672949652923,
"total_bits": 868903936,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908955097198486,
"total_bits": 1027360768,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910422477282976,
"total_bits": 1027758592,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951003546777525,
"total_bits": 1320437760,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952200086493241,
"total_bits": 1320835584,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955521343570006,
"total_bits": 1331475456,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957466525466818,
"total_bits": 1351163904,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954753792599628,
"total_bits": 1354914304,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957879769958948,
"total_bits": 1365521408,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996479947708155,
"total_bits": 1376030720,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967649128091963,
"total_bits": 1395695616,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980519408066022,
"total_bits": 1692944384,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983583532861973,
"total_bits": 1722851328,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986981334850976,
"total_bits": 1974749184,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991498387565738,
"total_bits": 2030641152,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995917912425571,
"total_bits": 2629060608,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.87.mlp": [
{
"accuracy": 0.9533006015576815,
"total_bits": 2361122896,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9550086699034038,
"total_bits": 2449203280,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9595317464125783,
"total_bits": 2734616576,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9618251511925145,
"total_bits": 3069243392,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776781562127566,
"total_bits": 3456750688,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798966536396428,
"total_bits": 3552524288,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822953672785508,
"total_bits": 3820372064,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876296716301065,
"total_bits": 4368108128,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889696491392035,
"total_bits": 4432541696,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886443669858732,
"total_bits": 4496020576,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901004637542524,
"total_bits": 4591794176,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940730274507874,
"total_bits": 5535290464,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949287285930232,
"total_bits": 5631064064,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964425512834599,
"total_bits": 6413603936,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969153612067825,
"total_bits": 6630297088,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974609621261296,
"total_bits": 7227985408,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986578015316474,
"total_bits": 8490470912,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.norm.norm": null,
"lm_head.linear": null
},
"last_module_idx": 178
}