diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json"
new file mode 100644--- /dev/null
+++ "b/reduced_ppl_average.json"
@@ -0,0 +1,13520 @@
+{
+    "original_ppl": 5.709594249725342,
+    "reduced_ppl": [
+        {
+            "bit": 1,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10567.6279296875
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10189.767578125
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10145.455078125
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10058.2685546875
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10255.1142578125
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10247.263671875
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10976.8837890625
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10739.5322265625
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10739.3369140625
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 8588.806640625
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 9590.7763671875
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 9879.37890625
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 9242.65234375
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10457.009765625
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10544.896484375
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10610.4453125
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 9306.521484375
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 8802.19140625
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 9998.1513671875
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10256.3759765625
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10959.85546875
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10516.7548828125
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 11356.791015625
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 11073.08984375
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 11204.8544921875
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 12153.826171875
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 9231.9580078125
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10075.943359375
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10825.87890625
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 12785.171875
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10821.224609375
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 9910.1611328125
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10058.9404296875
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10436.1171875
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 11146.353515625
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 11659.0087890625
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 9418.8369140625
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10180.2294921875
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 9962.26953125
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 11479.43359375
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10776.0859375
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 11470.4169921875
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10557.3232421875
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10713.5888671875
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10879.771484375
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10029.302734375
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 8872.9033203125
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 9753.16015625
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10820.5224609375
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10781.5654296875
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 9849.642578125
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 9180.4384765625
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 12868.7216796875
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10868.7900390625
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10450.2705078125
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 8471.5751953125
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10817.5302734375
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10570.3388671875
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 13298.19140625
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 9529.5908203125
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10790.4736328125
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10234.9873046875
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 9251.2587890625
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 11260.580078125
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10129.4736328125
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 8761.8564453125
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 9022.6669921875
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 11680.9775390625
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10368.7578125
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 9557.1943359375
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10703.908203125
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 12151.669921875
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 12307.1337890625
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10009.294921875
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 11075.8564453125
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 11463.69140625
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10337.84375
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10680.353515625
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10197.953125
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10211.8994140625
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 8991.3408203125
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 9676.6142578125
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 12080.5859375
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10091.8974609375
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10554.6748046875
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10436.3271484375
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10313.8154296875
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 12580.716796875
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10480.9912109375
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10401.0927734375
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10003.1015625
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10767.0361328125
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10837.3564453125
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 11167.5810546875
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 8934.3203125
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 11187.5576171875
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 9931.0703125
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 11020.18359375
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10910.08984375
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 9984.7353515625
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 9652.2451171875
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 8942.28125
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10678.275390625
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 11443.134765625
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 11020.7197265625
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10632.4658203125
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10628.0458984375
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 9440.8701171875
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 12414.9384765625
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10940.244140625
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 11481.5576171875
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 11989.66015625
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10691.451171875
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10677.4609375
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10757.7783203125
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 12009.7216796875
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10884.337890625
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 11461.220703125
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10906.6787109375
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10778.2041015625
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 11037.0107421875
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 11151.72265625
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10253.0703125
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 11608.63671875
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10195.521484375
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 11130.271484375
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10519.5439453125
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10713.19140625
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10709.994140625
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10053.6171875
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10573.142578125
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 11842.8134765625
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 12637.943359375
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10699.1728515625
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10658.5576171875
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10849.703125
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 11081.3291015625
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 11147.703125
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 11805.89453125
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10489.5810546875
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 11061.279296875
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10922.4375
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10153.4599609375
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 9357.04296875
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 11064.43359375
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 11501.986328125
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 11086.8779296875
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10847.095703125
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10781.2158203125
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10823.01953125
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10984.1513671875
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10976.4453125
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 11326.1162109375
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 11063.5048828125
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10700.029296875
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10697.7236328125
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 11089.6279296875
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10051.4892578125
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10695.642578125
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 11200.6240234375
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10779.40625
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10751.193359375
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10875.310546875
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10518.2802734375
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10321.490234375
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10772.458984375
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 11081.3291015625
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10928.4697265625
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10713.181640625
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10939.4716796875
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10388.9990234375
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 11035.9169921875
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10970.7724609375
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 11015.2666015625
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 11003.580078125
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10400.6767578125
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 9993.7470703125
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10240.6015625
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10649.12890625
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10727.525390625
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10895.4716796875
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10772.5107421875
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10912.525390625
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 11053.232421875
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10848.7099609375
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10753.203125
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 11092.41015625
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 11366.5537109375
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 11235.0185546875
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10819.3564453125
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10522.3037109375
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10533.1962890625
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10387.294921875
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10760.91796875
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10947.3408203125
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10882.51171875
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 10507.8232421875
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 10348.2197265625
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 10611.326171875
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 11039.19140625
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 10514.5693359375
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 11334.728515625
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 10771.5341796875
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 10928.54296875
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 10939.095703125
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 10196.8447265625
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 10541.9189453125
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 9971.365234375
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 10726.451171875
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 10932.640625
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 10955.3828125
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 10858.2216796875
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 11159.27734375
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 10506.009765625
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 9995.9775390625
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 10862.572265625
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 11789.1767578125
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 10662.990234375
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 10548.818359375
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 9830.6953125
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 10317.2978515625
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 8232.400390625
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 10862.7685546875
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 11919.38671875
+                }
+            ]
+        },
+        {
+            "bit": 2,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 34.57195281982422
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 34.52710723876953
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 33.759483337402344
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 32.43354415893555
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 32.05517578125
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 32.288536071777344
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 31.484949111938477
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 34.28741455078125
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 34.2617301940918
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 33.248924255371094
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 33.3477783203125
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 33.089942932128906
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 33.65793228149414
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 33.47496032714844
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 33.28439712524414
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 34.1829948425293
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 34.67659378051758
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 34.420021057128906
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 33.05862045288086
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 33.653533935546875
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 33.16836166381836
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 33.8890266418457
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 34.63047409057617
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 33.226661682128906
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 33.40636444091797
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 33.148216247558594
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 33.19480895996094
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 33.060516357421875
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 34.38041305541992
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 34.89942169189453
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 34.22050094604492
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 35.17528533935547
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 33.39213562011719
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 33.79841232299805
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 33.23938751220703
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 33.895816802978516
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 34.27817153930664
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 33.54795455932617
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 33.53366470336914
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 34.059391021728516
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 33.78882598876953
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 33.07892990112305
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 34.50130081176758
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 34.48667907714844
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 34.063663482666016
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 33.386802673339844
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 33.016632080078125
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 33.38557434082031
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 32.862335205078125
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 35.20330047607422
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 34.48147201538086
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 34.12400817871094
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 33.47517776489258
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 34.02745056152344
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 34.83621597290039
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 34.40919876098633
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 34.31414794921875
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 34.19636917114258
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 34.39153289794922
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 34.5456657409668
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 34.0633430480957
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 34.094749450683594
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 33.85380172729492
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 34.03188705444336
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 34.119022369384766
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 34.74598693847656
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 34.483734130859375
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 34.58083724975586
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 34.41065979003906
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 34.2165641784668
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 33.957088470458984
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 34.00929260253906
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 34.12889862060547
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 33.797481536865234
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 34.214481353759766
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 34.31638717651367
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 34.608943939208984
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 34.54572296142578
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 34.51774597167969
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 34.53952407836914
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 33.08501434326172
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 34.0992546081543
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 34.07937240600586
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 34.25077819824219
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 34.28379440307617
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 34.26295471191406
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 33.22490310668945
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 33.58615493774414
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 33.81552505493164
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 33.98564147949219
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 33.63060760498047
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 33.890804290771484
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 33.82011413574219
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 33.99924087524414
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 34.403194427490234
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 34.02565002441406
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 34.25650405883789
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 33.94101333618164
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 34.540428161621094
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 34.29387283325195
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 34.01964569091797
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 34.42142105102539
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 33.88022232055664
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 34.27000045776367
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 34.40312576293945
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 34.50580978393555
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 34.66890335083008
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 34.27513122558594
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 34.37164306640625
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 34.541316986083984
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 34.50294494628906
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 34.36033630371094
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 34.56711959838867
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 34.362464904785156
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 34.36616516113281
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 34.23607635498047
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 33.9645881652832
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 34.4565315246582
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 34.30780792236328
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 34.27949142456055
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 34.582603454589844
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 34.44756317138672
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 34.38725280761719
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 34.13260269165039
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 34.467872619628906
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 34.088993072509766
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 34.485233306884766
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 34.57403564453125
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 34.682552337646484
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 34.74006271362305
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 34.2639045715332
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 34.40919876098633
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 34.312774658203125
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 34.317691802978516
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 34.483306884765625
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 34.51898956298828
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 34.307220458984375
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 34.29369354248047
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 34.066001892089844
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 34.13399124145508
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 34.56247329711914
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 34.55335998535156
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 34.67139434814453
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 34.52983856201172
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 34.03361892700195
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 34.50508499145508
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 34.3360481262207
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 34.427818298339844
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 34.45600128173828
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 34.440826416015625
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 34.39469528198242
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 34.30693435668945
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 34.37007141113281
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 34.36513900756836
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 34.605342864990234
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 34.379478454589844
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 34.61307144165039
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 34.54313278198242
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 34.373775482177734
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 34.57048416137695
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 34.41924285888672
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 34.54680252075195
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 34.607139587402344
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 34.717864990234375
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 34.61105728149414
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 34.29473114013672
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 34.4323616027832
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 34.353153228759766
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 34.35285186767578
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 34.476287841796875
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 34.416080474853516
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 34.39631271362305
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 34.278724670410156
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 34.69623565673828
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 34.41499710083008
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 34.44273376464844
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 34.565521240234375
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 34.46662902832031
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 34.48087692260742
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 34.32445526123047
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 34.53607177734375
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 34.318450927734375
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 34.54118728637695
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 34.53817367553711
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 34.60563278198242
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 34.45487976074219
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 34.445526123046875
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 34.558074951171875
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 34.528133392333984
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 34.438350677490234
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 34.52284240722656
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 34.60752868652344
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 34.66035842895508
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 34.51670837402344
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 34.5708122253418
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 34.3725700378418
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 34.559669494628906
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 34.59638977050781
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 34.5826416015625
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 34.5693473815918
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 34.57836151123047
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 34.65898513793945
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 34.550697326660156
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 34.395263671875
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 34.35078811645508
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 34.641231536865234
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 34.51400375366211
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 34.5910530090332
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 34.4173698425293
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 34.416351318359375
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 34.47446060180664
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 34.593849182128906
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 34.69132995605469
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 34.640846252441406
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 34.203651428222656
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 34.22004699707031
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 34.34844207763672
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 34.51666259765625
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 34.57122802734375
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 34.6813850402832
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 34.71495056152344
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 34.37432098388672
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 34.53300094604492
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 34.46146774291992
+                }
+            ]
+        },
+        {
+            "bit": 3,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 7.9461774826049805
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 7.947465419769287
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 7.901550769805908
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 7.8674397468566895
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 7.887763500213623
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 7.896790027618408
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 7.880622386932373
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 7.922502517700195
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 7.933195114135742
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 7.884892463684082
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 7.896417140960693
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 7.87729549407959
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 7.880502223968506
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 7.883674144744873
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 7.930445671081543
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 7.915066242218018
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 7.9209184646606445
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 7.9380011558532715
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 7.896726131439209
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 7.9177069664001465
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 7.91702938079834
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 7.929443359375
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 7.937050819396973
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 7.9357147216796875
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 7.929398059844971
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 7.89821720123291
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 7.9130635261535645
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 7.917545795440674
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 7.949504852294922
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 7.944932460784912
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 7.931194305419922
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 7.928679943084717
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 7.912296295166016
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 7.932130813598633
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 7.91456413269043
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 7.939064979553223
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 7.946615219116211
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 7.9263153076171875
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 7.929776191711426
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 7.914409637451172
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 7.9311299324035645
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 7.927009105682373
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 7.950355529785156
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 7.948017120361328
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 7.942692279815674
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 7.93778133392334
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 7.906951904296875
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 7.928264141082764
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 7.93275260925293
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 7.961323261260986
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 7.958410263061523
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 7.963601112365723
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 7.959017276763916
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 7.9284868240356445
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 7.931981563568115
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 7.926553726196289
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 7.94442081451416
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 7.946796894073486
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 7.941961288452148
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 7.941811561584473
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 7.931285381317139
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 7.934602737426758
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 7.92728853225708
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 7.944366455078125
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 7.946315765380859
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 7.94504451751709
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 7.941877841949463
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 7.930385112762451
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 7.936042785644531
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 7.9395270347595215
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 7.938392639160156
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 7.937050819396973
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 7.937075614929199
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 7.931523323059082
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 7.928436279296875
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 7.9382429122924805
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 7.944479942321777
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 7.94779109954834
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 7.952007293701172
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 7.952056407928467
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 7.950871467590332
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 7.936583518981934
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 7.922083854675293
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 7.942915439605713
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 7.947587013244629
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 7.953937530517578
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 7.934799671173096
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 7.936850547790527
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 7.948805332183838
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 7.9409918785095215
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 7.935338497161865
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 7.936914920806885
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 7.937769889831543
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 7.939852237701416
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 7.9268083572387695
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 7.934116363525391
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 7.936207294464111
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 7.9416375160217285
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 7.944608688354492
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 7.9514079093933105
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 7.958968162536621
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 7.953381538391113
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 7.936254501342773
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 7.939049243927002
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 7.937917709350586
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 7.944266319274902
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 7.945455551147461
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 7.9394755363464355
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 7.9380784034729
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 7.938943386077881
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 7.937800407409668
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 7.939210414886475
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 7.943374156951904
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 7.945173263549805
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 7.939042091369629
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 7.937414169311523
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 7.935083389282227
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 7.938426494598389
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 7.940467357635498
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 7.939859390258789
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 7.940512657165527
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 7.943423271179199
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 7.939942836761475
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 7.930112838745117
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 7.938530921936035
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 7.935147762298584
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 7.954329490661621
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 7.948697566986084
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 7.942699432373047
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 7.939664363861084
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 7.929938793182373
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 7.941745281219482
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 7.937289237976074
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 7.941506862640381
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 7.941203594207764
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 7.939117431640625
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 7.938708782196045
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 7.928543567657471
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 7.928940773010254
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 7.932537078857422
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 7.948521137237549
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 7.949269771575928
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 7.94331693649292
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 7.943905830383301
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 7.927674293518066
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 7.946404457092285
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 7.933562278747559
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 7.94740104675293
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 7.945073127746582
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 7.94174337387085
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 7.943120002746582
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 7.93034553527832
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 7.941709041595459
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 7.934141159057617
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 7.943271636962891
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 7.9431047439575195
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 7.945976257324219
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 7.948796272277832
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 7.929816246032715
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 7.936256408691406
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 7.933286190032959
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 7.945980072021484
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 7.947525978088379
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 7.948060512542725
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 7.948775291442871
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 7.935506820678711
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 7.940329074859619
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 7.9392619132995605
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 7.945156097412109
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 7.94595193862915
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 7.946173191070557
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 7.944000720977783
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 7.935363292694092
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 7.936956405639648
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 7.934545516967773
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 7.946089744567871
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 7.94644832611084
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 7.944385051727295
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 7.94339656829834
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 7.931257247924805
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 7.932302474975586
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 7.938589572906494
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 7.945711135864258
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 7.946021556854248
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 7.9461750984191895
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 7.943582057952881
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 7.9326887130737305
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 7.938901424407959
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 7.942438125610352
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 7.945036888122559
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 7.9467267990112305
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 7.949273586273193
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 7.950535774230957
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 7.937090873718262
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 7.936392307281494
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 7.940166473388672
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 7.9436445236206055
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 7.944836139678955
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 7.942584037780762
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 7.943691730499268
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 7.935976505279541
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 7.937545299530029
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 7.940749645233154
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 7.94005823135376
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 7.940617084503174
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 7.953882217407227
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 7.945059776306152
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 7.935088634490967
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 7.941315174102783
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 7.942779541015625
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 7.9452738761901855
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 7.948841571807861
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 7.94928503036499
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 7.945047855377197
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 7.935410499572754
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 7.936846733093262
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 7.940468788146973
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 7.947516918182373
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 7.94775390625
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 7.960308074951172
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 7.9513959884643555
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 7.927347660064697
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 7.922753810882568
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 7.93113374710083
+                }
+            ]
+        },
+        {
+            "bit": 4,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 6.454537868499756
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 6.454726696014404
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 6.450519561767578
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 6.4437255859375
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 6.448850631713867
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 6.450132369995117
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 6.446074962615967
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 6.449563980102539
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 6.4544291496276855
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 6.448617935180664
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 6.446349143981934
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 6.445285797119141
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 6.443925857543945
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 6.445038318634033
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 6.448399066925049
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 6.453738689422607
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 6.4486260414123535
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 6.451790809631348
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 6.446408271789551
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 6.450733661651611
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 6.448801517486572
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 6.452375411987305
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 6.452647686004639
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 6.451661586761475
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 6.448610782623291
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 6.447442054748535
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 6.447699069976807
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 6.448030948638916
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 6.455854415893555
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 6.454732894897461
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 6.452969074249268
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 6.4545440673828125
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 6.447070121765137
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 6.449777603149414
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 6.455633640289307
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 6.453912258148193
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 6.454641342163086
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 6.453270435333252
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 6.455489158630371
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 6.451198577880859
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 6.452322959899902
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 6.452975273132324
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 6.455671310424805
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 6.455975532531738
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 6.456965923309326
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 6.455340385437012
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 6.446424961090088
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 6.451291084289551
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 6.4530415534973145
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 6.455935478210449
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 6.456725120544434
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 6.452491760253906
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 6.452752113342285
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 6.449517726898193
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 6.452578544616699
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 6.451826095581055
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 6.45448637008667
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 6.454720973968506
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 6.455731391906738
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 6.455089092254639
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 6.452181816101074
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 6.4526824951171875
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 6.454453468322754
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 6.455934524536133
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 6.456002235412598
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 6.4546217918396
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 6.453610420227051
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 6.452699661254883
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 6.453961372375488
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 6.454385757446289
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 6.452418327331543
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 6.453369140625
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 6.4515509605407715
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 6.4532928466796875
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 6.452413558959961
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 6.4538068771362305
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 6.455025672912598
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 6.456299304962158
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 6.454927921295166
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 6.4534173011779785
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 6.454418182373047
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 6.451657295227051
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 6.451346397399902
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 6.454096794128418
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 6.454782009124756
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 6.45551872253418
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 6.4534735679626465
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 6.453575611114502
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 6.453646183013916
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 6.454469203948975
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 6.456052780151367
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 6.453341484069824
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 6.454270839691162
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 6.451605796813965
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 6.452873706817627
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 6.453426837921143
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 6.452221870422363
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 6.452780246734619
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 6.455355644226074
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 6.455962657928467
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 6.456732273101807
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 6.453287124633789
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 6.453049182891846
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 6.454832077026367
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 6.454221725463867
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 6.454924583435059
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 6.454874515533447
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 6.451925277709961
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 6.452725410461426
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 6.4522833824157715
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 6.45523738861084
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 6.45308256149292
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 6.454366683959961
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 6.455395698547363
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 6.452813148498535
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 6.454213619232178
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 6.451274871826172
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 6.454441547393799
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 6.4550957679748535
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 6.453302383422852
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 6.454470634460449
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 6.453659534454346
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 6.453427791595459
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 6.450741767883301
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 6.453146934509277
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 6.454835891723633
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 6.4564948081970215
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 6.455182075500488
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 6.454783916473389
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 6.455849647521973
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 6.451282501220703
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 6.455349922180176
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 6.454250812530518
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 6.454679012298584
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 6.454629421234131
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 6.452934741973877
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 6.4529619216918945
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 6.450913429260254
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 6.453946113586426
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 6.453378200531006
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 6.4553914070129395
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 6.455657958984375
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 6.454373836517334
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 6.454061985015869
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 6.450056552886963
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 6.453320503234863
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 6.453465938568115
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 6.454935073852539
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 6.455124855041504
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 6.45426607131958
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 6.454684734344482
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 6.450778484344482
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 6.454276084899902
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 6.452948093414307
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 6.455362319946289
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 6.455121994018555
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 6.45496940612793
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 6.454651355743408
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 6.450726509094238
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 6.45337438583374
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 6.452306747436523
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 6.454626083374023
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 6.455564498901367
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 6.454758644104004
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 6.454658031463623
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 6.452275276184082
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 6.452835559844971
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 6.45163631439209
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 6.454537868499756
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 6.455155372619629
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 6.454677104949951
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 6.455167293548584
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 6.4516472816467285
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 6.453344345092773
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 6.451472282409668
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 6.4543914794921875
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 6.454807758331299
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 6.4543938636779785
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 6.454660415649414
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 6.4515910148620605
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 6.453164100646973
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 6.452962875366211
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 6.455028533935547
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 6.45471715927124
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 6.454559803009033
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 6.454847812652588
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 6.451287746429443
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 6.453179836273193
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 6.453149795532227
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 6.453858852386475
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 6.454281330108643
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 6.455502033233643
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 6.454575538635254
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 6.451561450958252
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 6.452361583709717
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 6.4535813331604
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 6.454469203948975
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 6.454554557800293
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 6.452547073364258
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 6.454784870147705
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 6.451087951660156
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 6.453288555145264
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 6.452445983886719
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 6.453505992889404
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 6.454779624938965
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 6.456064701080322
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 6.454588890075684
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 6.452078342437744
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 6.453407287597656
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 6.452796936035156
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 6.455315113067627
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 6.4545488357543945
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 6.455374240875244
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 6.4543914794921875
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 6.450343608856201
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 6.45657205581665
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 6.453243732452393
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 6.455255031585693
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 6.4553728103637695
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 6.457914352416992
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 6.456103801727295
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 6.44964075088501
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 6.447453498840332
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 6.448609352111816
+                }
+            ]
+        },
+        {
+            "bit": 5,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 6.10332727432251
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 6.103443145751953
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 6.102327823638916
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 6.10093879699707
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 6.102288246154785
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 6.10227108001709
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 6.102169513702393
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 6.102914333343506
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 6.1028876304626465
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 6.1018571853637695
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 6.102959156036377
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 6.101169109344482
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 6.100903511047363
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 6.10158109664917
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 6.102158546447754
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 6.102851390838623
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 6.103203773498535
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 6.100645065307617
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 6.100838661193848
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 6.1026201248168945
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 6.1004252433776855
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 6.102511405944824
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 6.102550506591797
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 6.102677345275879
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 6.101796627044678
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 6.1003804206848145
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 6.102692127227783
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 6.1017165184021
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 6.103428840637207
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 6.102806091308594
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 6.10255241394043
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 6.102294445037842
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 6.101330280303955
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 6.101095199584961
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 6.102182388305664
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 6.10371732711792
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 6.103259086608887
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 6.10362434387207
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 6.102778911590576
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 6.103330135345459
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 6.102243423461914
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 6.102015018463135
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 6.103821754455566
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 6.103170394897461
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 6.102950096130371
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 6.1029534339904785
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 6.100493907928467
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 6.101402759552002
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 6.102095127105713
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 6.1033759117126465
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 6.104278087615967
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 6.10400390625
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 6.104101181030273
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 6.101878643035889
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 6.1019816398620605
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 6.10313081741333
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 6.1031084060668945
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 6.103360176086426
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 6.1029582023620605
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 6.103797435760498
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 6.1028923988342285
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 6.102817058563232
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 6.1025390625
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 6.103361129760742
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 6.103934288024902
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 6.1032819747924805
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 6.10377836227417
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 6.103052139282227
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 6.103724479675293
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 6.1033244132995605
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 6.103392601013184
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 6.1027607917785645
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 6.10310173034668
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 6.102715015411377
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 6.1021318435668945
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 6.102659225463867
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 6.102339267730713
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 6.103653907775879
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 6.103553295135498
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 6.103222846984863
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 6.102321147918701
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 6.1020355224609375
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 6.102360248565674
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 6.103721618652344
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 6.10411262512207
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 6.103764057159424
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 6.102407932281494
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 6.103240013122559
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 6.102255344390869
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 6.102858543395996
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 6.102774620056152
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 6.102561950683594
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 6.102424621582031
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 6.10245943069458
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 6.102545738220215
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 6.102554798126221
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 6.103653430938721
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 6.103150844573975
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 6.103906631469727
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 6.103859901428223
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 6.102217674255371
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 6.103619575500488
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 6.102270126342773
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 6.1033854484558105
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 6.103147029876709
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 6.103157043457031
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 6.103323459625244
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 6.102534294128418
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 6.102122783660889
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 6.102392673492432
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 6.102656364440918
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 6.103926658630371
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 6.103625774383545
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 6.103555679321289
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 6.1027302742004395
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 6.102412223815918
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 6.101873874664307
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 6.103229999542236
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 6.102924346923828
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 6.1027679443359375
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 6.103058815002441
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 6.103218078613281
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 6.1030073165893555
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 6.101222038269043
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 6.102621555328369
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 6.102445602416992
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 6.103686809539795
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 6.103755474090576
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 6.103569984436035
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 6.1022820472717285
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 6.102065086364746
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 6.102996826171875
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 6.103424072265625
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 6.1032609939575195
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 6.103483200073242
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 6.102055072784424
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 6.102387428283691
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 6.101625442504883
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 6.102776527404785
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 6.102668285369873
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 6.103758811950684
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 6.103571891784668
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 6.102988243103027
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 6.102940559387207
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 6.101558685302734
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 6.103109359741211
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 6.102682113647461
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 6.103518009185791
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 6.103269100189209
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 6.103158950805664
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 6.102853298187256
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 6.101789951324463
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 6.103322982788086
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 6.102255344390869
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 6.103530406951904
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 6.103815078735352
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 6.103599548339844
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 6.103411674499512
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 6.101712703704834
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 6.102099418640137
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 6.102999687194824
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 6.10357141494751
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 6.103631496429443
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 6.103546619415283
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 6.103321552276611
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 6.101658821105957
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 6.102960109710693
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 6.1025776863098145
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 6.103583335876465
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 6.103630065917969
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 6.103322982788086
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 6.103185176849365
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 6.102198123931885
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 6.102322101593018
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 6.103001117706299
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 6.103229999542236
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 6.103567600250244
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 6.103597640991211
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 6.103576183319092
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 6.101297378540039
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 6.102327823638916
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 6.1029582023620605
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 6.103095054626465
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 6.103282928466797
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 6.103132724761963
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 6.102866172790527
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 6.102301597595215
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 6.101816177368164
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 6.1020965576171875
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 6.10333776473999
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 6.103382587432861
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 6.103857040405273
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 6.103193759918213
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 6.102137088775635
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 6.103390693664551
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 6.102590560913086
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 6.103017330169678
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 6.103185653686523
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 6.102924346923828
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 6.1027631759643555
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 6.1010870933532715
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 6.102463245391846
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 6.103203773498535
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 6.102989673614502
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 6.103216648101807
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 6.104397773742676
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 6.103231430053711
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 6.101535320281982
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 6.1034979820251465
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 6.1034040451049805
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 6.1031293869018555
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 6.10416316986084
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 6.103411674499512
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 6.103137969970703
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 6.101171016693115
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 6.100432872772217
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 6.101370811462402
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 6.102499961853027
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 6.103721618652344
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 6.105226516723633
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 6.103948593139648
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 6.100316047668457
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 6.098746299743652
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 6.1007561683654785
+                }
+            ]
+        },
+        {
+            "bit": 6,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.964634895324707
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.964853286743164
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.964779853820801
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.96457576751709
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.964901447296143
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.965076446533203
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.963896751403809
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.9644904136657715
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.964446067810059
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.964075088500977
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.9636616706848145
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.9634599685668945
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.963460922241211
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.963972568511963
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.963679313659668
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.964230060577393
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.963833332061768
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.9647674560546875
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.964493274688721
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.963866233825684
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.964780330657959
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.964466571807861
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.964788913726807
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.963519096374512
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.964328765869141
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.962802886962891
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.96372652053833
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.964566230773926
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.964580535888672
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.9645185470581055
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.964216232299805
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.964838981628418
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.963772296905518
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.965837001800537
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.964780330657959
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.964582443237305
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.964877128601074
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.963987827301025
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.96481466293335
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.963537693023682
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.964683532714844
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.964261054992676
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.964827537536621
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.964755535125732
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.964868545532227
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.964569091796875
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.963584899902344
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.964655876159668
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.9642839431762695
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.9651103019714355
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.964787483215332
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.964884281158447
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.965034008026123
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.963949680328369
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.964010238647461
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.963255882263184
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.964312553405762
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.964426517486572
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.96516227722168
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.963976860046387
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.963647365570068
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.964200019836426
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.964569568634033
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.965219497680664
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.964680194854736
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.964784145355225
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.964580535888672
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.9637675285339355
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.963949680328369
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.964078426361084
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.964617729187012
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.964517593383789
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.964454174041748
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.964346408843994
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.964151859283447
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.964664459228516
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.964797496795654
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.964831352233887
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.9650750160217285
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.96469783782959
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.964787483215332
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.964296340942383
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.96401834487915
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.964768886566162
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.965114593505859
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.965019226074219
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.965167045593262
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.964183330535889
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.964242935180664
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.964613914489746
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.965151309967041
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.964411735534668
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.964498519897461
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.9640607833862305
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.964939117431641
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.963999271392822
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.963999271392822
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.964320182800293
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.964476585388184
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.964928150177002
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.965571880340576
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.964280128479004
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.964073657989502
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.9643940925598145
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.964827537536621
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.964621067047119
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.964763641357422
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.964252948760986
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.964240074157715
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.964327335357666
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.964642524719238
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.964659214019775
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.964810848236084
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.9646830558776855
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.964807510375977
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.964704513549805
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.964130878448486
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.9653496742248535
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.9648542404174805
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.964399337768555
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.964700698852539
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.964797019958496
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.96461820602417
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.964078426361084
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.964522838592529
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.964656352996826
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.965064525604248
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.9648756980896
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.964382171630859
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.964518070220947
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.963754653930664
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.965043067932129
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.9638519287109375
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.964806079864502
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.964855670928955
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.964405059814453
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.964571475982666
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.963390827178955
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.964374542236328
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.964343070983887
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.964828968048096
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.964646816253662
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.9645915031433105
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.964428424835205
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.964176177978516
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.964416980743408
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.964077949523926
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.964686870574951
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.964921474456787
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.964619159698486
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.9647088050842285
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.963682651519775
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.964191913604736
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.9648261070251465
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.964590549468994
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.964537143707275
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.964656352996826
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.964632511138916
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.963538646697998
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.9644293785095215
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.963883399963379
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.964787483215332
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.964715003967285
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.964576244354248
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.964606285095215
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.964107036590576
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.964235305786133
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.964028358459473
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.964752197265625
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.964895248413086
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.964996814727783
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.964977741241455
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.963625431060791
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.9647111892700195
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.964138984680176
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.9649176597595215
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.964878082275391
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.964503765106201
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.964620590209961
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.963753700256348
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.964656352996826
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.964402198791504
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.964504718780518
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.9649176597595215
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.964693546295166
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.964619159698486
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.963278293609619
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.964744567871094
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.964730262756348
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.964637279510498
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.964547157287598
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.964498996734619
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.9650139808654785
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.963581562042236
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.9639692306518555
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.964494228363037
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.964633941650391
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.964561462402344
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.964945316314697
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.964648246765137
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.963507652282715
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.964051723480225
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.96372652053833
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.964704990386963
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.96455717086792
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.965011119842529
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.964638710021973
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.963899612426758
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.964288711547852
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.964208602905273
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.964895725250244
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.964487552642822
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.964959621429443
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.964356899261475
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.963806629180908
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.964376449584961
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.9645514488220215
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.9644083976745605
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.964900016784668
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.965603828430176
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.964944839477539
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.963250637054443
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.9628448486328125
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.9618940353393555
+                }
+            ]
+        },
+        {
+            "bit": 7,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.888533592224121
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.888609409332275
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.888508319854736
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.888683795928955
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.888564109802246
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.888050079345703
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.888296127319336
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.888637065887451
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.888768196105957
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.888139724731445
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.888930320739746
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.888051986694336
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.887570381164551
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.888113498687744
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.888205051422119
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.888602256774902
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.8882832527160645
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.888096809387207
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.887925148010254
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.888683795928955
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.887927532196045
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.888339996337891
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.8881306648254395
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.8882622718811035
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.887542724609375
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.887788772583008
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.888004302978516
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.887856483459473
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.888277053833008
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.888645648956299
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.888360977172852
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.8880085945129395
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.8879852294921875
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.887504577636719
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.887173175811768
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.888638496398926
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.888370513916016
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.888504981994629
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.887874126434326
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.888915538787842
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.889015197753906
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.888710975646973
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.888538837432861
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.88849401473999
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.888706684112549
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.8884735107421875
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.888428688049316
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.888202667236328
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.888516426086426
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.888519287109375
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.888779163360596
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.888981342315674
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.888237476348877
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.888044834136963
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.887911796569824
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.889214992523193
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.888627529144287
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.888639450073242
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.888596534729004
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.888563632965088
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.888638496398926
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.888349533081055
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.888360977172852
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.888574123382568
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.888939380645752
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.888828277587891
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.888360500335693
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.888573169708252
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.889019012451172
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.889037609100342
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.88852596282959
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.888480186462402
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.888377666473389
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.888641357421875
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.8884477615356445
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.888152122497559
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.888753890991211
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.888671875
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.88850212097168
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.888184547424316
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.888620853424072
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.888172626495361
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.888339042663574
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.888538360595703
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.888589382171631
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.888647556304932
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.888367176055908
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.888630390167236
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.8881964683532715
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.888460636138916
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.888651371002197
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.888354301452637
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.88836145401001
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.888493537902832
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.888124465942383
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.888256072998047
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.888362407684326
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.888558864593506
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.888823986053467
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.888772964477539
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.88839864730835
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.888772010803223
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.888427734375
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.888687610626221
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.888415336608887
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.888566970825195
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.88886022567749
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.888436794281006
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.888871669769287
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.888151168823242
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.888279914855957
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.888452053070068
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.8886847496032715
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.888696670532227
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.888360977172852
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.888774871826172
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.887986183166504
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.888555526733398
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.888382911682129
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.88839054107666
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.888365745544434
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.88832426071167
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.888453006744385
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.888110160827637
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.888450622558594
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.888492584228516
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.888844013214111
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.888654708862305
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.888604640960693
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.888652324676514
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.887988090515137
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.888352870941162
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.888880252838135
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.888491153717041
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.8884758949279785
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.888349533081055
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.888465404510498
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.888376235961914
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.8885602951049805
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.888815879821777
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.888659954071045
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.888651371002197
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.88850212097168
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.888739585876465
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.887884140014648
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.888887882232666
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.888637065887451
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.888782978057861
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.888707637786865
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.888581275939941
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.888576984405518
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.887940406799316
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.8884148597717285
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.88815975189209
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.888671398162842
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.888727188110352
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.8888397216796875
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.888592720031738
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.888171195983887
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.888138771057129
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.88839864730835
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.888693332672119
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.888722896575928
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.888566493988037
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.888630390167236
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.887894630432129
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.888371467590332
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.8883056640625
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.888612747192383
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.8885498046875
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.888686180114746
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.888600826263428
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.888392925262451
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.888243675231934
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.887849807739258
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.8886542320251465
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.888656139373779
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.888689994812012
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.888861656188965
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.88805627822876
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.8883466720581055
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.888495445251465
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.8885979652404785
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.888657093048096
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.888370513916016
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.888408660888672
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.888245582580566
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.887797832489014
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.88852071762085
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.888506889343262
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.888587474822998
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.888690948486328
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.888582706451416
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.888027667999268
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.8884406089782715
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.888558864593506
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.888631820678711
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.888638496398926
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.8885416984558105
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.888225555419922
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.888041973114014
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.888441562652588
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.888904094696045
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.888408660888672
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.888652801513672
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.888426780700684
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.888599395751953
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.887938976287842
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.888474464416504
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.888730525970459
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.8884687423706055
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.888638496398926
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.888818740844727
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.888711452484131
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.887609481811523
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.887635707855225
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.88871431350708
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.888470649719238
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.888686180114746
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.889449596405029
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.888946056365967
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.88793420791626
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.887157440185547
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.8879170417785645
+                }
+            ]
+        },
+        {
+            "bit": 8,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.846884727478027
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.846755027770996
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.846534729003906
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.846395492553711
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.846962928771973
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.847113132476807
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.846520900726318
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.846809387207031
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.846713066101074
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.846841335296631
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.846555233001709
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.846278190612793
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.846106052398682
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.846847057342529
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.84657621383667
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.846549987792969
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.8466362953186035
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.8466925621032715
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.846368789672852
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.846569538116455
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.846435070037842
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.846747875213623
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.8471479415893555
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.846069812774658
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.846944808959961
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.846384048461914
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.8465576171875
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.846332550048828
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.846738815307617
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.846657752990723
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.8466315269470215
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.846845626831055
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.846436500549316
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.84717321395874
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.846818923950195
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.846718788146973
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.846808910369873
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.846215724945068
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.846886157989502
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.846282005310059
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.846563816070557
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.8466267585754395
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.846828937530518
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.846994876861572
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.8467512130737305
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.846724033355713
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.8460187911987305
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.846347332000732
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.846561431884766
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.846907615661621
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.846936225891113
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.846603870391846
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.847090721130371
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.846323013305664
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.846722602844238
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.846107482910156
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.8468122482299805
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.8466105461120605
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.846934795379639
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.846707344055176
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.846489429473877
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.846768856048584
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.846676349639893
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.846982002258301
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.846992015838623
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.846632957458496
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.846888065338135
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.846412181854248
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.846067905426025
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.846593379974365
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.8468451499938965
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.846786975860596
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.84684419631958
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.847229957580566
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.846533298492432
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.846888065338135
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.846631050109863
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.847040176391602
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.847057819366455
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.84683084487915
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.846927642822266
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.846968650817871
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.846551418304443
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.84695291519165
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.846982002258301
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.846972465515137
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.846938133239746
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.846807956695557
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.846613883972168
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.846781253814697
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.846508502960205
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.84689998626709
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.846874237060547
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.846653938293457
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.846846580505371
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.846853256225586
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.846768856048584
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.846569538116455
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.846914768218994
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.846983909606934
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.847011089324951
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.846548080444336
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.846460819244385
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.846813678741455
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.846814155578613
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.8469462394714355
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.846782684326172
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.84659481048584
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.8464837074279785
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.8469648361206055
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.846916675567627
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.846741199493408
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.8471574783325195
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.846906661987305
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.846940994262695
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.846747875213623
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.846822738647461
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.84714412689209
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.846920967102051
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.846752166748047
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.846938610076904
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.846940040588379
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.846876621246338
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.846527576446533
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.846695423126221
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.8467864990234375
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.846994876861572
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.846986770629883
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.846661567687988
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.846889019012451
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.846558570861816
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.846768379211426
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.846653461456299
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.846931457519531
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.847026824951172
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.846749782562256
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.846560001373291
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.846208572387695
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.846831321716309
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.846343517303467
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.846974849700928
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.8468122482299805
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.846828937530518
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.846536636352539
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.846377372741699
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.8466315269470215
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.846134662628174
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.8467936515808105
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.846747398376465
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.846767902374268
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.8468122482299805
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.84634256362915
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.846466064453125
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.846446990966797
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.846789836883545
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.846935272216797
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.846797466278076
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.846704959869385
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.846429347991943
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.846837520599365
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.846591472625732
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.846845626831055
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.846902370452881
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.8468828201293945
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.846884727478027
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.846377372741699
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.846712589263916
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.846724033355713
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.846932411193848
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.846951484680176
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.846959114074707
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.846892833709717
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.845987319946289
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.84673547744751
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.846661567687988
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.846816539764404
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.8470354080200195
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.846877574920654
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.846774578094482
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.846625328063965
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.846673488616943
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.846678256988525
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.8467936515808105
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.846768856048584
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.846697807312012
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.846809387207031
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.846215724945068
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.847002983093262
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.846602916717529
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.846771717071533
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.846909523010254
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.8467912673950195
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.846930027008057
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.846335411071777
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.846407413482666
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.846812725067139
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.84692907333374
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.846823215484619
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.846906661987305
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.8469085693359375
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.846124649047852
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.846503734588623
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.846322536468506
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.846792221069336
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.84681510925293
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.847112655639648
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.84688663482666
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.846489906311035
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.846936225891113
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.846786975860596
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.846865653991699
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.847081184387207
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.846912860870361
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.846500396728516
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.84630012512207
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.846616744995117
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.84650993347168
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.846712589263916
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.846808910369873
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.847279071807861
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.846980571746826
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.845856189727783
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.8459672927856445
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.845388889312744
+                }
+            ]
+        },
+        {
+            "bit": 9,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.807320594787598
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.807447910308838
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.807397842407227
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.807557106018066
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.80731725692749
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.8072991371154785
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.807169437408447
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.80741024017334
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.807368278503418
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.806540489196777
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.807317733764648
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.807284832000732
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.806679725646973
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.807029724121094
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.806914806365967
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.80698823928833
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.806814193725586
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.80743408203125
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.807092189788818
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.807408809661865
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.806885242462158
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.807365894317627
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.807035446166992
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.80775260925293
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.8065619468688965
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.806995868682861
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.807311058044434
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.807106971740723
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.807308197021484
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.807418346405029
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.806776523590088
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.807122707366943
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.807094573974609
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.807129859924316
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.806788921356201
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.8074564933776855
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.80728006362915
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.8073344230651855
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.806854724884033
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.807610034942627
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.807583808898926
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.807412147521973
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.807064533233643
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.80736780166626
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.80729866027832
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.807192802429199
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.806939125061035
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.80729866027832
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.807465076446533
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.807305335998535
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.807262897491455
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.807672500610352
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.807365894317627
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.807225227355957
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.807041645050049
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.807146072387695
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.807360649108887
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.807380676269531
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.8074259757995605
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.807681083679199
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.807589530944824
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.80717658996582
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.807328701019287
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.807476997375488
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.807507514953613
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.807347297668457
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.807212829589844
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.8072710037231445
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.80779504776001
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.807518482208252
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.8072991371154785
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.807279109954834
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.807272434234619
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.807150363922119
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.807283878326416
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.807187557220459
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.807396411895752
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.807284832000732
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.807275295257568
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.807399272918701
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.807305812835693
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.80723762512207
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.80735445022583
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.807417869567871
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.807423114776611
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.8074541091918945
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.807394027709961
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.807457447052002
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.807441711425781
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.807389736175537
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.807151794433594
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.807308197021484
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.807285308837891
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.8072099685668945
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.807114601135254
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.807153224945068
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.8076395988464355
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.807408332824707
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.807433605194092
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.807414531707764
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.807464599609375
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.807225227355957
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.807118892669678
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.8074798583984375
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.807446002960205
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.807308197021484
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.807493686676025
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.807308673858643
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.8073601722717285
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.8069047927856445
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.807224273681641
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.80726957321167
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.807433128356934
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.807378768920898
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.807392597198486
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.807464599609375
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.807135105133057
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.807191848754883
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.80746603012085
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.807254791259766
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.807539939880371
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.807183265686035
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.807294845581055
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.8072052001953125
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.807293891906738
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.807287693023682
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.807423114776611
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.807347774505615
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.807369232177734
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.807397842407227
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.807180404663086
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.807460308074951
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.807249069213867
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.8072896003723145
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.807324409484863
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.806993007659912
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.807295799255371
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.807324409484863
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.807216167449951
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.8074049949646
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.807384967803955
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.807507038116455
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.807447910308838
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.8075151443481445
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.806991100311279
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.807608604431152
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.807516098022461
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.80740213394165
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.807366847991943
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.807394027709961
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.807435035705566
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.807272434234619
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.807493686676025
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.807199001312256
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.807322025299072
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.8074564933776855
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.8073296546936035
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.807380199432373
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.807191371917725
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.807180404663086
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.8075056076049805
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.807433128356934
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.807493209838867
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.807223796844482
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.807295799255371
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.807287693023682
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.807112693786621
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.807374477386475
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.807324409484863
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.807441234588623
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.807288646697998
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.8074116706848145
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.807272434234619
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.80704402923584
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.8070878982543945
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.8074564933776855
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.80736780166626
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.807421684265137
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.807408809661865
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.807161331176758
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.807325839996338
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.807275295257568
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.80742883682251
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.807389736175537
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.807441234588623
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.807344436645508
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.807398796081543
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.806964874267578
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.807369232177734
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.807373046875
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.807372093200684
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.807512283325195
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.807385444641113
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.807251453399658
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.807164669036865
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.807197570800781
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.807240009307861
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.807430267333984
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.807344913482666
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.80710506439209
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.806970596313477
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.806989669799805
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.8075642585754395
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.807403564453125
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.807308197021484
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.807372093200684
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.807277202606201
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.807153701782227
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.807036876678467
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.807571887969971
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.807326793670654
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.807351112365723
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.8075714111328125
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.807255268096924
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.80708646774292
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.807107925415039
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.807385444641113
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.807175159454346
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.807455062866211
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.807769298553467
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.807490348815918
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.807016372680664
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.806805610656738
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.806951999664307
+                }
+            ]
+        },
+        {
+            "bit": 10,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.7860870361328125
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.786165714263916
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.786190032958984
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.785765647888184
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.7859272956848145
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.78639030456543
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.785793304443359
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.786105632781982
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.785983562469482
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.7862372398376465
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.785848140716553
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.785799026489258
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.785938262939453
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.785671234130859
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.786051273345947
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.786233425140381
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.786227703094482
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.785523891448975
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.785677909851074
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.785831928253174
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.7858195304870605
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.786016464233398
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.786160945892334
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.78556489944458
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.786416053771973
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.7861504554748535
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.7858428955078125
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.785478591918945
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.78593111038208
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.785983562469482
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.786014556884766
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.785986423492432
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.785490989685059
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.786184310913086
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.7858195304870605
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.785972595214844
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.785904884338379
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.78585958480835
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.786273002624512
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.785771369934082
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.785844802856445
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.786144256591797
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.786036491394043
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.786085605621338
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.786350727081299
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.785879135131836
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.7859344482421875
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.785741806030273
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.785911560058594
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.78611421585083
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.786179542541504
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.786022186279297
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.785970687866211
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.785727024078369
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.786327838897705
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.785758972167969
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.78601598739624
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.78586483001709
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.786107540130615
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.785608291625977
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.7859110832214355
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.785917282104492
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.786117076873779
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.78607702255249
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.7861433029174805
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.785947322845459
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.785985946655273
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.7856950759887695
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.785669803619385
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.785665035247803
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.78601598739624
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.785956382751465
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.786189079284668
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.786167144775391
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.7859015464782715
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.78593635559082
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.785836219787598
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.78613805770874
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.785987854003906
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.786139488220215
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.785860061645508
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.785974025726318
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.7858567237854
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.78590202331543
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.7861127853393555
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.786235809326172
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.78599214553833
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.785956859588623
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.78582763671875
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.786026477813721
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.786167144775391
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.7860002517700195
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.785965919494629
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.785858154296875
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.785942554473877
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.786120891571045
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.7859601974487305
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.785771369934082
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.786098003387451
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.786062240600586
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.786027908325195
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.7860846519470215
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.785863399505615
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.78599214553833
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.786008358001709
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.786103248596191
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.786043643951416
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.785855293273926
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.785950660705566
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.786082744598389
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.786101818084717
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.786012649536133
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.786051273345947
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.786034107208252
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.785958290100098
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.7859930992126465
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.7859086990356445
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.7862467765808105
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.785863399505615
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.786041259765625
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.786000728607178
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.785933017730713
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.785943984985352
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.7859673500061035
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.785989284515381
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.785783767700195
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.786108016967773
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.786021709442139
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.786001682281494
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.785774230957031
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.785782814025879
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.785972595214844
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.785923957824707
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.786153316497803
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.786073207855225
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.786062240600586
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.785933017730713
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.785512924194336
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.786040306091309
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.785582542419434
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.786125659942627
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.786070346832275
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.785903453826904
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.785881519317627
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.7858052253723145
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.785869598388672
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.785634517669678
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.7860894203186035
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.785984992980957
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.78590202331543
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.785776615142822
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.785739898681641
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.785686016082764
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.7858710289001465
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.786015510559082
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.785984992980957
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.786074638366699
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.7859673500061035
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.7857794761657715
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.785918712615967
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.785735130310059
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.786016464233398
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.786036014556885
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.786085605621338
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.786025047302246
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.785719394683838
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.786073684692383
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.785758018493652
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.786116123199463
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.7859954833984375
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.7860894203186035
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.786044120788574
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.785635471343994
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.786078453063965
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.785892486572266
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.7859907150268555
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.786092281341553
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.785958290100098
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.785780429840088
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.7859063148498535
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.785931587219238
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.7859907150268555
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.786015510559082
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.78598690032959
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.785983562469482
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.786022186279297
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.785456657409668
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.786066055297852
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.7858195304870605
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.786002159118652
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.786025524139404
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.785871982574463
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.786020755767822
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.7857747077941895
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.785984516143799
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.786062717437744
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.785949230194092
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.786081314086914
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.7859272956848145
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.7860589027404785
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.785386562347412
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.7861127853393555
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.785547256469727
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.785964488983154
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.7859063148498535
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.786127090454102
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.785897731781006
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.785623550415039
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.785971164703369
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.78568696975708
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.785957336425781
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.7860236167907715
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.785878658294678
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.785817623138428
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.785684108734131
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.7856855392456055
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.785557270050049
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.78585958480835
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.785964488983154
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.786198139190674
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.7861008644104
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.785236835479736
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.785416126251221
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.785285472869873
+                }
+            ]
+        },
+        {
+            "bit": 11,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.763876914978027
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.763910293579102
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.763853549957275
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.76401424407959
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.763963222503662
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.7636260986328125
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.764211177825928
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.763911724090576
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.763829708099365
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.763540744781494
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.763612747192383
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.763721466064453
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.76350212097168
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.763619899749756
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.763597011566162
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.763787746429443
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.763580799102783
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.7638840675354
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.763800144195557
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.763984680175781
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.763599395751953
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.763930797576904
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.76369571685791
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.7638163566589355
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.7631916999816895
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.763453960418701
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.763749122619629
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.7637505531311035
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.763873100280762
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.763938903808594
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.763756275177002
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.763725280761719
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.763880729675293
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.763558864593506
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.763548851013184
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.763647556304932
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.763906478881836
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.763967037200928
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.763612747192383
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.763794422149658
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.764153003692627
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.763688087463379
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.763852119445801
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.763820648193359
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.763698101043701
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.7637410163879395
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.763702869415283
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.764163970947266
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.763774394989014
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.763908863067627
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.763912200927734
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.7638421058654785
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.763820648193359
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.763578414916992
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.763699054718018
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.764079570770264
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.763862133026123
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.763889312744141
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.7640557289123535
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.763990879058838
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.763952255249023
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.763833045959473
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.763771057128906
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.763866901397705
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.76412296295166
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.763780117034912
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.7637505531311035
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.763862133026123
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.763949394226074
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.764062404632568
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.763815402984619
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.763830184936523
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.763818740844727
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.76371431350708
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.763657569885254
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.76369571685791
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.764039039611816
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.763833045959473
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.763803958892822
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.763761520385742
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.764018535614014
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.76369571685791
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.763611793518066
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.763716220855713
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.763824939727783
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.763736724853516
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.763860702514648
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.763862133026123
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.763857841491699
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.763759613037109
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.763803005218506
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.7638092041015625
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.7638349533081055
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.763949871063232
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.763761043548584
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.763696193695068
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.763683319091797
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.763885498046875
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.763844013214111
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.7638373374938965
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.763776779174805
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.763764381408691
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.763606071472168
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.763893127441406
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.763791084289551
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.763823986053467
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.763840675354004
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.763827800750732
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.763771057128906
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.763636112213135
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.763758659362793
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.763873100280762
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.76384973526001
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.763851165771484
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.7639641761779785
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.763798713684082
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.763840675354004
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.763706684112549
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.763923168182373
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.763768196105957
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.763861179351807
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.763614177703857
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.763744831085205
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.763579368591309
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.763778209686279
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.763814449310303
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.763911724090576
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.763951301574707
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.763959884643555
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.7639641761779785
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.763771057128906
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.763907432556152
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.763689994812012
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.7637834548950195
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.7639617919921875
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.763702392578125
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.7637858390808105
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.763891696929932
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.763716220855713
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.76397180557251
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.763879776000977
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.763941764831543
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.763826370239258
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.763874530792236
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.763741493225098
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.764047145843506
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.7637410163879395
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.76384973526001
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.763911724090576
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.76379919052124
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.76381778717041
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.763568878173828
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.763959884643555
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.763635158538818
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.7637200355529785
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.763876438140869
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.763875961303711
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.763876914978027
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.7636542320251465
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.763672351837158
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.763879776000977
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.763868808746338
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.763914108276367
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.7638654708862305
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.7637410163879395
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.763660907745361
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.763449668884277
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.7637619972229
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.76393985748291
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.763997554779053
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.763876914978027
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.763798236846924
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.763744831085205
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.763727188110352
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.763614177703857
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.763899326324463
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.763771057128906
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.763908863067627
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.7639899253845215
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.76370096206665
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.763743877410889
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.763840198516846
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.763787746429443
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.76381254196167
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.763828754425049
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.76360559463501
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.763652801513672
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.763684272766113
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.763845443725586
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.763819217681885
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.76375675201416
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.763904094696045
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.76383638381958
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.763662338256836
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.763668060302734
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.763660907745361
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.763773441314697
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.763828754425049
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.763808250427246
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.763736724853516
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.763669490814209
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.763576984405518
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.763879776000977
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.763696670532227
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.763768196105957
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.763823509216309
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.763830184936523
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.763809680938721
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.7637152671813965
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.763912677764893
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.763919830322266
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.763867378234863
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.764018535614014
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.7637739181518555
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.763439178466797
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.76362943649292
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.7639055252075195
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.763725280761719
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.763826370239258
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.763906002044678
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.763828277587891
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.763727188110352
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.76338005065918
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.76322603225708
+                }
+            ]
+        },
+        {
+            "bit": 12,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.750020980834961
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.750077724456787
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.750213623046875
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.7500786781311035
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.7501044273376465
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.750396728515625
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.74979829788208
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.750003814697266
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.750091075897217
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.750096321105957
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.750409126281738
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.750068664550781
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.749920845031738
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.749948024749756
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.749916076660156
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.749958038330078
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.7501540184021
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.749876976013184
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.7501115798950195
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.749826908111572
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.749764919281006
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.750082492828369
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.750151634216309
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.749709606170654
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.750182628631592
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.750070095062256
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.74973201751709
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.749866008758545
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.750063419342041
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.749924182891846
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.749812602996826
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.749690532684326
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.749947547912598
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.750251293182373
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.749931812286377
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.750001907348633
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.749978542327881
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.7501020431518555
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.750133514404297
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.749849796295166
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.749995231628418
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.750255584716797
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.75007438659668
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.750039100646973
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.750225067138672
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.7498674392700195
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.750048637390137
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.74960994720459
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.750172138214111
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.750096321105957
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.750021934509277
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.7498884201049805
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.750110149383545
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.749950408935547
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.749808311462402
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.749939441680908
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.750105857849121
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.7499566078186035
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.75016450881958
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.750014305114746
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.749876499176025
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.7500810623168945
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.750097274780273
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.750137805938721
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.7501068115234375
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.750007629394531
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.749920845031738
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.749889373779297
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.750068664550781
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.749770164489746
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.74998140335083
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.7499895095825195
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.750071048736572
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.750212669372559
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.749993801116943
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.750055313110352
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.74984073638916
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.75002384185791
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.749930381774902
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.749964714050293
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.749937057495117
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.749963283538818
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.750051021575928
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.750151634216309
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.750094890594482
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.750199794769287
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.750016212463379
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.749950885772705
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.749963283538818
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.750150680541992
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.74990177154541
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.750068187713623
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.749967575073242
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.750001430511475
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.7498674392700195
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.750048637390137
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.750049591064453
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.749871730804443
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.750004768371582
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.750156402587891
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.7500319480896
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.750095844268799
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.749948978424072
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.749963283538818
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.750036239624023
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.749994277954102
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.750004768371582
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.7499799728393555
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.749864101409912
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.749906063079834
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.750154972076416
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.749868869781494
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.750053882598877
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.749982833862305
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.749892234802246
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.750008583068848
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.74986457824707
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.750156879425049
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.7498779296875
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.7501044273376465
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.750033378601074
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.750000476837158
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.7498297691345215
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.750145435333252
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.750127792358398
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.749939918518066
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.750089645385742
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.750092029571533
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.749913692474365
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.749995231628418
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.74984884262085
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.749996185302734
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.750014305114746
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.7500786781311035
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.750009536743164
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.750150203704834
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.75006628036499
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.7498250007629395
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.750083923339844
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.749686241149902
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.750180244445801
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.749969005584717
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.749982833862305
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.750037670135498
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.7498908042907715
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.749860763549805
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.749767780303955
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.75007963180542
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.750010013580322
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.750041961669922
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.750009536743164
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.749959945678711
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.749973773956299
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.7500200271606445
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.750053882598877
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.750007629394531
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.750096321105957
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.750121116638184
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.74991512298584
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.7500762939453125
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.749960422515869
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.7500200271606445
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.750077247619629
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.75005578994751
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.750024795532227
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.749948024749756
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.750172138214111
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.75
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.750148773193359
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.750057697296143
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.7499799728393555
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.7500786781311035
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.74978494644165
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.749974250793457
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.750060081481934
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.749953746795654
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.750132083892822
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.750023365020752
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.749988079071045
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.74991512298584
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.749953746795654
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.749972820281982
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.750073432922363
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.750045299530029
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.749997138977051
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.750068187713623
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.749803066253662
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.7501068115234375
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.749828815460205
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.749999046325684
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.750051021575928
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.750063896179199
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.750000476837158
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.749790668487549
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.749943256378174
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.750059127807617
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.749959468841553
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.7500319480896
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.750086784362793
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.749938488006592
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.7497076988220215
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.75007963180542
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.749924659729004
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.750041961669922
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.749931812286377
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.7501678466796875
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.749998569488525
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.749776363372803
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.750130653381348
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.749953746795654
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.750038146972656
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.750070095062256
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.749917507171631
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.7498884201049805
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.7498250007629395
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.7501678466796875
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.7499165534973145
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.749966621398926
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.750113487243652
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.750253200531006
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.750173568725586
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.749752521514893
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.74944543838501
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.749748229980469
+                }
+            ]
+        },
+        {
+            "bit": 13,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.735749244689941
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.735605716705322
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.735559463500977
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.735401630401611
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.735648155212402
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.735724925994873
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.735809803009033
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.735662937164307
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.735661506652832
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.735718250274658
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.735525608062744
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.735543251037598
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.735413074493408
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.735805034637451
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.7355475425720215
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.73565673828125
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.735165596008301
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.735630512237549
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.735355377197266
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.735599517822266
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.73553466796875
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.735842704772949
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.735633850097656
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.73579740524292
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.735411643981934
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.735323429107666
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.735809803009033
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.735527038574219
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.735580921173096
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.735686779022217
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.735784530639648
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.735810279846191
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.735538482666016
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.735507488250732
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.735589027404785
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.735638618469238
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.735682964324951
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.735683441162109
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.735445499420166
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.735910415649414
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.735760688781738
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.735584259033203
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.735650539398193
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.735589027404785
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.735595226287842
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.73568058013916
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.7354302406311035
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.735755920410156
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.735533237457275
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.7357025146484375
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.735727787017822
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.735914707183838
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.735564708709717
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.735570907592773
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.735718250274658
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.735691070556641
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.735559463500977
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.735669136047363
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.736020565032959
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.735628604888916
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.735891819000244
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.73552131652832
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.73578405380249
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.735596656799316
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.735816478729248
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.735847473144531
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.735689640045166
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.735713958740234
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.735567092895508
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.735745429992676
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.7356743812561035
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.73566198348999
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.735579967498779
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.735451698303223
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.735611915588379
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.7354865074157715
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.735877990722656
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.735663414001465
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.735743999481201
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.735640048980713
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.735723495483398
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.735437870025635
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.735717296600342
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.73552942276001
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.7357048988342285
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.735743045806885
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.735779762268066
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.735907077789307
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.735551834106445
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.735812664031982
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.735816478729248
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.735585689544678
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.735698699951172
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.735727787017822
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.735485076904297
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.735496520996094
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.735637187957764
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.73578405380249
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.735725402832031
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.7356953620910645
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.735657215118408
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.73576545715332
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.735666275024414
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.735670566558838
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.735718250274658
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.735681533813477
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.735720634460449
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.735648155212402
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.735612869262695
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.7354960441589355
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.735509872436523
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.7357869148254395
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.735675811767578
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.735697269439697
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.735825061798096
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.735678672790527
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.7356276512146
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.735659599304199
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.735790729522705
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.735655307769775
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.7357354164123535
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.735655307769775
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.735683917999268
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.735528945922852
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.7355241775512695
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.735678672790527
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.735630989074707
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.735649585723877
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.735651016235352
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.735783576965332
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.73565673828125
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.735672473907471
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.7355170249938965
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.735677242279053
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.735748291015625
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.735722064971924
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.735543251037598
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.735703945159912
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.735637187957764
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.735703468322754
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.7356367111206055
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.735689640045166
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.735782623291016
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.735724925994873
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.735607624053955
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.7357988357543945
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.735667705535889
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.735719680786133
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.7357048988342285
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.7355146408081055
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.735562801361084
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.735489845275879
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.735600471496582
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.735648155212402
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.735593795776367
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.735612869262695
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.735564708709717
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.735640048980713
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.735483646392822
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.735544681549072
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.735790729522705
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.7357330322265625
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.73561429977417
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.735673904418945
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.735618591308594
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.735533714294434
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.735453128814697
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.735683917999268
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.735688209533691
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.735688209533691
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.735729217529297
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.7357401847839355
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.735551834106445
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.735549449920654
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.735442161560059
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.735749244689941
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.735632419586182
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.735667705535889
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.735859394073486
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.735560894012451
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.735610485076904
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.735659599304199
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.735598564147949
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.735591888427734
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.735774517059326
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.735651016235352
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.735477447509766
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.7354536056518555
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.735658168792725
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.735630035400391
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.735575199127197
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.735724449157715
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.735669136047363
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.735570907592773
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.735556125640869
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.7355146408081055
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.735653877258301
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.735621452331543
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.735678672790527
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.7357940673828125
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.7357072830200195
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.735198497772217
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.735698699951172
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.735628604888916
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.735651016235352
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.735590934753418
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.735592365264893
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.735528469085693
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.735500812530518
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.735600471496582
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.735644340515137
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.73559045791626
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.735889911651611
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.735602855682373
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.735527038574219
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.735183238983154
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.735662937164307
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.735490322113037
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.735570430755615
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.735771179199219
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.735713958740234
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.735530376434326
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.735454559326172
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.735284328460693
+                }
+            ]
+        },
+        {
+            "bit": 14,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.727272033691406
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.727263927459717
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.727447986602783
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.727210521697998
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.7273149490356445
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.727382659912109
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.727174282073975
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.727188587188721
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.727271556854248
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.727020740509033
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.727312088012695
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.727245807647705
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.72697114944458
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.727160930633545
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.727227210998535
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.727433681488037
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.727734565734863
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.727424621582031
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.727383136749268
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.727261543273926
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.727002143859863
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.727267742156982
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.72728967666626
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.727218151092529
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.7272820472717285
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.7273101806640625
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.727075576782227
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.727088451385498
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.727281093597412
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.727230548858643
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.727171421051025
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.727014064788818
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.7270002365112305
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.7271623611450195
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.7271857261657715
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.727338790893555
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.727336406707764
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.727290630340576
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.727341651916504
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.726978302001953
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.727092742919922
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.727373123168945
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.727230072021484
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.727346897125244
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.727179527282715
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.727208614349365
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.727406024932861
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.727031230926514
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.727427959442139
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.727264404296875
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.727366924285889
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.727170944213867
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.727219104766846
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.727188587188721
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.727202415466309
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.727203845977783
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.727184295654297
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.727207660675049
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.727180480957031
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.727263927459717
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.727007865905762
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.727344512939453
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.727258205413818
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.72737979888916
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.727283954620361
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.727219581604004
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.7271528244018555
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.726965427398682
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.727278232574463
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.727016448974609
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.727213382720947
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.727233409881592
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.727285385131836
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.727283954620361
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.727242469787598
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.727166652679443
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.727079391479492
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.727243900299072
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.727213382720947
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.7272257804870605
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.727101802825928
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.72724723815918
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.727171421051025
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.727391719818115
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.727233409881592
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.727422714233398
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.727375030517578
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.727240562438965
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.727254390716553
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.727351665496826
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.727051734924316
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.727200984954834
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.727278232574463
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.727243900299072
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.727210521697998
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.727277755737305
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.727265357971191
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.727119445800781
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.727256774902344
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.727424621582031
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.72728967666626
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.7272162437438965
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.727060794830322
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.727225303649902
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.727285861968994
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.727351665496826
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.727291584014893
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.727375030517578
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.727297782897949
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.7272210121154785
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.727382659912109
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.727233409881592
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.727297782897949
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.727248191833496
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.727179050445557
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.727337837219238
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.727230548858643
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.727355480194092
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.727076530456543
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.727263927459717
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.727281093597412
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.727174282073975
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.727208614349365
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.727336406707764
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.727266311645508
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.727136135101318
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.727245330810547
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.727322578430176
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.727303981781006
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.727168083190918
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.727179050445557
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.727265357971191
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.727177619934082
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.727233409881592
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.727229595184326
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.727142810821533
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.727259159088135
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.72700309753418
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.727283954620361
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.726864337921143
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.727231502532959
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.727243900299072
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.727282524108887
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.727181911468506
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.727109432220459
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.727280139923096
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.727066993713379
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.727230548858643
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.727263927459717
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.727174758911133
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.727259159088135
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.727238178253174
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.72711181640625
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.727234840393066
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.727338790893555
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.727208614349365
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.727366924285889
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.727307319641113
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.727176666259766
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.727256774902344
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.727062225341797
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.727206707000732
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.727266311645508
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.727261066436768
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.727260589599609
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.727237701416016
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.727290630340576
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.727205276489258
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.727319717407227
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.7273149490356445
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.727163791656494
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.727313995361328
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.727117538452148
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.727297782897949
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.727334022521973
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.7272562980651855
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.7273383140563965
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.727236270904541
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.727217674255371
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.727133274078369
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.727233409881592
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.727234840393066
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.727259635925293
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.727290630340576
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.727233409881592
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.727207660675049
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.727382659912109
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.727365970611572
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.7271857261657715
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.727353572845459
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.727325439453125
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.727242946624756
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.727260589599609
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.72725772857666
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.72722053527832
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.727349281311035
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.727334022521973
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.727223873138428
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.727336406707764
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.727236747741699
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.726943016052246
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.727406978607178
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.727139472961426
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.72732400894165
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.727234840393066
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.727407455444336
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.727261066436768
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.7272748947143555
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.727225303649902
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.727275371551514
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.727323055267334
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.727292060852051
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.727206707000732
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.727131366729736
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.727207183837891
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.727283954620361
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.727217674255371
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.727317810058594
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.727277755737305
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.727339744567871
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.727353572845459
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.727086544036865
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.727168083190918
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.7270188331604
+                }
+            ]
+        },
+        {
+            "bit": 15,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 5.716471195220947
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 5.7165021896362305
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 5.716497421264648
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 5.716405868530273
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 5.716554641723633
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 5.716458320617676
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 5.716691017150879
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 5.716512203216553
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 5.716405391693115
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 5.716665744781494
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 5.716557025909424
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 5.716489791870117
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 5.716475486755371
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 5.716529369354248
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 5.716424942016602
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 5.716368675231934
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 5.71601676940918
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 5.716370582580566
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 5.7161784172058105
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 5.716451168060303
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 5.716567516326904
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 5.716444969177246
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 5.716471195220947
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 5.716361045837402
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 5.716320037841797
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 5.716214656829834
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 5.716564655303955
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 5.716272354125977
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 5.716392517089844
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 5.716505527496338
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 5.716416835784912
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 5.716427326202393
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 5.716489315032959
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 5.716395854949951
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 5.716413974761963
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 5.7164692878723145
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 5.716431140899658
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 5.7164387702941895
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 5.716334342956543
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 5.7166547775268555
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 5.716627597808838
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 5.716324806213379
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 5.716473579406738
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 5.7164306640625
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 5.716298580169678
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 5.716250896453857
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 5.716250896453857
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 5.716433525085449
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 5.716359615325928
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 5.716538429260254
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 5.716472625732422
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 5.716551303863525
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 5.716439723968506
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 5.716416358947754
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 5.716301441192627
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 5.716520309448242
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 5.716432094573975
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 5.716367244720459
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 5.7166218757629395
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 5.7164154052734375
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 5.716714382171631
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 5.716241359710693
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 5.716564178466797
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 5.716377258300781
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 5.716472148895264
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 5.716655731201172
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 5.716405868530273
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 5.716425895690918
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 5.716358661651611
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 5.716618061065674
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 5.716516494750977
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 5.716457843780518
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 5.716426372528076
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 5.716289520263672
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 5.71643590927124
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 5.7161641120910645
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 5.716543197631836
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 5.716419696807861
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 5.716540336608887
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 5.716250896453857
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 5.716429710388184
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 5.716384410858154
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 5.716546535491943
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 5.716248035430908
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 5.7163872718811035
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 5.716500282287598
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 5.716453552246094
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 5.716541290283203
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 5.716310501098633
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 5.7164998054504395
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 5.71669864654541
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 5.716495513916016
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 5.716455459594727
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 5.716395378112793
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 5.716398239135742
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 5.7163801193237305
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 5.716507911682129
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 5.716575622558594
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 5.716522693634033
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 5.7164106369018555
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 5.716610431671143
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 5.7163896560668945
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 5.7164835929870605
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 5.716461658477783
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 5.716370582580566
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 5.7164788246154785
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 5.716495990753174
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 5.716524600982666
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 5.716317176818848
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 5.716423034667969
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 5.716396808624268
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 5.716536045074463
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 5.716484546661377
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 5.7164506912231445
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 5.716594219207764
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 5.7163920402526855
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 5.7164225578308105
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 5.716315746307373
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 5.716479778289795
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 5.716516494750977
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 5.716393947601318
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 5.716498851776123
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 5.71648645401001
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 5.716279983520508
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 5.716372013092041
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 5.716432571411133
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 5.716487884521484
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 5.71643590927124
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 5.7163872718811035
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 5.716500282287598
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 5.716414928436279
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 5.716492176055908
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 5.716322898864746
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 5.716555595397949
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 5.716487884521484
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 5.716487407684326
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 5.716372013092041
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 5.716485023498535
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 5.716375827789307
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 5.716655731201172
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 5.716384410858154
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 5.716467380523682
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 5.716480255126953
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 5.716492652893066
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 5.716387748718262
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 5.716408729553223
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 5.716521263122559
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 5.716517925262451
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 5.716440200805664
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 5.71645975112915
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 5.716413974761963
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 5.716322898864746
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 5.716640472412109
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 5.716489315032959
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 5.716421604156494
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 5.716449737548828
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 5.716375350952148
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 5.7164788246154785
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 5.7164788246154785
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 5.716489315032959
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 5.716472625732422
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 5.7164835929870605
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 5.716504096984863
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 5.7164788246154785
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 5.7164812088012695
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 5.716255187988281
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 5.7164998054504395
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 5.716384410858154
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 5.716497421264648
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 5.7164387702941895
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 5.716585159301758
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 5.716421127319336
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 5.716434478759766
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 5.716467380523682
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 5.716280460357666
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 5.716424942016602
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 5.716375827789307
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 5.716519832611084
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 5.7164306640625
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 5.716427326202393
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 5.716678142547607
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 5.716527462005615
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 5.716350555419922
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 5.716449737548828
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 5.716461658477783
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 5.716452598571777
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 5.716210842132568
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 5.716273307800293
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 5.716616630554199
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 5.716398239135742
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 5.716460227966309
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 5.716630935668945
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 5.716404438018799
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 5.716335296630859
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 5.71648645401001
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 5.716475963592529
+                },
+                {
+                    "layer": "layer.28.self_attn.q_proj",
+                    "ppl": 5.716441631317139
+                },
+                {
+                    "layer": "layer.28.self_attn.k_proj",
+                    "ppl": 5.71636962890625
+                },
+                {
+                    "layer": "layer.28.self_attn.v_proj",
+                    "ppl": 5.7164306640625
+                },
+                {
+                    "layer": "layer.28.self_attn.o_proj",
+                    "ppl": 5.716403484344482
+                },
+                {
+                    "layer": "layer.28.mlp.gate_proj",
+                    "ppl": 5.716465950012207
+                },
+                {
+                    "layer": "layer.28.mlp.up_proj",
+                    "ppl": 5.716305732727051
+                },
+                {
+                    "layer": "layer.28.mlp.down_proj",
+                    "ppl": 5.716561794281006
+                },
+                {
+                    "layer": "layer.29.self_attn.q_proj",
+                    "ppl": 5.716446399688721
+                },
+                {
+                    "layer": "layer.29.self_attn.k_proj",
+                    "ppl": 5.7164483070373535
+                },
+                {
+                    "layer": "layer.29.self_attn.v_proj",
+                    "ppl": 5.7163310050964355
+                },
+                {
+                    "layer": "layer.29.self_attn.o_proj",
+                    "ppl": 5.716424465179443
+                },
+                {
+                    "layer": "layer.29.mlp.gate_proj",
+                    "ppl": 5.716359615325928
+                },
+                {
+                    "layer": "layer.29.mlp.up_proj",
+                    "ppl": 5.7164306640625
+                },
+                {
+                    "layer": "layer.29.mlp.down_proj",
+                    "ppl": 5.7163310050964355
+                },
+                {
+                    "layer": "layer.30.self_attn.q_proj",
+                    "ppl": 5.716396808624268
+                },
+                {
+                    "layer": "layer.30.self_attn.k_proj",
+                    "ppl": 5.71647834777832
+                },
+                {
+                    "layer": "layer.30.self_attn.v_proj",
+                    "ppl": 5.716632843017578
+                },
+                {
+                    "layer": "layer.30.self_attn.o_proj",
+                    "ppl": 5.716471195220947
+                },
+                {
+                    "layer": "layer.30.mlp.gate_proj",
+                    "ppl": 5.716372013092041
+                },
+                {
+                    "layer": "layer.30.mlp.up_proj",
+                    "ppl": 5.716257095336914
+                },
+                {
+                    "layer": "layer.30.mlp.down_proj",
+                    "ppl": 5.7162981033325195
+                },
+                {
+                    "layer": "layer.31.self_attn.q_proj",
+                    "ppl": 5.716318130493164
+                },
+                {
+                    "layer": "layer.31.self_attn.k_proj",
+                    "ppl": 5.716458320617676
+                },
+                {
+                    "layer": "layer.31.self_attn.v_proj",
+                    "ppl": 5.716487884521484
+                },
+                {
+                    "layer": "layer.31.self_attn.o_proj",
+                    "ppl": 5.716555595397949
+                },
+                {
+                    "layer": "layer.31.mlp.gate_proj",
+                    "ppl": 5.716307163238525
+                },
+                {
+                    "layer": "layer.31.mlp.up_proj",
+                    "ppl": 5.716017723083496
+                },
+                {
+                    "layer": "layer.31.mlp.down_proj",
+                    "ppl": 5.716212272644043
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file