feat: baseline + calibration script

.gitignore

@@ -0,0 +1,14 @@
+# Model weights - too large for git
+mistral-model/
+llama-model/
+
+# Python cache
+__pycache__/
+*.pyc
+*.pyo
+
+# Jupyter
+.ipynb_checkpoints/
+
+# OS
+.DS_Store

baseline.py

@@ -0,0 +1,83 @@
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import time, json, os, sys
+
+# ── config ──────────────────────────────────────────
+MODEL_NAME = sys.argv[1] if len(sys.argv) > 1 else "mistral-7b"
+MODEL_PATHS = {
+    "mistral-7b": "~/kv-hack/mistral-model",
+    "llama-3-8b": "~/kv-hack/llama-model",
+}
+model_path = os.path.expanduser(MODEL_PATHS[MODEL_NAME])
+results_dir = f"~/kv-hack/results/{MODEL_NAME}"
+os.makedirs(os.path.expanduser(results_dir), exist_ok=True)
+# ────────────────────────────────────────────────────
+
+print(f"Running baseline for: {MODEL_NAME}")
+print("Loading model...")
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = AutoModelForCausalLM.from_pretrained(
+    model_path,
+    dtype=torch.float16,
+    device_map="cuda"
+)
+model.eval()
+
+results = {}
+
+for ctx_len in [1024, 4096, 8192]:
+    print(f"\nTesting context length: {ctx_len}")
+    input_ids = torch.randint(1, 1000, (1, ctx_len)).cuda()
+
+    # warmup
+    with torch.no_grad():
+        for _ in range(2):
+            out = model(input_ids, use_cache=True)
+
+    torch.cuda.synchronize()
+    torch.cuda.reset_peak_memory_stats()
+
+    # measure
+    times = []
+    with torch.no_grad():
+        for _ in range(5):
+            t0 = time.time()
+            out = model(input_ids, use_cache=True)
+            torch.cuda.synchronize()
+            times.append(time.time() - t0)
+
+    peak_mem = torch.cuda.max_memory_allocated() / 1e9
+    avg_time = sum(times) / len(times)
+
+    results[ctx_len] = {
+        "peak_memory_gb": round(peak_mem, 2),
+        "avg_prefill_ms": round(avg_time * 1000, 1),
+    }
+    print(f"  Peak memory: {peak_mem:.2f} GB")
+    print(f"  Avg prefill: {avg_time*1000:.1f} ms")
+
+# decode speed
+print("\nTesting decode speed...")
+input_ids = torch.randint(1, 1000, (1, 512)).cuda()
+with torch.no_grad():
+    t0 = time.time()
+    out = model.generate(
+        input_ids,
+        max_new_tokens=100,
+        do_sample=False,
+        pad_token_id=tokenizer.eos_token_id
+    )
+    torch.cuda.synchronize()
+    elapsed = time.time() - t0
+
+tokens_per_sec = 100 / elapsed
+results["decode_tokens_per_sec"] = round(tokens_per_sec, 1)
+print(f"  Decode speed: {tokens_per_sec:.1f} tokens/sec")
+
+# save
+out_path = os.path.expanduser(f"{results_dir}/baseline.json")
+with open(out_path, "w") as f:
+    json.dump(results, f, indent=2)
+
+print(f"\n✅ Saved to {out_path}")
+print(json.dumps(results, indent=2))

calibrate.py

@@ -0,0 +1,145 @@
+import torch
+import json
+import os
+import sys
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from datasets import load_dataset
+from tqdm import tqdm
+
+# ── config ──────────────────────────────────────────
+MODEL_NAME = sys.argv[1] if len(sys.argv) > 1 else "mistral-7b"
+MODEL_PATHS = {
+    "mistral-7b": "~/kv-hack/mistral-model",
+    "llama-3-8b": "~/kv-hack/llama-model",
+}
+model_path = os.path.expanduser(MODEL_PATHS[MODEL_NAME])
+results_dir = os.path.expanduser(f"~/kv-hack/results/{MODEL_NAME}")
+os.makedirs(results_dir, exist_ok=True)
+# ────────────────────────────────────────────────────
+
+print(f"Running calibration for: {MODEL_NAME}")
+print("Loading model...")
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = AutoModelForCausalLM.from_pretrained(
+    model_path,
+    dtype=torch.float16,
+    device_map="cuda"
+)
+model.eval()
+
+# load calibration dataset
+print("Loading calibration data...")
+dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train")
+texts = [t for t in dataset["text"] if len(t.strip()) > 200][:256]
+
+def quantize_tensor(x, bits):
+    """Quantize tensor to given bits and dequantize back"""
+    if bits == 16:
+        return x
+    qmin, qmax = 0, 2**bits - 1
+    xmin = x.amin(dim=-1, keepdim=True)
+    xmax = x.amax(dim=-1, keepdim=True)
+    scale = (xmax - xmin).clamp(min=1e-8) / qmax
+    x_q = ((x - xmin) / scale).round().clamp(qmin, qmax)
+    return x_q * scale + xmin
+
+def get_kv_error(layer_idx, head_idx, bits, num_samples=32):
+    """Measure reconstruction error when quantizing a specific head's KV"""
+    errors = []
+
+    for text in texts[:num_samples]:
+        inputs = tokenizer(
+            text,
+            return_tensors="pt",
+            max_length=512,
+            truncation=True
+        ).to("cuda")
+
+        if inputs["input_ids"].shape[1] < 32:
+            continue
+
+        with torch.no_grad():
+            outputs = model(
+                **inputs,
+                output_attentions=False,
+                use_cache=True
+            )
+
+        kv_cache = outputs.past_key_values
+        k = kv_cache.layers[layer_idx].keys   # [1, heads, seq, head_dim]
+        v = kv_cache.layers[layer_idx].values
+
+        k_head = k[0, head_idx]
+        v_head = v[0, head_idx]
+
+        k_q = quantize_tensor(k_head, bits)
+        v_q = quantize_tensor(v_head, bits)
+
+        k_err = (k_head - k_q).pow(2).mean().item()
+        v_err = (v_head - v_q).pow(2).mean().item()
+        errors.append(k_err + v_err)
+
+    return sum(errors) / len(errors) if errors else float('inf')
+
+# get model dimensions
+print("Detecting model dimensions...")
+with torch.no_grad():
+    dummy = tokenizer("hello", return_tensors="pt").to("cuda")
+    out = model(**dummy, use_cache=True)
+    kv_cache = out.past_key_values
+    num_layers = len(kv_cache.layers)
+    num_heads = kv_cache.layers[0].keys.shape[1]
+
+print(f"num_layers: {num_layers}, num_heads: {num_heads}")
+
+
+print(f"Model: {num_layers} layers, {num_heads} heads per layer")
+print("Running per-head sensitivity analysis...")
+print("This will take ~15-20 minutes. Grab a coffee ☕")
+
+sensitivity_map = {}
+bit_allocation = {}
+
+for layer_idx in tqdm(range(num_layers), desc="Layers"):
+    sensitivity_map[layer_idx] = {}
+    bit_allocation[layer_idx] = {}
+
+    for head_idx in range(num_heads):
+        err_2bit = get_kv_error(layer_idx, head_idx, 2, num_samples=32)
+        err_4bit = get_kv_error(layer_idx, head_idx, 4, num_samples=32)
+        err_8bit = get_kv_error(layer_idx, head_idx, 8, num_samples=32)
+
+        sensitivity_map[layer_idx][head_idx] = {
+            "2bit": round(err_2bit, 6),
+            "4bit": round(err_4bit, 6),
+            "8bit": round(err_8bit, 6),
+        }
+
+        if err_2bit < 1e-4:
+            optimal_bits = 2
+        elif err_4bit < 1e-4:
+            optimal_bits = 4
+        else:
+            optimal_bits = 8
+
+        bit_allocation[layer_idx][head_idx] = optimal_bits
+
+# summary
+all_bits = [bit_allocation[l][h] for l in bit_allocation for h in bit_allocation[l]]
+avg_bits = sum(all_bits) / len(all_bits)
+dist = {2: all_bits.count(2), 4: all_bits.count(4), 8: all_bits.count(8)}
+compression = 16 / avg_bits
+
+print(f"\n✅ Calibration complete!")
+print(f"Bit distribution: {dist}")
+print(f"Average bits: {avg_bits:.2f}")
+print(f"Compression vs FP16: {compression:.1f}x")
+
+# save
+with open(f"{results_dir}/sensitivity_map.json", "w") as f:
+    json.dump(sensitivity_map, f, indent=2)
+
+with open(f"{results_dir}/bit_allocation.json", "w") as f:
+    json.dump(bit_allocation, f, indent=2)
+
+print(f"✅ Saved to {results_dir}/")

results/llama-3-8b/baseline.json

@@ -0,0 +1,15 @@
+{
+  "1024": {
+    "peak_memory_gb": 16.87,
+    "avg_prefill_ms": 89.1
+  },
+  "4096": {
+    "peak_memory_gb": 19.28,
+    "avg_prefill_ms": 352.0
+  },
+  "8192": {
+    "peak_memory_gb": 22.49,
+    "avg_prefill_ms": 738.8
+  },
+  "decode_tokens_per_sec": 35.2
+}

results/mistral-7b/baseline.json

@@ -0,0 +1,15 @@
+{
+  "1024": {
+    "peak_memory_gb": 14.96,
+    "avg_prefill_ms": 86.0
+  },
+  "4096": {
+    "peak_memory_gb": 16.34,
+    "avg_prefill_ms": 333.6
+  },
+  "8192": {
+    "peak_memory_gb": 18.17,
+    "avg_prefill_ms": 704.5
+  },
+  "decode_tokens_per_sec": 35.4
+}

results/mistral-7b/bit_allocation.json

@@ -0,0 +1,322 @@
+{
+  "0": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "1": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "2": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "3": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "4": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "5": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "6": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "7": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "8": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "9": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "10": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "11": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "12": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "13": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "14": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "15": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "16": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "17": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "18": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "19": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "20": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "21": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "22": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "23": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "24": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "25": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "26": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "27": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "28": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "29": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "30": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  },
+  "31": {
+    "0": 8,
+    "1": 8,
+    "2": 8,
+    "3": 8,
+    "4": 8,
+    "5": 8,
+    "6": 8,
+    "7": 8
+  }
+}

results/mistral-7b/sensitivity_map.json

@@ -0,0 +1,1346 @@
+{
+  "0": {
+    "0": {
+      "2bit": 1.275651,
+      "4bit": 0.050053,
+      "8bit": 0.000181
+    },
+    "1": {
+      "2bit": 0.875911,
+      "4bit": 0.022568,
+      "8bit": 8.4e-05
+    },
+    "2": {
+      "2bit": 0.352025,
+      "4bit": 0.016612,
+      "8bit": 6e-05
+    },
+    "3": {
+      "2bit": 0.206239,
+      "4bit": 0.009488,
+      "8bit": 3.4e-05
+    },
+    "4": {
+      "2bit": 0.122591,
+      "4bit": 0.00476,
+      "8bit": 1.7e-05
+    },
+    "5": {
+      "2bit": 0.303784,
+      "4bit": 0.01026,
+      "8bit": 3.8e-05
+    },
+    "6": {
+      "2bit": 0.861406,
+      "4bit": 0.034991,
+      "8bit": 0.000124
+    },
+    "7": {
+      "2bit": 1.169842,
+      "4bit": 0.041992,
+      "8bit": 0.000154
+    }
+  },
+  "1": {
+    "0": {
+      "2bit": 1.52893,
+      "4bit": 0.052165,
+      "8bit": 0.000179
+    },
+    "1": {
+      "2bit": 1.11751,
+      "4bit": 0.047545,
+      "8bit": 0.000174
+    },
+    "2": {
+      "2bit": 1.410196,
+      "4bit": 0.056624,
+      "8bit": 0.000205
+    },
+    "3": {
+      "2bit": 1.020669,
+      "4bit": 0.039215,
+      "8bit": 0.000141
+    },
+    "4": {
+      "2bit": 1.20992,
+      "4bit": 0.045675,
+      "8bit": 0.000166
+    },
+    "5": {
+      "2bit": 1.460292,
+      "4bit": 0.052388,
+      "8bit": 0.000193
+    },
+    "6": {
+      "2bit": 0.634232,
+      "4bit": 0.02644,
+      "8bit": 9.7e-05
+    },
+    "7": {
+      "2bit": 1.043682,
+      "4bit": 0.039064,
+      "8bit": 0.000141
+    }
+  },
+  "2": {
+    "0": {
+      "2bit": 2.299476,
+      "4bit": 0.078987,
+      "8bit": 0.000286
+    },
+    "1": {
+      "2bit": 1.363926,
+      "4bit": 0.081288,
+      "8bit": 0.000286
+    },
+    "2": {
+      "2bit": 2.503927,
+      "4bit": 0.091683,
+      "8bit": 0.00032
+    },
+    "3": {
+      "2bit": 0.807431,
+      "4bit": 0.039283,
+      "8bit": 0.000146
+    },
+    "4": {
+      "2bit": 1.176411,
+      "4bit": 0.044124,
+      "8bit": 0.00016
+    },
+    "5": {
+      "2bit": 1.832608,
+      "4bit": 0.068379,
+      "8bit": 0.000249
+    },
+    "6": {
+      "2bit": 1.473303,
+      "4bit": 0.096679,
+      "8bit": 0.00034
+    },
+    "7": {
+      "2bit": 2.977381,
+      "4bit": 0.096091,
+      "8bit": 0.000351
+    }
+  },
+  "3": {
+    "0": {
+      "2bit": 0.993681,
+      "4bit": 0.042615,
+      "8bit": 0.000156
+    },
+    "1": {
+      "2bit": 1.837716,
+      "4bit": 0.06231,
+      "8bit": 0.000225
+    },
+    "2": {
+      "2bit": 2.007418,
+      "4bit": 0.067538,
+      "8bit": 0.000242
+    },
+    "3": {
+      "2bit": 1.825185,
+      "4bit": 0.068191,
+      "8bit": 0.000251
+    },
+    "4": {
+      "2bit": 0.998271,
+      "4bit": 0.037708,
+      "8bit": 0.000138
+    },
+    "5": {
+      "2bit": 1.696072,
+      "4bit": 0.068816,
+      "8bit": 0.000259
+    },
+    "6": {
+      "2bit": 2.722304,
+      "4bit": 0.116425,
+      "8bit": 0.000427
+    },
+    "7": {
+      "2bit": 2.207015,
+      "4bit": 0.083942,
+      "8bit": 0.000299
+    }
+  },
+  "4": {
+    "0": {
+      "2bit": 0.911254,
+      "4bit": 0.034233,
+      "8bit": 0.000124
+    },
+    "1": {
+      "2bit": 1.669543,
+      "4bit": 0.074327,
+      "8bit": 0.000275
+    },
+    "2": {
+      "2bit": 2.616326,
+      "4bit": 0.083657,
+      "8bit": 0.000306
+    },
+    "3": {
+      "2bit": 1.392276,
+      "4bit": 0.051905,
+      "8bit": 0.000187
+    },
+    "4": {
+      "2bit": 1.398877,
+      "4bit": 0.060551,
+      "8bit": 0.000213
+    },
+    "5": {
+      "2bit": 1.775969,
+      "4bit": 0.097584,
+      "8bit": 0.000345
+    },
+    "6": {
+      "2bit": 0.735877,
+      "4bit": 0.030724,
+      "8bit": 0.00011
+    },
+    "7": {
+      "2bit": 1.986083,
+      "4bit": 0.093931,
+      "8bit": 0.000333
+    }
+  },
+  "5": {
+    "0": {
+      "2bit": 1.646835,
+      "4bit": 0.072494,
+      "8bit": 0.000269
+    },
+    "1": {
+      "2bit": 2.589399,
+      "4bit": 0.097729,
+      "8bit": 0.00035
+    },
+    "2": {
+      "2bit": 1.087101,
+      "4bit": 0.041702,
+      "8bit": 0.000152
+    },
+    "3": {
+      "2bit": 1.805192,
+      "4bit": 0.073288,
+      "8bit": 0.000257
+    },
+    "4": {
+      "2bit": 1.40546,
+      "4bit": 0.070294,
+      "8bit": 0.000262
+    },
+    "5": {
+      "2bit": 1.311666,
+      "4bit": 0.061274,
+      "8bit": 0.000216
+    },
+    "6": {
+      "2bit": 1.299382,
+      "4bit": 0.050681,
+      "8bit": 0.000185
+    },
+    "7": {
+      "2bit": 1.360648,
+      "4bit": 0.062991,
+      "8bit": 0.000235
+    }
+  },
+  "6": {
+    "0": {
+      "2bit": 2.172036,
+      "4bit": 0.075287,
+      "8bit": 0.000272
+    },
+    "1": {
+      "2bit": 3.144641,
+      "4bit": 0.092277,
+      "8bit": 0.000336
+    },
+    "2": {
+      "2bit": 1.608095,
+      "4bit": 0.082313,
+      "8bit": 0.000292
+    },
+    "3": {
+      "2bit": 2.546202,
+      "4bit": 0.082835,
+      "8bit": 0.000298
+    },
+    "4": {
+      "2bit": 2.563858,
+      "4bit": 0.077362,
+      "8bit": 0.000281
+    },
+    "5": {
+      "2bit": 1.45694,
+      "4bit": 0.064594,
+      "8bit": 0.00024
+    },
+    "6": {
+      "2bit": 1.372081,
+      "4bit": 0.057948,
+      "8bit": 0.000214
+    },
+    "7": {
+      "2bit": 2.494781,
+      "4bit": 0.16486,
+      "8bit": 0.000581
+    }
+  },
+  "7": {
+    "0": {
+      "2bit": 1.963428,
+      "4bit": 0.082925,
+      "8bit": 0.000291
+    },
+    "1": {
+      "2bit": 1.632993,
+      "4bit": 0.060651,
+      "8bit": 0.000221
+    },
+    "2": {
+      "2bit": 1.787988,
+      "4bit": 0.076841,
+      "8bit": 0.000288
+    },
+    "3": {
+      "2bit": 1.388928,
+      "4bit": 0.054667,
+      "8bit": 0.000197
+    },
+    "4": {
+      "2bit": 1.665172,
+      "4bit": 0.072744,
+      "8bit": 0.000259
+    },
+    "5": {
+      "2bit": 1.588525,
+      "4bit": 0.054458,
+      "8bit": 0.000197
+    },
+    "6": {
+      "2bit": 1.923881,
+      "4bit": 0.068322,
+      "8bit": 0.000251
+    },
+    "7": {
+      "2bit": 1.328365,
+      "4bit": 0.044822,
+      "8bit": 0.000163
+    }
+  },
+  "8": {
+    "0": {
+      "2bit": 2.403139,
+      "4bit": 0.126164,
+      "8bit": 0.000467
+    },
+    "1": {
+      "2bit": 2.082456,
+      "4bit": 0.098564,
+      "8bit": 0.000347
+    },
+    "2": {
+      "2bit": 1.387552,
+      "4bit": 0.069329,
+      "8bit": 0.000245
+    },
+    "3": {
+      "2bit": 1.672266,
+      "4bit": 0.064297,
+      "8bit": 0.000231
+    },
+    "4": {
+      "2bit": 1.023611,
+      "4bit": 0.04128,
+      "8bit": 0.000153
+    },
+    "5": {
+      "2bit": 1.653714,
+      "4bit": 0.065364,
+      "8bit": 0.000233
+    },
+    "6": {
+      "2bit": 2.049061,
+      "4bit": 0.069308,
+      "8bit": 0.000253
+    },
+    "7": {
+      "2bit": 1.781504,
+      "4bit": 0.08367,
+      "8bit": 0.000313
+    }
+  },
+  "9": {
+    "0": {
+      "2bit": 0.918524,
+      "4bit": 0.037062,
+      "8bit": 0.000137
+    },
+    "1": {
+      "2bit": 1.273425,
+      "4bit": 0.049196,
+      "8bit": 0.000178
+    },
+    "2": {
+      "2bit": 1.576742,
+      "4bit": 0.068121,
+      "8bit": 0.000253
+    },
+    "3": {
+      "2bit": 1.219413,
+      "4bit": 0.045919,
+      "8bit": 0.000166
+    },
+    "4": {
+      "2bit": 1.273967,
+      "4bit": 0.050365,
+      "8bit": 0.000185
+    },
+    "5": {
+      "2bit": 1.33001,
+      "4bit": 0.053997,
+      "8bit": 0.000193
+    },
+    "6": {
+      "2bit": 1.227499,
+      "4bit": 0.052792,
+      "8bit": 0.000187
+    },
+    "7": {
+      "2bit": 1.416939,
+      "4bit": 0.060307,
+      "8bit": 0.000223
+    }
+  },
+  "10": {
+    "0": {
+      "2bit": 1.412758,
+      "4bit": 0.051217,
+      "8bit": 0.000184
+    },
+    "1": {
+      "2bit": 1.113165,
+      "4bit": 0.048842,
+      "8bit": 0.00018
+    },
+    "2": {
+      "2bit": 1.711709,
+      "4bit": 0.071217,
+      "8bit": 0.000253
+    },
+    "3": {
+      "2bit": 2.097556,
+      "4bit": 0.113594,
+      "8bit": 0.0004
+    },
+    "4": {
+      "2bit": 1.393269,
+      "4bit": 0.049211,
+      "8bit": 0.000177
+    },
+    "5": {
+      "2bit": 1.650199,
+      "4bit": 0.0649,
+      "8bit": 0.000232
+    },
+    "6": {
+      "2bit": 2.387558,
+      "4bit": 0.100509,
+      "8bit": 0.000358
+    },
+    "7": {
+      "2bit": 2.05618,
+      "4bit": 0.078283,
+      "8bit": 0.000281
+    }
+  },
+  "11": {
+    "0": {
+      "2bit": 0.78626,
+      "4bit": 0.029687,
+      "8bit": 0.000107
+    },
+    "1": {
+      "2bit": 1.489031,
+      "4bit": 0.068943,
+      "8bit": 0.000258
+    },
+    "2": {
+      "2bit": 1.061362,
+      "4bit": 0.046822,
+      "8bit": 0.000174
+    },
+    "3": {
+      "2bit": 1.445414,
+      "4bit": 0.05927,
+      "8bit": 0.000209
+    },
+    "4": {
+      "2bit": 2.216359,
+      "4bit": 0.074722,
+      "8bit": 0.00027
+    },
+    "5": {
+      "2bit": 2.193741,
+      "4bit": 0.079334,
+      "8bit": 0.000287
+    },
+    "6": {
+      "2bit": 2.017558,
+      "4bit": 0.080408,
+      "8bit": 0.000295
+    },
+    "7": {
+      "2bit": 1.906577,
+      "4bit": 0.077715,
+      "8bit": 0.000283
+    }
+  },
+  "12": {
+    "0": {
+      "2bit": 1.749719,
+      "4bit": 0.087972,
+      "8bit": 0.00031
+    },
+    "1": {
+      "2bit": 2.169801,
+      "4bit": 0.07679,
+      "8bit": 0.000275
+    },
+    "2": {
+      "2bit": 2.21439,
+      "4bit": 0.091871,
+      "8bit": 0.000335
+    },
+    "3": {
+      "2bit": 1.476136,
+      "4bit": 0.07285,
+      "8bit": 0.000258
+    },
+    "4": {
+      "2bit": 1.452321,
+      "4bit": 0.056441,
+      "8bit": 0.000206
+    },
+    "5": {
+      "2bit": 1.342228,
+      "4bit": 0.055908,
+      "8bit": 0.0002
+    },
+    "6": {
+      "2bit": 1.544378,
+      "4bit": 0.061596,
+      "8bit": 0.000223
+    },
+    "7": {
+      "2bit": 1.050014,
+      "4bit": 0.04722,
+      "8bit": 0.000168
+    }
+  },
+  "13": {
+    "0": {
+      "2bit": 1.624628,
+      "4bit": 0.085572,
+      "8bit": 0.000302
+    },
+    "1": {
+      "2bit": 1.997908,
+      "4bit": 0.087123,
+      "8bit": 0.000323
+    },
+    "2": {
+      "2bit": 1.999066,
+      "4bit": 0.087278,
+      "8bit": 0.000322
+    },
+    "3": {
+      "2bit": 1.395491,
+      "4bit": 0.061202,
+      "8bit": 0.000226
+    },
+    "4": {
+      "2bit": 2.343725,
+      "4bit": 0.097322,
+      "8bit": 0.000343
+    },
+    "5": {
+      "2bit": 1.710008,
+      "4bit": 0.081722,
+      "8bit": 0.000288
+    },
+    "6": {
+      "2bit": 1.815651,
+      "4bit": 0.069256,
+      "8bit": 0.000252
+    },
+    "7": {
+      "2bit": 2.440488,
+      "4bit": 0.116316,
+      "8bit": 0.000408
+    }
+  },
+  "14": {
+    "0": {
+      "2bit": 1.322071,
+      "4bit": 0.054413,
+      "8bit": 0.000201
+    },
+    "1": {
+      "2bit": 1.034706,
+      "4bit": 0.039935,
+      "8bit": 0.000146
+    },
+    "2": {
+      "2bit": 1.570443,
+      "4bit": 0.061607,
+      "8bit": 0.000226
+    },
+    "3": {
+      "2bit": 1.920502,
+      "4bit": 0.072978,
+      "8bit": 0.000267
+    },
+    "4": {
+      "2bit": 1.107241,
+      "4bit": 0.046085,
+      "8bit": 0.000171
+    },
+    "5": {
+      "2bit": 1.736773,
+      "4bit": 0.067422,
+      "8bit": 0.000243
+    },
+    "6": {
+      "2bit": 1.745196,
+      "4bit": 0.072617,
+      "8bit": 0.000259
+    },
+    "7": {
+      "2bit": 0.986826,
+      "4bit": 0.041658,
+      "8bit": 0.000148
+    }
+  },
+  "15": {
+    "0": {
+      "2bit": 1.314292,
+      "4bit": 0.053463,
+      "8bit": 0.000197
+    },
+    "1": {
+      "2bit": 1.34888,
+      "4bit": 0.060297,
+      "8bit": 0.000214
+    },
+    "2": {
+      "2bit": 1.837393,
+      "4bit": 0.068633,
+      "8bit": 0.00025
+    },
+    "3": {
+      "2bit": 1.82893,
+      "4bit": 0.086975,
+      "8bit": 0.000323
+    },
+    "4": {
+      "2bit": 1.82945,
+      "4bit": 0.066882,
+      "8bit": 0.000241
+    },
+    "5": {
+      "2bit": 1.514484,
+      "4bit": 0.064664,
+      "8bit": 0.000239
+    },
+    "6": {
+      "2bit": 1.492166,
+      "4bit": 0.061424,
+      "8bit": 0.00022
+    },
+    "7": {
+      "2bit": 1.865242,
+      "4bit": 0.075858,
+      "8bit": 0.000274
+    }
+  },
+  "16": {
+    "0": {
+      "2bit": 1.669918,
+      "4bit": 0.082674,
+      "8bit": 0.000292
+    },
+    "1": {
+      "2bit": 1.574098,
+      "4bit": 0.062568,
+      "8bit": 0.000225
+    },
+    "2": {
+      "2bit": 1.468039,
+      "4bit": 0.061834,
+      "8bit": 0.00022
+    },
+    "3": {
+      "2bit": 1.782679,
+      "4bit": 0.073019,
+      "8bit": 0.00026
+    },
+    "4": {
+      "2bit": 1.308494,
+      "4bit": 0.054399,
+      "8bit": 0.000195
+    },
+    "5": {
+      "2bit": 1.680173,
+      "4bit": 0.064292,
+      "8bit": 0.000234
+    },
+    "6": {
+      "2bit": 1.309707,
+      "4bit": 0.054918,
+      "8bit": 0.000202
+    },
+    "7": {
+      "2bit": 1.601349,
+      "4bit": 0.069984,
+      "8bit": 0.000248
+    }
+  },
+  "17": {
+    "0": {
+      "2bit": 1.968627,
+      "4bit": 0.086298,
+      "8bit": 0.00032
+    },
+    "1": {
+      "2bit": 1.41837,
+      "4bit": 0.066183,
+      "8bit": 0.000235
+    },
+    "2": {
+      "2bit": 1.751224,
+      "4bit": 0.091822,
+      "8bit": 0.000325
+    },
+    "3": {
+      "2bit": 1.66518,
+      "4bit": 0.08592,
+      "8bit": 0.00032
+    },
+    "4": {
+      "2bit": 2.6849,
+      "4bit": 0.095523,
+      "8bit": 0.000344
+    },
+    "5": {
+      "2bit": 2.200622,
+      "4bit": 0.090865,
+      "8bit": 0.000325
+    },
+    "6": {
+      "2bit": 1.405657,
+      "4bit": 0.060244,
+      "8bit": 0.000215
+    },
+    "7": {
+      "2bit": 1.097151,
+      "4bit": 0.042971,
+      "8bit": 0.000156
+    }
+  },
+  "18": {
+    "0": {
+      "2bit": 2.010584,
+      "4bit": 0.078786,
+      "8bit": 0.000289
+    },
+    "1": {
+      "2bit": 1.670967,
+      "4bit": 0.079038,
+      "8bit": 0.000279
+    },
+    "2": {
+      "2bit": 1.396168,
+      "4bit": 0.062601,
+      "8bit": 0.000223
+    },
+    "3": {
+      "2bit": 1.462345,
+      "4bit": 0.055801,
+      "8bit": 0.000204
+    },
+    "4": {
+      "2bit": 1.904111,
+      "4bit": 0.071439,
+      "8bit": 0.000257
+    },
+    "5": {
+      "2bit": 1.329314,
+      "4bit": 0.052623,
+      "8bit": 0.000189
+    },
+    "6": {
+      "2bit": 1.718576,
+      "4bit": 0.093508,
+      "8bit": 0.000331
+    },
+    "7": {
+      "2bit": 1.933535,
+      "4bit": 0.086362,
+      "8bit": 0.000318
+    }
+  },
+  "19": {
+    "0": {
+      "2bit": 1.534418,
+      "4bit": 0.081535,
+      "8bit": 0.000304
+    },
+    "1": {
+      "2bit": 1.857097,
+      "4bit": 0.067898,
+      "8bit": 0.000247
+    },
+    "2": {
+      "2bit": 2.240644,
+      "4bit": 0.088436,
+      "8bit": 0.000316
+    },
+    "3": {
+      "2bit": 1.620667,
+      "4bit": 0.067017,
+      "8bit": 0.000239
+    },
+    "4": {
+      "2bit": 1.589832,
+      "4bit": 0.060374,
+      "8bit": 0.000216
+    },
+    "5": {
+      "2bit": 3.267183,
+      "4bit": 0.105293,
+      "8bit": 0.000391
+    },
+    "6": {
+      "2bit": 1.32155,
+      "4bit": 0.065992,
+      "8bit": 0.000247
+    },
+    "7": {
+      "2bit": 1.600902,
+      "4bit": 0.07471,
+      "8bit": 0.000278
+    }
+  },
+  "20": {
+    "0": {
+      "2bit": 1.816447,
+      "4bit": 0.070234,
+      "8bit": 0.000253
+    },
+    "1": {
+      "2bit": 1.978655,
+      "4bit": 0.082553,
+      "8bit": 0.000308
+    },
+    "2": {
+      "2bit": 1.260029,
+      "4bit": 0.055176,
+      "8bit": 0.000196
+    },
+    "3": {
+      "2bit": 1.904745,
+      "4bit": 0.069508,
+      "8bit": 0.000253
+    },
+    "4": {
+      "2bit": 1.5048,
+      "4bit": 0.061951,
+      "8bit": 0.000229
+    },
+    "5": {
+      "2bit": 1.57513,
+      "4bit": 0.074261,
+      "8bit": 0.000262
+    },
+    "6": {
+      "2bit": 1.69522,
+      "4bit": 0.065696,
+      "8bit": 0.000238
+    },
+    "7": {
+      "2bit": 1.816767,
+      "4bit": 0.068307,
+      "8bit": 0.00025
+    }
+  },
+  "21": {
+    "0": {
+      "2bit": 1.153863,
+      "4bit": 0.040871,
+      "8bit": 0.000149
+    },
+    "1": {
+      "2bit": 1.814577,
+      "4bit": 0.073578,
+      "8bit": 0.000263
+    },
+    "2": {
+      "2bit": 1.362879,
+      "4bit": 0.051833,
+      "8bit": 0.000188
+    },
+    "3": {
+      "2bit": 1.166707,
+      "4bit": 0.04918,
+      "8bit": 0.000176
+    },
+    "4": {
+      "2bit": 1.296553,
+      "4bit": 0.052385,
+      "8bit": 0.000193
+    },
+    "5": {
+      "2bit": 3.967201,
+      "4bit": 0.124592,
+      "8bit": 0.000468
+    },
+    "6": {
+      "2bit": 1.52414,
+      "4bit": 0.085691,
+      "8bit": 0.000302
+    },
+    "7": {
+      "2bit": 2.410778,
+      "4bit": 0.109507,
+      "8bit": 0.000407
+    }
+  },
+  "22": {
+    "0": {
+      "2bit": 0.747227,
+      "4bit": 0.030391,
+      "8bit": 0.000108
+    },
+    "1": {
+      "2bit": 0.910305,
+      "4bit": 0.036248,
+      "8bit": 0.000133
+    },
+    "2": {
+      "2bit": 1.360477,
+      "4bit": 0.051683,
+      "8bit": 0.000188
+    },
+    "3": {
+      "2bit": 1.91634,
+      "4bit": 0.124982,
+      "8bit": 0.000463
+    },
+    "4": {
+      "2bit": 0.754953,
+      "4bit": 0.028669,
+      "8bit": 0.000104
+    },
+    "5": {
+      "2bit": 1.513672,
+      "4bit": 0.077863,
+      "8bit": 0.000274
+    },
+    "6": {
+      "2bit": 1.416754,
+      "4bit": 0.065633,
+      "8bit": 0.000246
+    },
+    "7": {
+      "2bit": 1.103243,
+      "4bit": 0.049249,
+      "8bit": 0.000183
+    }
+  },
+  "23": {
+    "0": {
+      "2bit": 1.724428,
+      "4bit": 0.085561,
+      "8bit": 0.000318
+    },
+    "1": {
+      "2bit": 1.592232,
+      "4bit": 0.064306,
+      "8bit": 0.000226
+    },
+    "2": {
+      "2bit": 1.310122,
+      "4bit": 0.049167,
+      "8bit": 0.000177
+    },
+    "3": {
+      "2bit": 0.951035,
+      "4bit": 0.034097,
+      "8bit": 0.000123
+    },
+    "4": {
+      "2bit": 1.758739,
+      "4bit": 0.083453,
+      "8bit": 0.000311
+    },
+    "5": {
+      "2bit": 1.647038,
+      "4bit": 0.061161,
+      "8bit": 0.00022
+    },
+    "6": {
+      "2bit": 2.10231,
+      "4bit": 0.104304,
+      "8bit": 0.000387
+    },
+    "7": {
+      "2bit": 2.364744,
+      "4bit": 0.094391,
+      "8bit": 0.00033
+    }
+  },
+  "24": {
+    "0": {
+      "2bit": 1.151069,
+      "4bit": 0.050224,
+      "8bit": 0.000185
+    },
+    "1": {
+      "2bit": 1.123055,
+      "4bit": 0.041769,
+      "8bit": 0.000152
+    },
+    "2": {
+      "2bit": 1.196377,
+      "4bit": 0.054532,
+      "8bit": 0.000202
+    },
+    "3": {
+      "2bit": 1.314301,
+      "4bit": 0.048999,
+      "8bit": 0.000179
+    },
+    "4": {
+      "2bit": 1.047907,
+      "4bit": 0.046386,
+      "8bit": 0.000173
+    },
+    "5": {
+      "2bit": 1.679045,
+      "4bit": 0.076513,
+      "8bit": 0.000271
+    },
+    "6": {
+      "2bit": 1.21999,
+      "4bit": 0.045077,
+      "8bit": 0.000163
+    },
+    "7": {
+      "2bit": 2.599075,
+      "4bit": 0.104117,
+      "8bit": 0.000389
+    }
+  },
+  "25": {
+    "0": {
+      "2bit": 1.472107,
+      "4bit": 0.063345,
+      "8bit": 0.000236
+    },
+    "1": {
+      "2bit": 1.502398,
+      "4bit": 0.069731,
+      "8bit": 0.00026
+    },
+    "2": {
+      "2bit": 1.57387,
+      "4bit": 0.059204,
+      "8bit": 0.000216
+    },
+    "3": {
+      "2bit": 1.629566,
+      "4bit": 0.068947,
+      "8bit": 0.000246
+    },
+    "4": {
+      "2bit": 0.910637,
+      "4bit": 0.03775,
+      "8bit": 0.000135
+    },
+    "5": {
+      "2bit": 1.551672,
+      "4bit": 0.061317,
+      "8bit": 0.00023
+    },
+    "6": {
+      "2bit": 1.11796,
+      "4bit": 0.048358,
+      "8bit": 0.000172
+    },
+    "7": {
+      "2bit": 1.577454,
+      "4bit": 0.067808,
+      "8bit": 0.000252
+    }
+  },
+  "26": {
+    "0": {
+      "2bit": 1.231514,
+      "4bit": 0.05502,
+      "8bit": 0.000203
+    },
+    "1": {
+      "2bit": 1.106922,
+      "4bit": 0.042905,
+      "8bit": 0.000157
+    },
+    "2": {
+      "2bit": 1.269325,
+      "4bit": 0.050961,
+      "8bit": 0.00018
+    },
+    "3": {
+      "2bit": 0.933496,
+      "4bit": 0.036494,
+      "8bit": 0.000133
+    },
+    "4": {
+      "2bit": 0.801933,
+      "4bit": 0.030255,
+      "8bit": 0.00011
+    },
+    "5": {
+      "2bit": 1.335897,
+      "4bit": 0.06658,
+      "8bit": 0.000235
+    },
+    "6": {
+      "2bit": 1.621925,
+      "4bit": 0.069036,
+      "8bit": 0.000244
+    },
+    "7": {
+      "2bit": 1.989624,
+      "4bit": 0.084026,
+      "8bit": 0.000309
+    }
+  },
+  "27": {
+    "0": {
+      "2bit": 2.556004,
+      "4bit": 0.114416,
+      "8bit": 0.000404
+    },
+    "1": {
+      "2bit": 1.349316,
+      "4bit": 0.05246,
+      "8bit": 0.000192
+    },
+    "2": {
+      "2bit": 3.095371,
+      "4bit": 0.131381,
+      "8bit": 0.000488
+    },
+    "3": {
+      "2bit": 1.425734,
+      "4bit": 0.052843,
+      "8bit": 0.000192
+    },
+    "4": {
+      "2bit": 1.196075,
+      "4bit": 0.049564,
+      "8bit": 0.000177
+    },
+    "5": {
+      "2bit": 2.229198,
+      "4bit": 0.096098,
+      "8bit": 0.000337
+    },
+    "6": {
+      "2bit": 0.938061,
+      "4bit": 0.035635,
+      "8bit": 0.00013
+    },
+    "7": {
+      "2bit": 1.25318,
+      "4bit": 0.04978,
+      "8bit": 0.000183
+    }
+  },
+  "28": {
+    "0": {
+      "2bit": 1.667999,
+      "4bit": 0.075404,
+      "8bit": 0.000266
+    },
+    "1": {
+      "2bit": 1.987015,
+      "4bit": 0.079003,
+      "8bit": 0.000283
+    },
+    "2": {
+      "2bit": 1.484013,
+      "4bit": 0.063911,
+      "8bit": 0.000225
+    },
+    "3": {
+      "2bit": 1.981274,
+      "4bit": 0.072937,
+      "8bit": 0.000261
+    },
+    "4": {
+      "2bit": 0.918701,
+      "4bit": 0.040401,
+      "8bit": 0.000144
+    },
+    "5": {
+      "2bit": 0.945871,
+      "4bit": 0.038321,
+      "8bit": 0.000141
+    },
+    "6": {
+      "2bit": 0.933807,
+      "4bit": 0.035623,
+      "8bit": 0.00013
+    },
+    "7": {
+      "2bit": 1.214512,
+      "4bit": 0.048709,
+      "8bit": 0.000179
+    }
+  },
+  "29": {
+    "0": {
+      "2bit": 1.492014,
+      "4bit": 0.066029,
+      "8bit": 0.000245
+    },
+    "1": {
+      "2bit": 3.013687,
+      "4bit": 0.143306,
+      "8bit": 0.000532
+    },
+    "2": {
+      "2bit": 1.229637,
+      "4bit": 0.047642,
+      "8bit": 0.000172
+    },
+    "3": {
+      "2bit": 1.523476,
+      "4bit": 0.067737,
+      "8bit": 0.000252
+    },
+    "4": {
+      "2bit": 1.546013,
+      "4bit": 0.060067,
+      "8bit": 0.00022
+    },
+    "5": {
+      "2bit": 1.292553,
+      "4bit": 0.058714,
+      "8bit": 0.000208
+    },
+    "6": {
+      "2bit": 1.107986,
+      "4bit": 0.044617,
+      "8bit": 0.000159
+    },
+    "7": {
+      "2bit": 2.442818,
+      "4bit": 0.089641,
+      "8bit": 0.000322
+    }
+  },
+  "30": {
+    "0": {
+      "2bit": 1.299538,
+      "4bit": 0.047529,
+      "8bit": 0.000172
+    },
+    "1": {
+      "2bit": 0.89185,
+      "4bit": 0.038019,
+      "8bit": 0.000136
+    },
+    "2": {
+      "2bit": 1.310513,
+      "4bit": 0.053666,
+      "8bit": 0.000197
+    },
+    "3": {
+      "2bit": 1.010298,
+      "4bit": 0.04109,
+      "8bit": 0.000151
+    },
+    "4": {
+      "2bit": 1.146852,
+      "4bit": 0.043392,
+      "8bit": 0.000158
+    },
+    "5": {
+      "2bit": 2.16995,
+      "4bit": 0.080771,
+      "8bit": 0.000284
+    },
+    "6": {
+      "2bit": 1.234642,
+      "4bit": 0.045607,
+      "8bit": 0.000166
+    },
+    "7": {
+      "2bit": 1.553858,
+      "4bit": 0.061193,
+      "8bit": 0.000218
+    }
+  },
+  "31": {
+    "0": {
+      "2bit": 0.85825,
+      "4bit": 0.037179,
+      "8bit": 0.000137
+    },
+    "1": {
+      "2bit": 1.297428,
+      "4bit": 0.052789,
+      "8bit": 0.00019
+    },
+    "2": {
+      "2bit": 1.489906,
+      "4bit": 0.05656,
+      "8bit": 0.000206
+    },
+    "3": {
+      "2bit": 1.519638,
+      "4bit": 0.057934,
+      "8bit": 0.000207
+    },
+    "4": {
+      "2bit": 1.264908,
+      "4bit": 0.055957,
+      "8bit": 0.000205
+    },
+    "5": {
+      "2bit": 2.053621,
+      "4bit": 0.077441,
+      "8bit": 0.00029
+    },
+    "6": {
+      "2bit": 1.156225,
+      "4bit": 0.044922,
+      "8bit": 0.000161
+    },
+    "7": {
+      "2bit": 1.146229,
+      "4bit": 0.042921,
+      "8bit": 0.000156
+    }
+  }
+}