Add throughput estimates, architecture details, export feature, and tips guide

New features:
- Tokens/second throughput estimates based on GPU memory bandwidth
- Model architecture details table (hidden size, heads, GQA ratio, vocab, max context)
- Context length scaling table showing KV cache at different lengths
- Export Report button to generate shareable text reports
- Tips & Guide tab with VRAM rules of thumb and recommended setups
- GPU bandwidth specs for all hardware (Consumer, Apple Silicon, Cloud)
- Best value AND fastest GPU recommendations
- Quantization quality indicators
- More example models (Hermes, Yi)
- Cross-model comparison set (Llama vs Mistral vs Qwen vs Gemma)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

app.py

@@ -8,39 +8,39 @@ from functools import lru_cache
 
 api = HfApi()
 
-# Consumer GPUs (no hourly cost)
+# Consumer GPUs: (VRAM GB, Memory Bandwidth GB/s)
 CONSUMER_GPUS = {
-    "RTX 3080": 10,
-    "RTX 3080 Ti": 12,
-    "RTX 3090": 24,
-    "RTX 3090 Ti": 24,
-    "RTX 4080": 16,
-    "RTX 4080 Super": 16,
-    "RTX 4090": 24,
-    "RTX 5090": 32,
+    "RTX 3080": (10, 760),
+    "RTX 3080 Ti": (12, 912),
+    "RTX 3090": (24, 936),
+    "RTX 3090 Ti": (24, 1008),
+    "RTX 4080": (16, 717),
+    "RTX 4080 Super": (16, 736),
+    "RTX 4090": (24, 1008),
+    "RTX 5090": (32, 1792),
 }
 
-# Apple Silicon (no hourly cost)
+# Apple Silicon: (Unified Memory GB, Memory Bandwidth GB/s)
 APPLE_GPUS = {
-    "M1 Max": 64,
-    "M2 Max": 96,
-    "M2 Ultra": 192,
-    "M3 Max": 128,
-    "M4 Max": 128,
+    "M1 Max": (64, 400),
+    "M2 Max": (96, 400),
+    "M2 Ultra": (192, 800),
+    "M3 Max": (128, 400),
+    "M4 Max": (128, 546),
 }
 
-# Cloud/Datacenter GPUs (with hourly costs from major providers)
+# Cloud/Datacenter GPUs: (VRAM GB, $/hr, Memory Bandwidth GB/s)
 CLOUD_GPUS = {
-    "T4": (16, 0.35),
-    "L4": (24, 0.70),
-    "A10G": (24, 1.00),
-    "RTX A5000": (24, 0.80),
-    "RTX A6000": (48, 1.50),
-    "L40S": (48, 1.20),
-    "A100 40GB": (40, 3.00),
-    "A100 80GB": (80, 5.00),
-    "H100 80GB": (80, 8.00),
-    "H100 NVL": (94, 10.00),
+    "T4": (16, 0.35, 320),
+    "L4": (24, 0.70, 300),
+    "A10G": (24, 1.00, 600),
+    "RTX A5000": (24, 0.80, 768),
+    "RTX A6000": (48, 1.50, 768),
+    "L40S": (48, 1.20, 864),
+    "A100 40GB": (40, 3.00, 1555),
+    "A100 80GB": (80, 5.00, 2039),
+    "H100 80GB": (80, 8.00, 3350),
+    "H100 NVL": (94, 10.00, 3938),
 }
 
 DTYPE_BYTES = {
@@ -59,15 +59,6 @@ FRAMEWORKS = {
     "Ollama": 1.08,
 }
 
-CONTEXT_PRESETS = {
-    "2K (fast chat)": 2048,
-    "4K (standard)": 4096,
-    "8K (extended)": 8192,
-    "16K (long docs)": 16384,
-    "32K (very long)": 32768,
-    "128K (full context)": 131072,
-}
-
 
 def bytes_to_gb(b):
     return b / (1024 ** 3)
@@ -101,6 +92,18 @@ def get_params(info):
     return 0, "F16"
 
 
+def estimate_throughput(params, bandwidth_gbs, batch_size, dtype_bytes):
+    """Estimate tokens/second based on memory bandwidth (rough approximation)"""
+    # Simplified: tok/s ~ bandwidth / (params * dtype_bytes / batch_size)
+    # This is a rough estimate; actual throughput depends on many factors
+    model_gb = (params * dtype_bytes) / (1024**3)
+    if model_gb == 0:
+        return 0
+    # Rough heuristic: memory-bound inference
+    tokens_per_sec = (bandwidth_gbs / model_gb) * batch_size * 0.5  # 50% efficiency factor
+    return max(1, int(tokens_per_sec))
+
+
 def calculate(model_id, context, batch, mode, framework, num_gpus, lora_rank):
     """Main calculation function"""
     try:
@@ -128,10 +131,14 @@ def calculate(model_id, context, batch, mode, framework, num_gpus, lora_rank):
         layers = config.get("num_hidden_layers", config.get("n_layer", 32))
         kv_heads = config.get("num_key_value_heads", config.get("num_attention_heads", 32))
         head_dim = config.get("head_dim", 128)
+        hidden_size = config.get("hidden_size", 4096)
+        num_heads = config.get("num_attention_heads", 32)
+        vocab_size = config.get("vocab_size", 32000)
+        intermediate_size = config.get("intermediate_size", hidden_size * 4)
+        max_position = config.get("max_position_embeddings", 4096)
+
         if not head_dim:
-            hidden = config.get("hidden_size", 4096)
-            heads = config.get("num_attention_heads", 32)
-            head_dim = hidden // heads if heads else 128
+            head_dim = hidden_size // num_heads if num_heads else 128
 
         kv_bytes = 2 * layers * batch * context * kv_heads * head_dim * dtype_bytes
         kv_gb = bytes_to_gb(kv_bytes)
@@ -141,6 +148,20 @@ def calculate(model_id, context, batch, mode, framework, num_gpus, lora_rank):
         out.append("**" + str(round(params_b, 1)) + "B parameters** | " + dtype + " | " + str(layers) + " layers")
         out.append("")
 
+        # Architecture details
+        out.append("### Model Architecture")
+        out.append("| Property | Value |")
+        out.append("|----------|-------|")
+        out.append("| Hidden Size | " + str(hidden_size) + " |")
+        out.append("| Attention Heads | " + str(num_heads) + " |")
+        out.append("| KV Heads (GQA) | " + str(kv_heads) + " |")
+        out.append("| Layers | " + str(layers) + " |")
+        out.append("| Vocab Size | " + str(vocab_size) + " |")
+        out.append("| Max Context | " + str(max_position) + " |")
+        if kv_heads != num_heads:
+            out.append("| GQA Ratio | " + str(num_heads) + ":" + str(kv_heads) + " |")
+        out.append("")
+
         if mode == "Training (Full)":
             grad_gb = weights_gb
             opt_gb = bytes_to_gb(params * 8)
@@ -199,46 +220,56 @@ def calculate(model_id, context, batch, mode, framework, num_gpus, lora_rank):
         out.append("")
         out.append("## Total Required: " + str(round(total, 1)) + " GB")
 
-        # Consumer GPUs section with colors
+        # Consumer GPUs section with colors and throughput
         out.append("")
         out.append("### Consumer GPUs")
-        out.append("| GPU | VRAM | Status | Headroom |")
-        out.append("|-----|------|--------|----------|")
-        for gpu, vram in CONSUMER_GPUS.items():
+        out.append("| GPU | VRAM | Status | Headroom | Est. tok/s |")
+        out.append("|-----|------|--------|----------|------------|")
+        for gpu, (vram, bandwidth) in CONSUMER_GPUS.items():
             hr = vram - effective
             if hr >= 2:
                 status = "🟢 Good fit"
             elif hr >= 0:
-                status = "🟡 Tight fit"
+                status = "🟡 Tight"
             else:
-                status = "🔴 Too small"
+                status = "🔴 No"
             sign = "+" if hr >= 0 else ""
-            out.append("| " + gpu + " | " + str(vram) + "GB | " + status + " | " + sign + str(round(hr, 1)) + "GB |")
+            if hr >= 0 and mode == "Inference":
+                tps = estimate_throughput(params, bandwidth, batch, dtype_bytes)
+                tps_str = str(tps)
+            else:
+                tps_str = "-"
+            out.append("| " + gpu + " | " + str(vram) + "GB | " + status + " | " + sign + str(round(hr, 1)) + "GB | " + tps_str + " |")
 
         # Apple Silicon section
         out.append("")
         out.append("### Apple Silicon (Unified Memory)")
-        out.append("| Chip | Memory | Status | Headroom |")
-        out.append("|------|--------|--------|----------|")
-        for gpu, vram in APPLE_GPUS.items():
+        out.append("| Chip | Memory | Status | Headroom | Est. tok/s |")
+        out.append("|------|--------|--------|----------|------------|")
+        for gpu, (vram, bandwidth) in APPLE_GPUS.items():
             hr = vram - effective
             if hr >= 10:
                 status = "🟢 Excellent"
             elif hr >= 0:
                 status = "🟡 Usable"
             else:
-                status = "🔴 Too small"
+                status = "🔴 No"
             sign = "+" if hr >= 0 else ""
-            out.append("| " + gpu + " | " + str(vram) + "GB | " + status + " | " + sign + str(round(hr, 1)) + "GB |")
+            if hr >= 0 and mode == "Inference":
+                tps = estimate_throughput(params, bandwidth, batch, dtype_bytes)
+                tps_str = str(tps)
+            else:
+                tps_str = "-"
+            out.append("| " + gpu + " | " + str(vram) + "GB | " + status + " | " + sign + str(round(hr, 1)) + "GB | " + tps_str + " |")
 
-        # Cloud GPUs section with costs
+        # Cloud GPUs section with costs and throughput
         out.append("")
         out.append("### Cloud GPU Options")
-        out.append("| GPU | VRAM | Status | $/hour | $/day (8hr) | $/month |")
-        out.append("|-----|------|--------|--------|-------------|---------|")
+        out.append("| GPU | VRAM | Status | $/hour | $/day | Est. tok/s |")
+        out.append("|-----|------|--------|--------|-------|------------|")
 
         cloud_options = []
-        for gpu, (vram, cost) in CLOUD_GPUS.items():
+        for gpu, (vram, cost, bandwidth) in CLOUD_GPUS.items():
             hr = vram - effective
             if hr >= 2:
                 status = "🟢 Good"
@@ -247,34 +278,65 @@ def calculate(model_id, context, batch, mode, framework, num_gpus, lora_rank):
             else:
                 status = "🔴 No"
             daily = cost * 8
-            monthly = cost * 176  # 22 days * 8 hours
-            cloud_options.append((gpu, vram, hr, status, cost, daily, monthly))
+            if hr >= 0 and mode == "Inference":
+                tps = estimate_throughput(params, bandwidth, batch, dtype_bytes)
+            else:
+                tps = 0
+            cloud_options.append((gpu, vram, hr, status, cost, daily, bandwidth, tps))
 
         # Sort by cost for those that fit
         cloud_options.sort(key=lambda x: (x[2] < 0, x[4]))
 
-        for gpu, vram, hr, status, cost, daily, monthly in cloud_options:
+        for gpu, vram, hr, status, cost, daily, bandwidth, tps in cloud_options:
             sign = "+" if hr >= 0 else ""
-            out.append("| " + gpu + " | " + str(vram) + "GB | " + status + " | $" + str(round(cost, 2)) + " | $" + str(round(daily, 2)) + " | $" + str(int(monthly)) + " |")
+            tps_str = str(tps) if tps > 0 else "-"
+            out.append("| " + gpu + " | " + str(vram) + "GB | " + status + " | $" + str(round(cost, 2)) + " | $" + str(round(daily, 2)) + " | " + tps_str + " |")
 
         # Best value recommendation
-        fitting_gpus = [(gpu, cost) for gpu, (vram, cost) in CLOUD_GPUS.items() if vram >= effective]
+        fitting_gpus = [(gpu, cost, tps) for gpu, vram, hr, status, cost, daily, bw, tps in cloud_options if hr >= 0]
         if fitting_gpus:
             fitting_gpus.sort(key=lambda x: x[1])
             best = fitting_gpus[0]
             out.append("")
-            out.append("**Best value cloud option:** " + best[0] + " at $" + str(round(best[1], 2)) + "/hour")
+            rec = "**Recommended:** " + best[0] + " at $" + str(round(best[1], 2)) + "/hour"
+            if best[2] > 0:
+                rec += " (~" + str(best[2]) + " tok/s)"
+            out.append(rec)
+
+            # Best performance option
+            if len(fitting_gpus) > 1:
+                fitting_gpus.sort(key=lambda x: -x[2])
+                fastest = fitting_gpus[0]
+                if fastest[0] != best[0] and fastest[2] > 0:
+                    out.append("**Fastest:** " + fastest[0] + " (~" + str(fastest[2]) + " tok/s)")
 
         # Quantization suggestions if model is large
         if effective > 24:
             out.append("")
             out.append("### Quantization Options (to fit consumer GPUs)")
-            out.append("| Method | Estimated Size | Fits 24GB |")
-            out.append("|--------|----------------|-----------|")
-            for name, mult in [("INT8", 1.0), ("4-bit (GPTQ/AWQ)", 0.5), ("3-bit", 0.375), ("2-bit (extreme)", 0.25)]:
+            out.append("| Method | Est. Size | Fits 24GB | Quality |")
+            out.append("|--------|-----------|-----------|---------|")
+            quant_options = [
+                ("INT8", 1.0, "Excellent"),
+                ("4-bit (GPTQ/AWQ)", 0.5, "Very Good"),
+                ("3-bit", 0.375, "Good"),
+                ("2-bit (extreme)", 0.25, "Degraded"),
+            ]
+            for name, mult, quality in quant_options:
                 size = bytes_to_gb(params * mult) * 1.1
                 fits = "🟢 Yes" if size <= 24 else "🔴 No"
-                out.append("| " + name + " | " + str(round(size, 1)) + "GB | " + fits + " |")
+                out.append("| " + name + " | " + str(round(size, 1)) + "GB | " + fits + " | " + quality + " |")
+
+        # Context scaling info
+        out.append("")
+        out.append("### Context Length Scaling")
+        out.append("| Context | KV Cache | Total Est. |")
+        out.append("|---------|----------|------------|")
+        for ctx_opt in [2048, 4096, 8192, 16384, 32768]:
+            if ctx_opt <= max_position:
+                kv_opt = bytes_to_gb(2 * layers * batch * ctx_opt * kv_heads * head_dim * dtype_bytes)
+                total_opt = weights_gb + kv_opt
+                out.append("| " + str(ctx_opt) + " | " + str(round(kv_opt, 1)) + "GB | " + str(round(total_opt, 1)) + "GB |")
 
         return "\n".join(out)
     except Exception as e:
@@ -331,10 +393,46 @@ def compare(models_text, context):
         return "Error: " + str(e)
 
 
+def generate_report(model_id, context, batch, mode, framework, num_gpus, lora_rank):
+    """Generate a shareable text report"""
+    try:
+        if not model_id or not model_id.strip():
+            return "Enter a model ID first"
+
+        result = calculate(model_id, context, batch, mode, framework, num_gpus, lora_rank)
+
+        report = []
+        report.append("=" * 50)
+        report.append("VRAM CALCULATOR REPORT")
+        report.append("=" * 50)
+        report.append("")
+        report.append("Settings:")
+        report.append("  Model: " + model_id)
+        report.append("  Mode: " + mode)
+        report.append("  Context: " + str(context))
+        report.append("  Batch Size: " + str(batch))
+        report.append("  Framework: " + framework)
+        report.append("  GPUs: " + str(num_gpus))
+        if "LoRA" in mode:
+            report.append("  LoRA Rank: " + str(lora_rank))
+        report.append("")
+        report.append("-" * 50)
+        report.append("")
+        report.append(result)
+        report.append("")
+        report.append("-" * 50)
+        report.append("Generated by VRAM Calculator")
+        report.append("https://huggingface.co/spaces/Livengood/Instance-VRAM-Calculator")
+
+        return "\n".join(report)
+    except Exception as e:
+        return "Error generating report: " + str(e)
+
+
 # Build the interface
 with gr.Blocks(title="VRAM Calculator", theme=gr.themes.Soft()) as demo:
     gr.Markdown("# VRAM Calculator for LLMs")
-    gr.Markdown("Estimate VRAM requirements for HuggingFace models - inference, training, LoRA, and QLoRA")
+    gr.Markdown("Estimate VRAM requirements and throughput for HuggingFace models")
 
     with gr.Tabs():
         with gr.TabItem("Calculator"):
@@ -391,8 +489,12 @@ with gr.Blocks(title="VRAM Calculator", theme=gr.themes.Soft()) as demo:
                     info="Higher = more parameters"
                 )
 
-            calc_btn = gr.Button("Calculate VRAM", variant="primary")
+            with gr.Row():
+                calc_btn = gr.Button("Calculate VRAM", variant="primary")
+                export_btn = gr.Button("Export Report", variant="secondary")
+
             output = gr.Markdown()
+            export_output = gr.Textbox(label="Exportable Report", lines=10, visible=False)
 
             calc_btn.click(
                 fn=calculate,
@@ -400,6 +502,16 @@ with gr.Blocks(title="VRAM Calculator", theme=gr.themes.Soft()) as demo:
                 outputs=output
             )
 
+            def show_export(model_id, context, batch, mode, framework, num_gpus, lora_rank):
+                report = generate_report(model_id, context, batch, mode, framework, num_gpus, lora_rank)
+                return gr.update(visible=True, value=report)
+
+            export_btn.click(
+                fn=show_export,
+                inputs=[model_in, ctx_in, batch_in, mode_in, framework_in, gpus_in, lora_in],
+                outputs=export_output
+            )
+
             gr.Markdown("### Popular Models")
             gr.Examples(
                 examples=[
@@ -415,6 +527,8 @@ with gr.Blocks(title="VRAM Calculator", theme=gr.themes.Soft()) as demo:
                     ["google/gemma-2-27b"],
                     ["microsoft/phi-3-mini-4k-instruct"],
                     ["deepseek-ai/DeepSeek-V2-Lite"],
+                    ["NousResearch/Hermes-3-Llama-3.1-8B"],
+                    ["01-ai/Yi-1.5-34B"],
                 ],
                 inputs=[model_in],
                 label="Click to load"
@@ -450,37 +564,101 @@ with gr.Blocks(title="VRAM Calculator", theme=gr.themes.Soft()) as demo:
                     ["mistralai/Mistral-7B-v0.1\nmistralai/Mixtral-8x7B-v0.1"],
                     ["Qwen/Qwen2.5-7B\nQwen/Qwen2.5-14B\nQwen/Qwen2.5-72B"],
                     ["google/gemma-2-2b\ngoogle/gemma-2-9b\ngoogle/gemma-2-27b"],
+                    ["meta-llama/Llama-3.1-8B\nmistralai/Mistral-7B-v0.1\nQwen/Qwen2.5-7B\ngoogle/gemma-2-9b"],
                 ],
                 inputs=[cmp_in],
                 label="Click to load comparison"
             )
 
         with gr.TabItem("GPU Reference"):
-            gr.Markdown("## GPU VRAM Reference")
+            gr.Markdown("## GPU VRAM & Bandwidth Reference")
+            gr.Markdown("Memory bandwidth significantly affects inference speed (tokens/second)")
+
             gr.Markdown("### Consumer GPUs (NVIDIA GeForce)")
-            consumer_md = "| GPU | VRAM | Notes |\n|-----|------|-------|\n"
-            for gpu, vram in CONSUMER_GPUS.items():
-                consumer_md += "| " + gpu + " | " + str(vram) + "GB | Consumer |\n"
+            consumer_md = "| GPU | VRAM | Bandwidth | Best For |\n|-----|------|-----------|----------|\n"
+            for gpu, (vram, bw) in CONSUMER_GPUS.items():
+                if vram <= 12:
+                    use = "Small models (3-7B)"
+                elif vram <= 16:
+                    use = "7B models"
+                else:
+                    use = "7B-13B models, fine-tuning"
+                consumer_md += "| " + gpu + " | " + str(vram) + "GB | " + str(bw) + " GB/s | " + use + " |\n"
             gr.Markdown(consumer_md)
 
             gr.Markdown("### Apple Silicon")
-            apple_md = "| Chip | Unified Memory | Notes |\n|------|----------------|-------|\n"
-            for gpu, vram in APPLE_GPUS.items():
-                apple_md += "| " + gpu + " | " + str(vram) + "GB | Shared CPU/GPU |\n"
+            apple_md = "| Chip | Unified Memory | Bandwidth | Notes |\n|------|----------------|-----------|-------|\n"
+            for gpu, (vram, bw) in APPLE_GPUS.items():
+                apple_md += "| " + gpu + " | " + str(vram) + "GB | " + str(bw) + " GB/s | Shared CPU/GPU |\n"
             gr.Markdown(apple_md)
 
             gr.Markdown("### Cloud/Datacenter GPUs")
-            cloud_md = "| GPU | VRAM | Typical $/hr | Best For |\n|-----|------|--------------|----------|\n"
-            for gpu, (vram, cost) in CLOUD_GPUS.items():
+            cloud_md = "| GPU | VRAM | Bandwidth | $/hr | Best For |\n|-----|------|-----------|------|----------|\n"
+            for gpu, (vram, cost, bw) in CLOUD_GPUS.items():
                 if vram <= 24:
                     use = "7B models, fine-tuning"
                 elif vram <= 48:
                     use = "13B-30B models"
                 else:
                     use = "70B+ models, training"
-                cloud_md += "| " + gpu + " | " + str(vram) + "GB | $" + str(round(cost, 2)) + " | " + use + " |\n"
+                cloud_md += "| " + gpu + " | " + str(vram) + "GB | " + str(bw) + " GB/s | $" + str(round(cost, 2)) + " | " + use + " |\n"
             gr.Markdown(cloud_md)
 
+            gr.Markdown("### Understanding Throughput")
+            gr.Markdown("""
+**Tokens per second (tok/s)** estimates are based on memory bandwidth and model size.
+
+- **Memory-bound inference**: Most LLM inference is limited by how fast weights can be loaded from memory
+- **Bandwidth formula**: `tok/s ≈ (bandwidth / model_size) × batch_size × efficiency`
+- **Batching**: Higher batch sizes improve throughput but use more VRAM for KV cache
+- **Quantization**: 4-bit models load 4x faster but may have quality tradeoffs
+
+*Estimates are approximate. Actual performance depends on implementation, optimizations, and workload.*
+""")
+
+        with gr.TabItem("Tips & Guide"):
+            gr.Markdown("""
+## Quick Guide
+
+### Choosing the Right Mode
+
+| Mode | Use Case | VRAM Multiplier |
+|------|----------|-----------------|
+| **Inference** | Running predictions | 1x weights + KV cache |
+| **Training (Full)** | Training from scratch | 4-6x weights |
+| **LoRA** | Fine-tuning with adapters | 1.3x weights |
+| **QLoRA** | Memory-efficient fine-tuning | 0.5x weights + adapters |
+
+### VRAM Rule of Thumb
+
+- **Inference**: `params × 2 bytes` (FP16) + KV cache
+- **Training**: `params × 18-20 bytes` (weights + gradients + optimizer + activations)
+- **QLoRA**: `params × 0.5-0.6 bytes` (4-bit) + small adapter overhead
+
+### Fitting Large Models
+
+1. **Use quantization** (INT8, 4-bit) to reduce memory 2-4x
+2. **Reduce context length** to shrink KV cache
+3. **Use multi-GPU** for tensor parallelism
+4. **Try QLoRA** instead of full fine-tuning
+
+### Recommended Setups
+
+| Model Size | Inference | QLoRA Training |
+|------------|-----------|----------------|
+| 7B | RTX 3090/4090 (24GB) | RTX 3090/4090 |
+| 13B | A10G or 2x RTX 3090 | RTX 4090 (4-bit) |
+| 30B | A100 40GB or 2x RTX 4090 | A10G (4-bit) |
+| 70B | A100 80GB or 4x RTX 4090 | A100 40GB (4-bit) |
+
+### Cost Optimization Tips
+
+1. **Start small**: Test with smaller models first
+2. **Use spot instances**: 60-90% cheaper for training
+3. **Right-size**: Don't overpay for unused VRAM
+4. **Consider Apple Silicon**: M2/M3/M4 Max good for local inference
+""")
+
     gr.Markdown("---")
     gr.Markdown("*Estimates are approximate. Actual usage varies by implementation, batch size, and optimizations.*")