Upload folder using huggingface_hub

flash_attn/benchmark.html

@@ -3722,7 +3722,7 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
 <span onclick="toggleOutput('benchmark')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-benchmark" onclick="toggleUvLogsFromHeader('benchmark')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: benchmark | 77.34s
+Cell: benchmark | 80.35s
  | <button class="run-btn" onclick="runCell('benchmark')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('benchmark')">Copy</button>
 <a href="cells/benchmark.py" target="_blank" class="raw-btn">Raw</a>
@@ -4429,105 +4429,105 @@ xFormers not found.
 
 
 ===== Testing shape: (1, 4224, 24, 128) =====
-torch_cudnn                   : absmax=0.001009, mae=0.000075, mse=0.000000
-torch_cudnn_compile_d         : absmax=0.001009, mae=0.000075, mse=0.000000
-torch_cudnn_compile_ma        : absmax=0.001009, mae=0.000075, mse=0.000000
-torch_flash                   : absmax=0.001009, mae=0.000075, mse=0.000000
-torch_flash_compile_d         : absmax=0.001009, mae=0.000075, mse=0.000000
-torch_flash_compile_ma        : absmax=0.001009, mae=0.000075, mse=0.000000
-hf_flash_attn                 : absmax=0.001009, mae=0.000075, mse=0.000000
-hf_flash_attn3                : absmax=0.001009, mae=0.000075, mse=0.000000
+torch_cudnn                   : absmax=0.001554, mae=0.000075, mse=0.000000
+torch_cudnn_compile_d         : absmax=0.001554, mae=0.000075, mse=0.000000
+torch_cudnn_compile_ma        : absmax=0.001554, mae=0.000075, mse=0.000000
+torch_flash                   : absmax=0.001554, mae=0.000075, mse=0.000000
+torch_flash_compile_d         : absmax=0.001554, mae=0.000075, mse=0.000000
+torch_flash_compile_ma        : absmax=0.001554, mae=0.000075, mse=0.000000
+hf_flash_attn                 : absmax=0.001554, mae=0.000075, mse=0.000000
+hf_flash_attn3                : absmax=0.001554, mae=0.000075, mse=0.000000
 
 
 ===== Testing shape: (1, 4352, 24, 128) =====
-torch_cudnn                   : absmax=0.001015, mae=0.000073, mse=0.000000
-torch_cudnn_compile_d         : absmax=0.001015, mae=0.000073, mse=0.000000
-torch_cudnn_compile_ma        : absmax=0.001015, mae=0.000073, mse=0.000000
-torch_flash                   : absmax=0.001015, mae=0.000073, mse=0.000000
-torch_flash_compile_d         : absmax=0.001015, mae=0.000073, mse=0.000000
-torch_flash_compile_ma        : absmax=0.001015, mae=0.000073, mse=0.000000
-hf_flash_attn                 : absmax=0.001015, mae=0.000073, mse=0.000000
-hf_flash_attn3                : absmax=0.001015, mae=0.000073, mse=0.000000
+torch_cudnn                   : absmax=0.001499, mae=0.000074, mse=0.000000
+torch_cudnn_compile_d         : absmax=0.001499, mae=0.000074, mse=0.000000
+torch_cudnn_compile_ma        : absmax=0.001499, mae=0.000074, mse=0.000000
+torch_flash                   : absmax=0.001499, mae=0.000074, mse=0.000000
+torch_flash_compile_d         : absmax=0.001499, mae=0.000074, mse=0.000000
+torch_flash_compile_ma        : absmax=0.001499, mae=0.000074, mse=0.000000
+hf_flash_attn                 : absmax=0.001499, mae=0.000074, mse=0.000000
+hf_flash_attn3                : absmax=0.001499, mae=0.000074, mse=0.000000
 
 
 ===== Testing shape: (1, 4416, 24, 128) =====
-torch_cudnn                   : absmax=0.001374, mae=0.000073, mse=0.000000
-torch_cudnn_compile_d         : absmax=0.001374, mae=0.000073, mse=0.000000
-torch_cudnn_compile_ma        : absmax=0.001374, mae=0.000073, mse=0.000000
-torch_flash                   : absmax=0.001374, mae=0.000073, mse=0.000000
-torch_flash_compile_d         : absmax=0.001374, mae=0.000073, mse=0.000000
-torch_flash_compile_ma        : absmax=0.001374, mae=0.000073, mse=0.000000
-hf_flash_attn                 : absmax=0.001374, mae=0.000073, mse=0.000000
-hf_flash_attn3                : absmax=0.001374, mae=0.000073, mse=0.000000
+torch_cudnn                   : absmax=0.001278, mae=0.000073, mse=0.000000
+torch_cudnn_compile_d         : absmax=0.001278, mae=0.000073, mse=0.000000
+torch_cudnn_compile_ma        : absmax=0.001278, mae=0.000073, mse=0.000000
+torch_flash                   : absmax=0.001278, mae=0.000073, mse=0.000000
+torch_flash_compile_d         : absmax=0.001278, mae=0.000073, mse=0.000000
+torch_flash_compile_ma        : absmax=0.001278, mae=0.000073, mse=0.000000
+hf_flash_attn                 : absmax=0.001278, mae=0.000073, mse=0.000000
+hf_flash_attn3                : absmax=0.001278, mae=0.000073, mse=0.000000
 
 
 ===== Testing shape: (1, 4480, 24, 128) =====
-torch_cudnn                   : absmax=0.001190, mae=0.000072, mse=0.000000
-torch_cudnn_compile_d         : absmax=0.001190, mae=0.000072, mse=0.000000
-torch_cudnn_compile_ma        : absmax=0.001190, mae=0.000072, mse=0.000000
-torch_flash                   : absmax=0.001190, mae=0.000072, mse=0.000000
-torch_flash_compile_d         : absmax=0.001190, mae=0.000072, mse=0.000000
-torch_flash_compile_ma        : absmax=0.001190, mae=0.000072, mse=0.000000
-hf_flash_attn                 : absmax=0.001190, mae=0.000072, mse=0.000000
-hf_flash_attn3                : absmax=0.001190, mae=0.000072, mse=0.000000
+torch_cudnn                   : absmax=0.001270, mae=0.000073, mse=0.000000
+torch_cudnn_compile_d         : absmax=0.001270, mae=0.000073, mse=0.000000
+torch_cudnn_compile_ma        : absmax=0.001270, mae=0.000073, mse=0.000000
+torch_flash                   : absmax=0.001270, mae=0.000073, mse=0.000000
+torch_flash_compile_d         : absmax=0.001270, mae=0.000073, mse=0.000000
+torch_flash_compile_ma        : absmax=0.001270, mae=0.000073, mse=0.000000
+hf_flash_attn                 : absmax=0.001270, mae=0.000073, mse=0.000000
+hf_flash_attn3                : absmax=0.001270, mae=0.000073, mse=0.000000
 
 
 ===== Testing shape: (1, 4544, 24, 128) =====
-torch_cudnn                   : absmax=0.001189, mae=0.000072, mse=0.000000
-torch_cudnn_compile_d         : absmax=0.001189, mae=0.000072, mse=0.000000
-torch_cudnn_compile_ma        : absmax=0.001189, mae=0.000072, mse=0.000000
-torch_flash                   : absmax=0.001189, mae=0.000072, mse=0.000000
-torch_flash_compile_d         : absmax=0.001189, mae=0.000072, mse=0.000000
-torch_flash_compile_ma        : absmax=0.001189, mae=0.000072, mse=0.000000
-hf_flash_attn                 : absmax=0.001189, mae=0.000072, mse=0.000000
-hf_flash_attn3                : absmax=0.001189, mae=0.000072, mse=0.000000
+torch_cudnn                   : absmax=0.001696, mae=0.000072, mse=0.000000
+torch_cudnn_compile_d         : absmax=0.001696, mae=0.000072, mse=0.000000
+torch_cudnn_compile_ma        : absmax=0.001696, mae=0.000072, mse=0.000000
+torch_flash                   : absmax=0.001696, mae=0.000072, mse=0.000000
+torch_flash_compile_d         : absmax=0.001696, mae=0.000072, mse=0.000000
+torch_flash_compile_ma        : absmax=0.001696, mae=0.000072, mse=0.000000
+hf_flash_attn                 : absmax=0.001696, mae=0.000072, mse=0.000000
+hf_flash_attn3                : absmax=0.001696, mae=0.000072, mse=0.000000
 
 
 ===== Testing shape: (1, 4608, 24, 128) =====
-torch_cudnn                   : absmax=0.000851, mae=0.000072, mse=0.000000
-torch_cudnn_compile_d         : absmax=0.000851, mae=0.000072, mse=0.000000
-torch_cudnn_compile_ma        : absmax=0.000851, mae=0.000072, mse=0.000000
-torch_flash                   : absmax=0.001095, mae=0.000072, mse=0.000000
-torch_flash_compile_d         : absmax=0.001095, mae=0.000072, mse=0.000000
-torch_flash_compile_ma        : absmax=0.001095, mae=0.000072, mse=0.000000
-hf_flash_attn                 : absmax=0.001095, mae=0.000072, mse=0.000000
-hf_flash_attn3                : absmax=0.000872, mae=0.000072, mse=0.000000
+torch_cudnn                   : absmax=0.001111, mae=0.000071, mse=0.000000
+torch_cudnn_compile_d         : absmax=0.001111, mae=0.000071, mse=0.000000
+torch_cudnn_compile_ma        : absmax=0.001111, mae=0.000071, mse=0.000000
+torch_flash                   : absmax=0.001111, mae=0.000071, mse=0.000000
+torch_flash_compile_d         : absmax=0.001111, mae=0.000071, mse=0.000000
+torch_flash_compile_ma        : absmax=0.001111, mae=0.000071, mse=0.000000
+hf_flash_attn                 : absmax=0.001111, mae=0.000071, mse=0.000000
+hf_flash_attn3                : absmax=0.001111, mae=0.000071, mse=0.000000
 Attention Benchmark:
    seq_len  torch_cudnn  torch_cudnn_compile_d  torch_cudnn_compile_ma  torch_flash  torch_flash_compile_d  torch_flash_compile_ma  hf_flash_attn  hf_flash_attn3
-0   4224.0     3.798368               3.791408                4.181328     3.967520               3.956704                4.310320       3.395904        3.330144
-1   4352.0     4.079536               4.073056                4.418592     4.399408               4.394336                4.733312       3.836944        3.757936
-2   4416.0     4.143008               4.138512                4.483328     4.452528               4.444224                4.789856       3.894816        3.862496
-3   4480.0     4.205120               4.199184                4.552352     4.529248               4.523552                4.870752       3.953808        3.870560
-4   4544.0     4.437120               4.428784                4.782656     4.584704               4.576832                4.933216       4.008992        3.975952
-5   4608.0     4.500704               4.493792                4.869824     4.658752               4.654752                5.028256       4.066272        3.985760
+0   4224.0     3.801536               3.793184                4.180320     3.967200               3.980720                4.318736       3.397664        3.333776
+1   4352.0     4.081888               4.075536                4.421904     4.400096               4.393040                4.733888       3.836736        3.760320
+2   4416.0     4.144176               4.139104                4.485120     4.453280               4.447408                4.796512       3.895568        3.863744
+3   4480.0     4.208256               4.204688                4.554240     4.529152               4.521568                4.874592       3.952256        3.870208
+4   4544.0     4.437280               4.431168                4.790432     4.584704               4.579008                4.938656       4.010512        3.978752
+5   4608.0     4.504352               4.497600                4.870880     4.661536               4.652544                5.031328       4.065504        3.985712
 </div>
 <div class="uv-install-logs" id="uv-logs-benchmark">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading kiwisolver (1.4MiB)
-Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
-Downloading nvidia-cublas-cu12 (566.8MiB)
-Downloading pandas (11.8MiB)
 Downloading hf-xet (3.0MiB)
-Downloading nvidia-curand-cu12 (60.7MiB)
-Downloading fonttools (4.7MiB)
-Downloading networkx (1.9MiB)
+Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
+Downloading sympy (6.0MiB)
+Downloading pandas (11.8MiB)
 Downloading numpy (16.2MiB)
-Downloading torch (846.9MiB)
 Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading nvidia-cudnn-cu12 (674.0MiB)
+Downloading networkx (1.9MiB)
 Downloading nvidia-nccl-cu12 (307.4MiB)
-Downloading setuptools (1.1MiB)
-Downloading nvidia-cufft-cu12 (184.2MiB)
-Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
-Downloading sympy (6.0MiB)
-Downloading nvidia-cufile-cu12 (1.1MiB)
+Downloading nvidia-curand-cu12 (60.7MiB)
 Downloading nvidia-nvjitlink-cu12 (37.4MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading matplotlib (8.3MiB)
+Downloading nvidia-cublas-cu12 (566.8MiB)
+Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
 Downloading nvidia-cusolver-cu12 (255.1MiB)
 Downloading pillow (6.3MiB)
-Downloading matplotlib (8.3MiB)
+Downloading nvidia-cudnn-cu12 (674.0MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading nvidia-cufile-cu12 (1.1MiB)
+Downloading nvidia-cufft-cu12 (184.2MiB)
+Downloading fonttools (4.7MiB)
+Downloading setuptools (1.1MiB)
 Downloading triton (148.3MiB)
+Downloading torch (846.9MiB)
+Downloading kiwisolver (1.4MiB)
  Downloading nvidia-cufile-cu12
  Downloading kiwisolver
  Downloading hf-xet
@@ -4541,8 +4541,8 @@ Downloading triton (148.3MiB)
  Downloading numpy
  Downloading nvidia-nvjitlink-cu12
  Downloading nvidia-curand-cu12
- Downloading nvidia-cuda-nvrtc-cu12
  Downloading pandas
+ Downloading nvidia-cuda-nvrtc-cu12
  Downloading triton
  Downloading nvidia-cufft-cu12
  Downloading nvidia-cusolver-cu12
@@ -4552,18 +4552,18 @@ Downloading triton (148.3MiB)
  Downloading nvidia-cublas-cu12
  Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 49 packages in 563ms
+Installed 49 packages in 546ms
 </div>
 </div>
 <div class="cell-stderr">Fetching 20 files:   0%|          | 0/20 [00:00&lt;?, ?it/s]
-Fetching 20 files:   5%|▌         | 1/20 [00:00&lt;00:04,  4.34it/s]
-Fetching 20 files:  10%|█         | 2/20 [00:02&lt;00:25,  1.42s/it]
-Fetching 20 files: 100%|██████████| 20/20 [00:02&lt;00:00,  8.05it/s]
+Fetching 20 files:   5%|▌         | 1/20 [00:00&lt;00:04,  4.75it/s]
+Fetching 20 files:  10%|█         | 2/20 [00:01&lt;00:16,  1.06it/s]
+Fetching 20 files: 100%|██████████| 20/20 [00:01&lt;00:00, 12.01it/s]
 
 Fetching 4 files:   0%|          | 0/4 [00:00&lt;?, ?it/s]
-Fetching 4 files:  25%|██▌       | 1/4 [00:00&lt;00:00,  5.49it/s]
-Fetching 4 files:  50%|█████     | 2/4 [00:01&lt;00:01,  1.15it/s]
-Fetching 4 files: 100%|██████████| 4/4 [00:01&lt;00:00,  2.60it/s]</div>
+Fetching 4 files:  25%|██▌       | 1/4 [00:00&lt;00:01,  1.98it/s]
+Fetching 4 files:  50%|█████     | 2/4 [00:01&lt;00:01,  1.38it/s]
+Fetching 4 files: 100%|██████████| 4/4 [00:01&lt;00:00,  2.89it/s]</div>
 <div class="cell-artifacts">
 <h4>Artifacts:</h4>
 <a href="artifacts/benchmark/dump_attention_benchmark/Attention Benchmark.png" class="artifact" target="_blank">dump_attention_benchmark/Attention Benchmark.png</a>

moe_benchmarks/megablocks/cells/forward_and_backward_no_kernel.py

@@ -0,0 +1,196 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "accelerate>=1.10.1",
+#     "torch>=2.7.0",
+#     "kernels==0.10.0",
+#     "transformers@https://github.com/huggingface/transformers.git",
+#     "ipdb>=0.13.13",
+#     "matplotlib>=3.7.2",
+#     "numpy>=1.24.3",
+# ]
+# ///
+
+import torch
+from transformers import GptOssForCausalLM, PreTrainedTokenizerFast, Mxfp4Config
+import time
+import torch.nn as nn
+from kernels import register_kernel_mapping, Mode, LayerRepository, replace_kernel_forward_from_hub
+import sys
+import torch.profiler
+import gc
+import logging
+from transformers.models.gpt_oss.modeling_gpt_oss import GptOssRMSNorm
+
+# remove liger kernel for testing 
+replace_kernel_forward_from_hub(GptOssRMSNorm, None)
+
+# set to debug logging
+logging.basicConfig(level=logging.INFO)
+
+def reset_peak_memory_stats():
+    """Clear CUDA cache and reset memory allocation counters."""
+    torch.cuda.empty_cache()
+    if torch.cuda.is_available():
+        torch.cuda.reset_peak_memory_stats()
+    gc.collect()
+
+def get_memory_stats():
+    """Get current and peak CUDA memory usage."""
+    if not torch.cuda.is_available():
+        return {"allocated_gb": 0, "peak_gb": 0, "reserved_gb": 0}
+    return {
+        "allocated_gb": torch.cuda.memory_allocated() / 1e9,
+        "peak_gb": torch.cuda.max_memory_allocated() / 1e9,
+        "reserved_gb": torch.cuda.memory_reserved() / 1e9,
+    }
+
+def override_kernel_layer_name(cls_name: str, value) -> bool:
+    """Helper to dynamically override the kernel_layer_name in a model class."""
+    for mod in sys.modules.values():
+        if mod is None:
+            continue
+        obj = getattr(mod, cls_name, None)
+        if isinstance(obj, type) and issubclass(obj, nn.Module):
+            setattr(obj, "kernel_layer_name", value)
+            print(f"Overrode {cls_name}.kernel_layer_name to {value}")
+            return True
+    return False
+
+
+# Init the model the normal way
+model_id = "openai/gpt-oss-20b"
+tokenizer = PreTrainedTokenizerFast.from_pretrained(model_id)
+quantization_config = Mxfp4Config(dequantize=True)
+
+model = GptOssForCausalLM.from_pretrained(
+    model_id,
+    dtype="bfloat16",
+    device_map="auto",
+    use_kernels=False,
+    quantization_config=quantization_config,
+).eval()
+
+messages = [
+    {"role": "system", "content": "What is Tensor Parallelism?"},
+]
+
+inputs = tokenizer.apply_chat_template(
+    messages,
+    add_generation_prompt=True,
+    return_tensors="pt",
+    return_dict=True,
+    reasoning_effort="low",
+).to("cuda")
+
+max_tokens = 128  # Reduced to help with memory usage
+
+# Clear memory before backward pass
+reset_peak_memory_stats()
+print(f"Pre-generation memory: {get_memory_stats()}")
+
+# forward and backward pass
+with torch.autograd.set_grad_enabled(True):
+    start_time = time.perf_counter()
+    generated = model.generate(
+        **inputs,
+        max_new_tokens=max_tokens,
+        do_sample=False,
+        temperature=None,
+    )
+    end_time = time.perf_counter()
+    print(tokenizer.decode(generated[0], skip_special_tokens=False))
+    print(f"Generation took {end_time - start_time:.2f} seconds")
+    print(f"Post-generation memory: {get_memory_stats()}")
+
+    # Use gradient checkpointing to reduce memory usage
+    if hasattr(model, 'gradient_checkpointing_enable'):
+        model.gradient_checkpointing_enable()
+        print("Enabled gradient checkpointing")
+
+    # Reduce sequence length if needed for memory
+    max_seq_len = 512  # Limit sequence length for backward pass
+    if generated.size(1) > max_seq_len:
+        print(f"Truncating sequence from {generated.size(1)} to {max_seq_len} tokens")
+        full_sequence = generated[:, -max_seq_len:]
+    else:
+        full_sequence = generated
+
+    # Get model outputs for the full sequence
+    model.train()  # Enable dropout and other training behaviors
+
+    try:
+        outputs = model(
+            input_ids=full_sequence,
+            labels=full_sequence,  # This will compute loss internally
+            return_dict=True
+        )
+        print(f"Post-forward memory: {get_memory_stats()}")
+
+        # If model doesn't compute loss, compute it manually
+        if outputs.loss is None:
+            shift_logits = outputs.logits[..., :-1, :].contiguous()
+            shift_labels = full_sequence[..., 1:].contiguous()
+
+            # Use CrossEntropyLoss with ignore_index for padding tokens
+            loss_fct = torch.nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id if tokenizer.pad_token_id is not None else -100)
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)),
+                shift_labels.view(-1)
+            )
+        else:
+            loss = outputs.loss
+
+        print(f"Loss: {loss.item():.4f}")
+
+        # Clear intermediate tensors to save memory
+        del outputs
+        torch.cuda.empty_cache()
+
+        # Perform backward pass with memory management
+        print("Running backward pass...")
+        print(f"Pre-backward memory: {get_memory_stats()}")
+
+        loss.backward()
+        print(f"Post-backward memory: {get_memory_stats()}")
+
+    except torch.cuda.OutOfMemoryError as e:
+        print(f"OOM during forward/backward pass: {e}")
+        print("Try reducing max_tokens or max_seq_len")
+        raise
+
+    # Calculate gradient statistics and print sample gradients
+    total_norm = 0.0
+    param_count = 0
+    grad_samples = {}
+
+    for name, p in model.named_parameters():
+        if p.grad is not None:
+            param_count += 1
+            grad_norm = p.grad.data.norm(2).item()
+            total_norm += grad_norm ** 2
+
+            # Collect gradient statistics for key layers
+            if any(key in name for key in ['embed', 'lm_head', 'mlp.up', 'mlp.down', 'self_attn.q_proj', 'norm']):
+                grad_samples[name] = {
+                    'norm': grad_norm,
+                    'mean': p.grad.data.mean().item(),
+                    'std': p.grad.data.std().item(),
+                    'max': p.grad.data.max().item(),
+                    'min': p.grad.data.min().item(),
+                }
+
+    total_norm = total_norm ** 0.5
+
+    print(f"\nGradient norm: {total_norm:.4f}")
+    print(f"Parameters with gradients: {param_count}")
+
+    # Print sample gradients from important layers
+    print("\nSample gradient statistics:")
+    for i, (name, stats) in enumerate(list(grad_samples.items())[:10]):
+        print(f"  {name[:60]:<60} | norm: {stats['norm']:.4e} | mean: {stats['mean']:.4e} | std: {stats['std']:.4e}")
+
+    # Optional: zero gradients for next iteration
+    model.zero_grad()
+    model.eval()  # Switch back to eval mode
+

moe_benchmarks/megablocks/megablocks_only.html

@@ -3720,126 +3720,217 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
 <h2>Forward</h2>
 <h2>Forward and Backward</h2>
 <p>Next, we'll attempt to run a forward and backward pass without any custom kernels. This will likely run out of memory since the default implementation is not optimized for memory usage.</p>
-<h1>Kernels</h1>
-<p>Next we can run with Megablocks kernels enabled.</p>
-<h3>Forward</h3>
-<p>First, we run a forward pass with Megablocks kernels.</p>
-<div class="cell cell-failed" id="cell-forward_only">
+<div class="cell cell-failed" id="cell-forward_and_backward_no_kernel">
 <div class="cell-header">
 <span class="collapse-indicators">
-<span onclick="toggleCode('forward_only')" style="cursor: pointer;">▼ code</span> 
-<span onclick="toggleOutput('forward_only')" style="cursor: pointer;">▼ output</span>
- <span id="uv-indicator-forward_only" style="cursor: default; opacity: 0.3;">▶ uv-logs</span>
+<span onclick="toggleCode('forward_and_backward_no_kernel')" style="cursor: pointer;">▼ code</span> 
+<span onclick="toggleOutput('forward_and_backward_no_kernel')" style="cursor: pointer;">▼ output</span>
+ <span id="uv-indicator-forward_and_backward_no_kernel" style="cursor: default; opacity: 0.3;">▶ uv-logs</span>
 </span> | 
-Cell: forward_only | 17.22s | FAILED
- | <button class="run-btn" onclick="runCell('forward_only')">▶ run</button>
-<button class="copy-btn" onclick="copyCell('forward_only')">Copy</button>
-<a href="cells/forward_only.py" target="_blank" class="raw-btn">Raw</a>
+Cell: forward_and_backward_no_kernel | 17.02s | FAILED
+ | <button class="run-btn" onclick="runCell('forward_and_backward_no_kernel')">▶ run</button>
+<button class="copy-btn" onclick="copyCell('forward_and_backward_no_kernel')">Copy</button>
+<a href="cells/forward_and_backward_no_kernel.py" target="_blank" class="raw-btn">Raw</a>
 </div>
-<div id="code-forward_only" class="cell-code" data-lines="101">
+<div id="code-forward_and_backward_no_kernel" class="cell-code" data-lines="196">
 <div class="highlight-with-lines">
-<div class="line-numbers" id="lines-forward_only">
-<a class="line-number" data-cell="forward_only" data-line="1" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 1, true);">1</a>
-<a class="line-number" data-cell="forward_only" data-line="2" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 2, true);">2</a>
-<a class="line-number" data-cell="forward_only" data-line="3" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 3, true);">3</a>
-<a class="line-number" data-cell="forward_only" data-line="4" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 4, true);">4</a>
-<a class="line-number" data-cell="forward_only" data-line="5" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 5, true);">5</a>
-<a class="line-number" data-cell="forward_only" data-line="6" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 6, true);">6</a>
-<a class="line-number" data-cell="forward_only" data-line="7" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 7, true);">7</a>
-<a class="line-number" data-cell="forward_only" data-line="8" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 8, true);">8</a>
-<a class="line-number" data-cell="forward_only" data-line="9" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 9, true);">9</a>
-<a class="line-number" data-cell="forward_only" data-line="10" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 10, true);">10</a>
-<a class="line-number" data-cell="forward_only" data-line="11" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 11, true);">11</a>
-<a class="line-number" data-cell="forward_only" data-line="12" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 12, true);">12</a>
-<a class="line-number" data-cell="forward_only" data-line="13" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 13, true);">13</a>
-<a class="line-number" data-cell="forward_only" data-line="14" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 14, true);">14</a>
-<a class="line-number" data-cell="forward_only" data-line="15" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 15, true);">15</a>
-<a class="line-number" data-cell="forward_only" data-line="16" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 16, true);">16</a>
-<a class="line-number" data-cell="forward_only" data-line="17" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 17, true);">17</a>
-<a class="line-number" data-cell="forward_only" data-line="18" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 18, true);">18</a>
-<a class="line-number" data-cell="forward_only" data-line="19" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 19, true);">19</a>
-<a class="line-number" data-cell="forward_only" data-line="20" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 20, true);">20</a>
-<a class="line-number" data-cell="forward_only" data-line="21" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 21, true);">21</a>
-<a class="line-number" data-cell="forward_only" data-line="22" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 22, true);">22</a>
-<a class="line-number" data-cell="forward_only" data-line="23" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 23, true);">23</a>
-<a class="line-number" data-cell="forward_only" data-line="24" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 24, true);">24</a>
-<a class="line-number" data-cell="forward_only" data-line="25" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 25, true);">25</a>
-<a class="line-number" data-cell="forward_only" data-line="26" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 26, true);">26</a>
-<a class="line-number" data-cell="forward_only" data-line="27" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 27, true);">27</a>
-<a class="line-number" data-cell="forward_only" data-line="28" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 28, true);">28</a>
-<a class="line-number" data-cell="forward_only" data-line="29" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 29, true);">29</a>
-<a class="line-number" data-cell="forward_only" data-line="30" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 30, true);">30</a>
-<a class="line-number" data-cell="forward_only" data-line="31" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 31, true);">31</a>
-<a class="line-number" data-cell="forward_only" data-line="32" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 32, true);">32</a>
-<a class="line-number" data-cell="forward_only" data-line="33" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 33, true);">33</a>
-<a class="line-number" data-cell="forward_only" data-line="34" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 34, true);">34</a>
-<a class="line-number" data-cell="forward_only" data-line="35" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 35, true);">35</a>
-<a class="line-number" data-cell="forward_only" data-line="36" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 36, true);">36</a>
-<a class="line-number" data-cell="forward_only" data-line="37" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 37, true);">37</a>
-<a class="line-number" data-cell="forward_only" data-line="38" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 38, true);">38</a>
-<a class="line-number" data-cell="forward_only" data-line="39" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 39, true);">39</a>
-<a class="line-number" data-cell="forward_only" data-line="40" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 40, true);">40</a>
-<a class="line-number" data-cell="forward_only" data-line="41" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 41, true);">41</a>
-<a class="line-number" data-cell="forward_only" data-line="42" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 42, true);">42</a>
-<a class="line-number" data-cell="forward_only" data-line="43" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 43, true);">43</a>
-<a class="line-number" data-cell="forward_only" data-line="44" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 44, true);">44</a>
-<a class="line-number" data-cell="forward_only" data-line="45" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 45, true);">45</a>
-<a class="line-number" data-cell="forward_only" data-line="46" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 46, true);">46</a>
-<a class="line-number" data-cell="forward_only" data-line="47" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 47, true);">47</a>
-<a class="line-number" data-cell="forward_only" data-line="48" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 48, true);">48</a>
-<a class="line-number" data-cell="forward_only" data-line="49" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 49, true);">49</a>
-<a class="line-number" data-cell="forward_only" data-line="50" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 50, true);">50</a>
-<a class="line-number" data-cell="forward_only" data-line="51" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 51, true);">51</a>
-<a class="line-number" data-cell="forward_only" data-line="52" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 52, true);">52</a>
-<a class="line-number" data-cell="forward_only" data-line="53" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 53, true);">53</a>
-<a class="line-number" data-cell="forward_only" data-line="54" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 54, true);">54</a>
-<a class="line-number" data-cell="forward_only" data-line="55" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 55, true);">55</a>
-<a class="line-number" data-cell="forward_only" data-line="56" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 56, true);">56</a>
-<a class="line-number" data-cell="forward_only" data-line="57" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 57, true);">57</a>
-<a class="line-number" data-cell="forward_only" data-line="58" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 58, true);">58</a>
-<a class="line-number" data-cell="forward_only" data-line="59" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 59, true);">59</a>
-<a class="line-number" data-cell="forward_only" data-line="60" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 60, true);">60</a>
-<a class="line-number" data-cell="forward_only" data-line="61" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 61, true);">61</a>
-<a class="line-number" data-cell="forward_only" data-line="62" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 62, true);">62</a>
-<a class="line-number" data-cell="forward_only" data-line="63" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 63, true);">63</a>
-<a class="line-number" data-cell="forward_only" data-line="64" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 64, true);">64</a>
-<a class="line-number" data-cell="forward_only" data-line="65" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 65, true);">65</a>
-<a class="line-number" data-cell="forward_only" data-line="66" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 66, true);">66</a>
-<a class="line-number" data-cell="forward_only" data-line="67" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 67, true);">67</a>
-<a class="line-number" data-cell="forward_only" data-line="68" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 68, true);">68</a>
-<a class="line-number" data-cell="forward_only" data-line="69" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 69, true);">69</a>
-<a class="line-number" data-cell="forward_only" data-line="70" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 70, true);">70</a>
-<a class="line-number" data-cell="forward_only" data-line="71" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 71, true);">71</a>
-<a class="line-number" data-cell="forward_only" data-line="72" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 72, true);">72</a>
-<a class="line-number" data-cell="forward_only" data-line="73" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 73, true);">73</a>
-<a class="line-number" data-cell="forward_only" data-line="74" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 74, true);">74</a>
-<a class="line-number" data-cell="forward_only" data-line="75" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 75, true);">75</a>
-<a class="line-number" data-cell="forward_only" data-line="76" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 76, true);">76</a>
-<a class="line-number" data-cell="forward_only" data-line="77" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 77, true);">77</a>
-<a class="line-number" data-cell="forward_only" data-line="78" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 78, true);">78</a>
-<a class="line-number" data-cell="forward_only" data-line="79" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 79, true);">79</a>
-<a class="line-number" data-cell="forward_only" data-line="80" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 80, true);">80</a>
-<a class="line-number" data-cell="forward_only" data-line="81" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 81, true);">81</a>
-<a class="line-number" data-cell="forward_only" data-line="82" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 82, true);">82</a>
-<a class="line-number" data-cell="forward_only" data-line="83" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 83, true);">83</a>
-<a class="line-number" data-cell="forward_only" data-line="84" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 84, true);">84</a>
-<a class="line-number" data-cell="forward_only" data-line="85" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 85, true);">85</a>
-<a class="line-number" data-cell="forward_only" data-line="86" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 86, true);">86</a>
-<a class="line-number" data-cell="forward_only" data-line="87" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 87, true);">87</a>
-<a class="line-number" data-cell="forward_only" data-line="88" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 88, true);">88</a>
-<a class="line-number" data-cell="forward_only" data-line="89" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 89, true);">89</a>
-<a class="line-number" data-cell="forward_only" data-line="90" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 90, true);">90</a>
-<a class="line-number" data-cell="forward_only" data-line="91" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 91, true);">91</a>
-<a class="line-number" data-cell="forward_only" data-line="92" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 92, true);">92</a>
-<a class="line-number" data-cell="forward_only" data-line="93" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 93, true);">93</a>
-<a class="line-number" data-cell="forward_only" data-line="94" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 94, true);">94</a>
-<a class="line-number" data-cell="forward_only" data-line="95" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 95, true);">95</a>
-<a class="line-number" data-cell="forward_only" data-line="96" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 96, true);">96</a>
-<a class="line-number" data-cell="forward_only" data-line="97" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 97, true);">97</a>
-<a class="line-number" data-cell="forward_only" data-line="98" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 98, true);">98</a>
-<a class="line-number" data-cell="forward_only" data-line="99" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 99, true);">99</a>
-<a class="line-number" data-cell="forward_only" data-line="100" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 100, true);">100</a>
-<a class="line-number" data-cell="forward_only" data-line="101" href="#cell-forward_only" onclick="event.preventDefault(); selectCellLine('forward_only', 101, true);">101</a>
+<div class="line-numbers" id="lines-forward_and_backward_no_kernel">
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="1" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 1, true);">1</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="2" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 2, true);">2</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="3" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 3, true);">3</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="4" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 4, true);">4</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="5" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 5, true);">5</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="6" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 6, true);">6</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="7" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 7, true);">7</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="8" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 8, true);">8</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="9" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 9, true);">9</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="10" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 10, true);">10</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="11" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 11, true);">11</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="12" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 12, true);">12</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="13" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 13, true);">13</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="14" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 14, true);">14</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="15" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 15, true);">15</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="16" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 16, true);">16</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="17" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 17, true);">17</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="18" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 18, true);">18</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="19" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 19, true);">19</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="20" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 20, true);">20</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="21" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 21, true);">21</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="22" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 22, true);">22</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="23" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 23, true);">23</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="24" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 24, true);">24</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="25" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 25, true);">25</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="26" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 26, true);">26</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="27" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 27, true);">27</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="28" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 28, true);">28</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="29" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 29, true);">29</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="30" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 30, true);">30</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="31" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 31, true);">31</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="32" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 32, true);">32</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="33" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 33, true);">33</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="34" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 34, true);">34</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="35" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 35, true);">35</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="36" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 36, true);">36</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="37" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 37, true);">37</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="38" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 38, true);">38</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="39" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 39, true);">39</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="40" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 40, true);">40</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="41" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 41, true);">41</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="42" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 42, true);">42</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="43" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 43, true);">43</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="44" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 44, true);">44</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="45" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 45, true);">45</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="46" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 46, true);">46</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="47" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 47, true);">47</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="48" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 48, true);">48</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="49" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 49, true);">49</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="50" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 50, true);">50</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="51" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 51, true);">51</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="52" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 52, true);">52</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="53" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 53, true);">53</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="54" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 54, true);">54</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="55" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 55, true);">55</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="56" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 56, true);">56</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="57" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 57, true);">57</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="58" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 58, true);">58</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="59" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 59, true);">59</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="60" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 60, true);">60</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="61" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 61, true);">61</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="62" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 62, true);">62</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="63" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 63, true);">63</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="64" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 64, true);">64</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="65" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 65, true);">65</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="66" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 66, true);">66</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="67" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 67, true);">67</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="68" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 68, true);">68</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="69" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 69, true);">69</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="70" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 70, true);">70</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="71" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 71, true);">71</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="72" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 72, true);">72</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="73" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 73, true);">73</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="74" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 74, true);">74</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="75" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 75, true);">75</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="76" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 76, true);">76</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="77" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 77, true);">77</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="78" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 78, true);">78</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="79" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 79, true);">79</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="80" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 80, true);">80</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="81" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 81, true);">81</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="82" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 82, true);">82</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="83" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 83, true);">83</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="84" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 84, true);">84</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="85" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 85, true);">85</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="86" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 86, true);">86</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="87" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 87, true);">87</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="88" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 88, true);">88</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="89" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 89, true);">89</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="90" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 90, true);">90</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="91" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 91, true);">91</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="92" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 92, true);">92</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="93" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 93, true);">93</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="94" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 94, true);">94</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="95" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 95, true);">95</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="96" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 96, true);">96</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="97" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 97, true);">97</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="98" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 98, true);">98</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="99" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 99, true);">99</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="100" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 100, true);">100</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="101" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 101, true);">101</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="102" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 102, true);">102</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="103" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 103, true);">103</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="104" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 104, true);">104</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="105" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 105, true);">105</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="106" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 106, true);">106</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="107" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 107, true);">107</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="108" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 108, true);">108</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="109" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 109, true);">109</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="110" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 110, true);">110</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="111" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 111, true);">111</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="112" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 112, true);">112</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="113" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 113, true);">113</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="114" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 114, true);">114</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="115" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 115, true);">115</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="116" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 116, true);">116</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="117" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 117, true);">117</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="118" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 118, true);">118</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="119" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 119, true);">119</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="120" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 120, true);">120</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="121" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 121, true);">121</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="122" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 122, true);">122</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="123" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 123, true);">123</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="124" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 124, true);">124</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="125" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 125, true);">125</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="126" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 126, true);">126</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="127" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 127, true);">127</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="128" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 128, true);">128</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="129" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 129, true);">129</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="130" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 130, true);">130</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="131" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 131, true);">131</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="132" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 132, true);">132</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="133" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 133, true);">133</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="134" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 134, true);">134</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="135" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 135, true);">135</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="136" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 136, true);">136</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="137" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 137, true);">137</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="138" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 138, true);">138</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="139" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 139, true);">139</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="140" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 140, true);">140</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="141" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 141, true);">141</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="142" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 142, true);">142</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="143" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 143, true);">143</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="144" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 144, true);">144</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="145" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 145, true);">145</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="146" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 146, true);">146</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="147" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 147, true);">147</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="148" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 148, true);">148</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="149" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 149, true);">149</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="150" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 150, true);">150</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="151" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 151, true);">151</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="152" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 152, true);">152</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="153" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 153, true);">153</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="154" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 154, true);">154</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="155" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 155, true);">155</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="156" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 156, true);">156</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="157" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 157, true);">157</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="158" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 158, true);">158</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="159" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 159, true);">159</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="160" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 160, true);">160</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="161" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 161, true);">161</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="162" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 162, true);">162</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="163" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 163, true);">163</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="164" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 164, true);">164</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="165" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 165, true);">165</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="166" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 166, true);">166</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="167" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 167, true);">167</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="168" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 168, true);">168</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="169" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 169, true);">169</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="170" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 170, true);">170</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="171" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 171, true);">171</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="172" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 172, true);">172</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="173" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 173, true);">173</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="174" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 174, true);">174</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="175" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 175, true);">175</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="176" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 176, true);">176</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="177" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 177, true);">177</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="178" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 178, true);">178</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="179" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 179, true);">179</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="180" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 180, true);">180</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="181" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 181, true);">181</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="182" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 182, true);">182</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="183" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 183, true);">183</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="184" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 184, true);">184</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="185" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 185, true);">185</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="186" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 186, true);">186</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="187" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 187, true);">187</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="188" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 188, true);">188</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="189" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 189, true);">189</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="190" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 190, true);">190</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="191" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 191, true);">191</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="192" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 192, true);">192</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="193" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 193, true);">193</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="194" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 194, true);">194</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="195" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 195, true);">195</a>
+<a class="line-number" data-cell="forward_and_backward_no_kernel" data-line="196" href="#cell-forward_and_backward_no_kernel" onclick="event.preventDefault(); selectCellLine('forward_and_backward_no_kernel', 196, true);">196</a>
 </div>
 <div class="code-wrap">
 <div class="highlight"><pre><span></span><span class="c1"># /// script</span>
@@ -3866,7 +3957,7 @@ Cell: forward_only | 17.22s | FAILED
 <span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
 <span class="kn">from</span><span class="w"> </span><span class="nn">transformers.models.gpt_oss.modeling_gpt_oss</span><span class="w"> </span><span class="kn">import</span> <span class="n">GptOssRMSNorm</span>
 
-
+<span class="c1"># remove liger kernel for testing </span>
 <span class="n">replace_kernel_forward_from_hub</span><span class="p">(</span><span class="n">GptOssRMSNorm</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
 
 <span class="c1"># set to debug logging</span>
@@ -3907,13 +3998,11 @@ Cell: forward_only | 17.22s | FAILED
 <span class="n">tokenizer</span> <span class="o">=</span> <span class="n">PreTrainedTokenizerFast</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span><span class="n">model_id</span><span class="p">)</span>
 <span class="n">quantization_config</span> <span class="o">=</span> <span class="n">Mxfp4Config</span><span class="p">(</span><span class="n">dequantize</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
 
-
-
 <span class="n">model</span> <span class="o">=</span> <span class="n">GptOssForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
     <span class="n">model_id</span><span class="p">,</span>
     <span class="n">dtype</span><span class="o">=</span><span class="s2">&quot;bfloat16&quot;</span><span class="p">,</span>
     <span class="n">device_map</span><span class="o">=</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span>
-    <span class="n">use_kernels</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+    <span class="n">use_kernels</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="n">quantization_config</span><span class="o">=</span><span class="n">quantization_config</span><span class="p">,</span>
 <span class="p">)</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
 
@@ -3929,9 +4018,14 @@ Cell: forward_only | 17.22s | FAILED
     <span class="n">reasoning_effort</span><span class="o">=</span><span class="s2">&quot;low&quot;</span><span class="p">,</span>
 <span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="s2">&quot;cuda&quot;</span><span class="p">)</span>
 
-<span class="n">max_tokens</span> <span class="o">=</span> <span class="mi">256</span>
+<span class="n">max_tokens</span> <span class="o">=</span> <span class="mi">128</span>  <span class="c1"># Reduced to help with memory usage</span>
 
-<span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">inference_mode</span><span class="p">():</span>
+<span class="c1"># Clear memory before backward pass</span>
+<span class="n">reset_peak_memory_stats</span><span class="p">()</span>
+<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Pre-generation memory: </span><span class="si">{</span><span class="n">get_memory_stats</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+<span class="c1"># forward and backward pass</span>
+<span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">set_grad_enabled</span><span class="p">(</span><span class="kc">True</span><span class="p">):</span>
     <span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">perf_counter</span><span class="p">()</span>
     <span class="n">generated</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">generate</span><span class="p">(</span>
         <span class="o">**</span><span class="n">inputs</span><span class="p">,</span>
@@ -3940,16 +4034,107 @@ Cell: forward_only | 17.22s | FAILED
         <span class="n">temperature</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="p">)</span>
     <span class="n">end_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">perf_counter</span><span class="p">()</span>
-
-<span class="nb">print</span><span class="p">(</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">generated</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
-<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Generation took </span><span class="si">{</span><span class="n">end_time</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">start_time</span><span class="si">:</span><span class="s2">.2f</span><span class="si">}</span><span class="s2"> seconds&quot;</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">generated</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Generation took </span><span class="si">{</span><span class="n">end_time</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">start_time</span><span class="si">:</span><span class="s2">.2f</span><span class="si">}</span><span class="s2"> seconds&quot;</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Post-generation memory: </span><span class="si">{</span><span class="n">get_memory_stats</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+    <span class="c1"># Use gradient checkpointing to reduce memory usage</span>
+    <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="s1">&#39;gradient_checkpointing_enable&#39;</span><span class="p">):</span>
+        <span class="n">model</span><span class="o">.</span><span class="n">gradient_checkpointing_enable</span><span class="p">()</span>
+        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Enabled gradient checkpointing&quot;</span><span class="p">)</span>
+
+    <span class="c1"># Reduce sequence length if needed for memory</span>
+    <span class="n">max_seq_len</span> <span class="o">=</span> <span class="mi">512</span>  <span class="c1"># Limit sequence length for backward pass</span>
+    <span class="k">if</span> <span class="n">generated</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">max_seq_len</span><span class="p">:</span>
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Truncating sequence from </span><span class="si">{</span><span class="n">generated</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">max_seq_len</span><span class="si">}</span><span class="s2"> tokens&quot;</span><span class="p">)</span>
+        <span class="n">full_sequence</span> <span class="o">=</span> <span class="n">generated</span><span class="p">[:,</span> <span class="o">-</span><span class="n">max_seq_len</span><span class="p">:]</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">full_sequence</span> <span class="o">=</span> <span class="n">generated</span>
+
+    <span class="c1"># Get model outputs for the full sequence</span>
+    <span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>  <span class="c1"># Enable dropout and other training behaviors</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">outputs</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span>
+            <span class="n">input_ids</span><span class="o">=</span><span class="n">full_sequence</span><span class="p">,</span>
+            <span class="n">labels</span><span class="o">=</span><span class="n">full_sequence</span><span class="p">,</span>  <span class="c1"># This will compute loss internally</span>
+            <span class="n">return_dict</span><span class="o">=</span><span class="kc">True</span>
+        <span class="p">)</span>
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Post-forward memory: </span><span class="si">{</span><span class="n">get_memory_stats</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+        <span class="c1"># If model doesn&#39;t compute loss, compute it manually</span>
+        <span class="k">if</span> <span class="n">outputs</span><span class="o">.</span><span class="n">loss</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">shift_logits</span> <span class="o">=</span> <span class="n">outputs</span><span class="o">.</span><span class="n">logits</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="p">:]</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span>
+            <span class="n">shift_labels</span> <span class="o">=</span> <span class="n">full_sequence</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">1</span><span class="p">:]</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span>
+
+            <span class="c1"># Use CrossEntropyLoss with ignore_index for padding tokens</span>
+            <span class="n">loss_fct</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">CrossEntropyLoss</span><span class="p">(</span><span class="n">ignore_index</span><span class="o">=</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">pad_token_id</span> <span class="k">if</span> <span class="n">tokenizer</span><span class="o">.</span><span class="n">pad_token_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="o">-</span><span class="mi">100</span><span class="p">)</span>
+            <span class="n">loss</span> <span class="o">=</span> <span class="n">loss_fct</span><span class="p">(</span>
+                <span class="n">shift_logits</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">shift_logits</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)),</span>
+                <span class="n">shift_labels</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">=</span> <span class="n">outputs</span><span class="o">.</span><span class="n">loss</span>
+
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Loss: </span><span class="si">{</span><span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span><span class="si">:</span><span class="s2">.4f</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+        <span class="c1"># Clear intermediate tensors to save memory</span>
+        <span class="k">del</span> <span class="n">outputs</span>
+        <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">empty_cache</span><span class="p">()</span>
+
+        <span class="c1"># Perform backward pass with memory management</span>
+        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Running backward pass...&quot;</span><span class="p">)</span>
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Pre-backward memory: </span><span class="si">{</span><span class="n">get_memory_stats</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+        <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Post-backward memory: </span><span class="si">{</span><span class="n">get_memory_stats</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+    <span class="k">except</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">OutOfMemoryError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;OOM during forward/backward pass: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Try reducing max_tokens or max_seq_len&quot;</span><span class="p">)</span>
+        <span class="k">raise</span>
+
+    <span class="c1"># Calculate gradient statistics and print sample gradients</span>
+    <span class="n">total_norm</span> <span class="o">=</span> <span class="mf">0.0</span>
+    <span class="n">param_count</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="n">grad_samples</span> <span class="o">=</span> <span class="p">{}</span>
+
+    <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
+        <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">grad</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">param_count</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="n">grad_norm</span> <span class="o">=</span> <span class="n">p</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+            <span class="n">total_norm</span> <span class="o">+=</span> <span class="n">grad_norm</span> <span class="o">**</span> <span class="mi">2</span>
+
+            <span class="c1"># Collect gradient statistics for key layers</span>
+            <span class="k">if</span> <span class="nb">any</span><span class="p">(</span><span class="n">key</span> <span class="ow">in</span> <span class="n">name</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;embed&#39;</span><span class="p">,</span> <span class="s1">&#39;lm_head&#39;</span><span class="p">,</span> <span class="s1">&#39;mlp.up&#39;</span><span class="p">,</span> <span class="s1">&#39;mlp.down&#39;</span><span class="p">,</span> <span class="s1">&#39;self_attn.q_proj&#39;</span><span class="p">,</span> <span class="s1">&#39;norm&#39;</span><span class="p">]):</span>
+                <span class="n">grad_samples</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
+                    <span class="s1">&#39;norm&#39;</span><span class="p">:</span> <span class="n">grad_norm</span><span class="p">,</span>
+                    <span class="s1">&#39;mean&#39;</span><span class="p">:</span> <span class="n">p</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span>
+                    <span class="s1">&#39;std&#39;</span><span class="p">:</span> <span class="n">p</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">std</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span>
+                    <span class="s1">&#39;max&#39;</span><span class="p">:</span> <span class="n">p</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">max</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span>
+                    <span class="s1">&#39;min&#39;</span><span class="p">:</span> <span class="n">p</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">min</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span>
+                <span class="p">}</span>
+
+    <span class="n">total_norm</span> <span class="o">=</span> <span class="n">total_norm</span> <span class="o">**</span> <span class="mf">0.5</span>
+
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Gradient norm: </span><span class="si">{</span><span class="n">total_norm</span><span class="si">:</span><span class="s2">.4f</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Parameters with gradients: </span><span class="si">{</span><span class="n">param_count</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+    <span class="c1"># Print sample gradients from important layers</span>
+    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Sample gradient statistics:&quot;</span><span class="p">)</span>
+    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">stats</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">grad_samples</span><span class="o">.</span><span class="n">items</span><span class="p">())[:</span><span class="mi">10</span><span class="p">]):</span>
+        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;  </span><span class="si">{</span><span class="n">name</span><span class="p">[:</span><span class="mi">60</span><span class="p">]</span><span class="si">:</span><span class="s2">&lt;60</span><span class="si">}</span><span class="s2"> | norm: </span><span class="si">{</span><span class="n">stats</span><span class="p">[</span><span class="s1">&#39;norm&#39;</span><span class="p">]</span><span class="si">:</span><span class="s2">.4e</span><span class="si">}</span><span class="s2"> | mean: </span><span class="si">{</span><span class="n">stats</span><span class="p">[</span><span class="s1">&#39;mean&#39;</span><span class="p">]</span><span class="si">:</span><span class="s2">.4e</span><span class="si">}</span><span class="s2"> | std: </span><span class="si">{</span><span class="n">stats</span><span class="p">[</span><span class="s1">&#39;std&#39;</span><span class="p">]</span><span class="si">:</span><span class="s2">.4e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+    <span class="c1"># Optional: zero gradients for next iteration</span>
+    <span class="n">model</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
+    <span class="n">model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>  <span class="c1"># Switch back to eval mode</span>
 </pre></div>
 
-<div class="code-line-highlight" id="line-highlight-forward_only"></div>
+<div class="code-line-highlight" id="line-highlight-forward_and_backward_no_kernel"></div>
 </div>
 </div>
 </div>
-<div id="output-forward_only" class="cell-output">
+<div id="output-forward_and_backward_no_kernel" class="cell-output">
 <div class="cell-stderr">warning: The requested interpreter resolved to Python 3.11.13, which is incompatible with the script&#x27;s Python requirement: `&gt;=3.12`
    Updating https://github.com/huggingface/transformers.git (HEAD)
     Updated https://github.com/huggingface/transformers.git (449533af73874470e914a203391635e04ac2ffc8)
@@ -3967,6 +4152,10 @@ Cell: forward_only | 17.22s | FAILED
 </div>
 </div>
 
+<h1>Kernels</h1>
+<p>Next we can run with Megablocks kernels enabled.</p>
+<h3>Forward</h3>
+<p>First, we run a forward pass with Megablocks kernels.</p>
 <h2>Forward and Backward</h2>
 <p>Next, we run a forward and backward pass with Megablocks kernels enabled. This should be more memory efficient and allow us to complete the backward pass without running out of memory.</p>
     </div>

moe_benchmarks/megablocks_yamoe/artifacts/binned_run/binned_results.json

@@ -9,16 +9,16 @@
     "vary_inputs": true
   },
   "stats": {
-    "avg_ms": 36.539247979999345,
-    "min_ms": 32.831213000008574,
-    "max_ms": 40.07397899999887,
-    "std_ms": 1.6136977896039295,
-    "p50_ms": 36.86953950000316,
-    "p95_ms": 39.00453930000651,
-    "p99_ms": 39.94982966000805,
+    "avg_ms": 36.727162420011155,
+    "min_ms": 33.18469400005597,
+    "max_ms": 39.16655900002297,
+    "std_ms": 1.746945665597604,
+    "p50_ms": 36.9490834999624,
+    "p95_ms": 38.664125449997755,
+    "p99_ms": 38.97265620000894,
     "num_iters": 50,
-    "tokens_per_s": 2736.7831996634814,
-    "throughput_variance": 122.96225627695094
+    "tokens_per_s": 2722.7804548688473,
+    "throughput_variance": 132.97041596249102
   },
   "output_sum": 3.97190523147583
 }

moe_benchmarks/megablocks_yamoe/artifacts/gptoss_run/gptoss_results.json

@@ -9,16 +9,16 @@
     "vary_inputs": true
   },
   "stats": {
-    "avg_ms": 45.23668440000279,
-    "min_ms": 39.77574300000697,
-    "max_ms": 49.24737599998252,
-    "std_ms": 2.8509890347901194,
-    "p50_ms": 46.46113100000093,
-    "p95_ms": 48.43337355002859,
-    "p99_ms": 49.07736706999401,
+    "avg_ms": 45.42313148000403,
+    "min_ms": 38.7863710000147,
+    "max_ms": 51.6831769999726,
+    "std_ms": 3.6706212937228724,
+    "p50_ms": 45.48181749999003,
+    "p95_ms": 50.886562249957024,
+    "p99_ms": 51.415023030010616,
     "num_iters": 50,
-    "tokens_per_s": 2210.595257507286,
-    "throughput_variance": 143.00506086156602
+    "tokens_per_s": 2201.521487879398,
+    "throughput_variance": 179.47325216420964
   },
   "output_sum": 11.53223705291748
 }

moe_benchmarks/megablocks_yamoe/artifacts/gptoss_training_run/gptoss_training_results.json

@@ -9,16 +9,16 @@
     "vary_inputs": true
   },
   "stats": {
-    "avg_ms": 45.75117022000086,
-    "min_ms": 38.86002600000893,
-    "max_ms": 50.81734300000562,
-    "std_ms": 2.8335743767450845,
-    "p50_ms": 45.83255949998488,
-    "p95_ms": 50.53969424997433,
-    "p99_ms": 50.77732372002288,
+    "avg_ms": 47.223061700003655,
+    "min_ms": 40.87468699992769,
+    "max_ms": 51.39806599993335,
+    "std_ms": 3.1109522065149875,
+    "p50_ms": 47.68390750001572,
+    "p95_ms": 51.25916855005812,
+    "p99_ms": 51.36193830001844,
     "num_iters": 50,
-    "tokens_per_s": 2185.7364416939745,
-    "throughput_variance": 141.0139740289509
+    "tokens_per_s": 2117.6094137071223,
+    "throughput_variance": 143.64050741882156
   },
   "output_sum": 11.53223705291748
 }

moe_benchmarks/megablocks_yamoe/artifacts/megablocks_run/megablocks_results.json

@@ -9,16 +9,16 @@
     "vary_inputs": true
   },
   "stats": {
-    "avg_ms": 3.8699466199977905,
-    "min_ms": 0.810037999940505,
-    "max_ms": 8.541234000063014,
-    "std_ms": 3.7276327220093757,
-    "p50_ms": 0.840403500035336,
-    "p95_ms": 8.54010504996836,
-    "p99_ms": 8.54096940002023,
+    "avg_ms": 3.829988559998583,
+    "min_ms": 0.8193480000500131,
+    "max_ms": 8.440041999961068,
+    "std_ms": 3.6590563761858443,
+    "p50_ms": 0.8861204999561778,
+    "p95_ms": 8.437156600047047,
+    "p99_ms": 8.439223699980403,
     "num_iters": 50,
-    "tokens_per_s": 25840.149702131315,
-    "throughput_variance": 53236.043048659514
+    "tokens_per_s": 26109.738562779676,
+    "throughput_variance": 51610.70870547587
   },
   "output_sum": 6.4738850593566895
 }

moe_benchmarks/megablocks_yamoe/artifacts/yamoe_run/yamoe_results.json

@@ -9,16 +9,16 @@
     "vary_inputs": true
   },
   "stats": {
-    "avg_ms": 4.249726199998349,
-    "min_ms": 4.124869000008857,
-    "max_ms": 4.300293000028432,
-    "std_ms": 0.0233051162040199,
-    "p50_ms": 4.253981000005069,
-    "p95_ms": 4.2678174999991825,
-    "p99_ms": 4.289845220023949,
+    "avg_ms": 4.248629200003506,
+    "min_ms": 4.131733000008353,
+    "max_ms": 4.277987000023131,
+    "std_ms": 0.02059489131384697,
+    "p50_ms": 4.252681999957986,
+    "p95_ms": 4.26323250002838,
+    "p99_ms": 4.27199430000428,
     "num_iters": 50,
-    "tokens_per_s": 23530.927710128442,
-    "throughput_variance": 131.31218448800846
+    "tokens_per_s": 23537.003417459324,
+    "throughput_variance": 116.25267146840179
   },
   "output_sum": 3.97190523147583
 }

moe_benchmarks/megablocks_yamoe/cells/setup2.py

@@ -0,0 +1,115 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "accelerate>=1.10.1",
+#     "torch>=2.7.0",
+#     "kernels==0.10.0",
+#     "transformers@https://github.com/huggingface/transformers.git",
+#     "ipdb>=0.13.13",
+#     "matplotlib>=3.7.2",
+#     "numpy>=1.24.3",
+# ]
+# ///
+
+import torch
+from transformers import GptOssForCausalLM, PreTrainedTokenizerFast, Mxfp4Config
+import time
+import torch.nn as nn
+from kernels import register_kernel_mapping, Mode, LayerRepository
+import sys
+import torch.profiler
+import gc
+import logging
+
+# set to debug logging
+logging.basicConfig(level=logging.INFO)
+
+def reset_peak_memory_stats():
+    """Clear CUDA cache and reset memory allocation counters."""
+    torch.cuda.empty_cache()
+    if torch.cuda.is_available():
+        torch.cuda.reset_peak_memory_stats()
+    gc.collect()
+
+def get_memory_stats():
+    """Get current and peak CUDA memory usage."""
+    if not torch.cuda.is_available():
+        return {"allocated_gb": 0, "peak_gb": 0, "reserved_gb": 0}
+    return {
+        "allocated_gb": torch.cuda.memory_allocated() / 1e9,
+        "peak_gb": torch.cuda.max_memory_allocated() / 1e9,
+        "reserved_gb": torch.cuda.memory_reserved() / 1e9,
+    }
+
+def override_kernel_layer_name(cls_name: str, value) -> bool:
+    """Helper to dynamically override the kernel_layer_name in a model class."""
+    for mod in sys.modules.values():
+        if mod is None:
+            continue
+        obj = getattr(mod, cls_name, None)
+        if isinstance(obj, type) and issubclass(obj, nn.Module):
+            setattr(obj, "kernel_layer_name", value)
+            print(f"Overrode {cls_name}.kernel_layer_name to {value}")
+            return True
+    return False
+
+
+# Init the model the normal way
+model_id = "openai/gpt-oss-20b"
+tokenizer = PreTrainedTokenizerFast.from_pretrained(model_id)
+quantization_config = Mxfp4Config(dequantize=True)
+
+
+from kernels import replace_kernel_forward_from_hub, register_kernel_mapping, LayerRepository, Mode
+
+from transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP, GptOssRMSNorm
+
+replace_kernel_forward_from_hub(GptOssRMSNorm, None)  # direct, type-safe
+custom_mapping = {
+    "Yamoe": {
+        "cuda": {
+            Mode.INFERENCE: LayerRepository(
+                repo_id="drbh/yamoe",
+                layer_name="Yamoe",
+                revision="v0.3.0",
+            )
+        }
+    }
+}
+register_kernel_mapping(custom_mapping)
+
+
+model = GptOssForCausalLM.from_pretrained(
+    model_id,
+    dtype="bfloat16",
+    device_map="auto",
+    use_kernels=True,
+    quantization_config=quantization_config,
+).eval()
+
+messages = [
+    {"role": "system", "content": "What is Tensor Parallelism?"},
+]
+
+inputs = tokenizer.apply_chat_template(
+    messages,
+    add_generation_prompt=True,
+    return_tensors="pt",
+    return_dict=True,
+    reasoning_effort="low",
+).to("cuda")
+
+max_tokens = 256
+
+with torch.inference_mode():
+    start_time = time.perf_counter()
+    generated = model.generate(
+        **inputs,
+        max_new_tokens=max_tokens,
+        do_sample=False,
+        temperature=None,
+    )
+    end_time = time.perf_counter()
+
+print(tokenizer.decode(generated[0], skip_special_tokens=False))
+print(f"Generation took {end_time - start_time:.2f} seconds")

moe_benchmarks/megablocks_yamoe/megablocks_yamoe.html

@@ -3718,138 +3718,136 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
         <h1>Comparison of Megablocks and Yamoe Kernels</h1>
 <p>This note compares the performance of the Megablocks and Yamoe kernels on the GPT-OSS-20B model.</p>
 <h2>Megablocks kernel</h2>
-<h2>Yamoe Kernel</h2>
-<div class="cell cell-failed" id="cell-setup">
+<div class="cell cell-failed" id="cell-setup2">
 <div class="cell-header">
 <span class="collapse-indicators">
-<span onclick="toggleCode('setup')" style="cursor: pointer;">▼ code</span> 
-<span onclick="toggleOutput('setup')" style="cursor: pointer;">▼ output</span>
- <span id="uv-indicator-setup" style="cursor: default; opacity: 0.3;">▶ uv-logs</span>
+<span onclick="toggleCode('setup2')" style="cursor: pointer;">▼ code</span> 
+<span onclick="toggleOutput('setup2')" style="cursor: pointer;">▼ output</span>
+ <span id="uv-indicator-setup2" style="cursor: default; opacity: 0.3;">▶ uv-logs</span>
 </span> | 
-Cell: setup | 17.01s | FAILED
- | <button class="run-btn" onclick="runCell('setup')">▶ run</button>
-<button class="copy-btn" onclick="copyCell('setup')">Copy</button>
-<a href="cells/setup.py" target="_blank" class="raw-btn">Raw</a>
+Cell: setup2 | 16.98s | FAILED
+ | <button class="run-btn" onclick="runCell('setup2')">▶ run</button>
+<button class="copy-btn" onclick="copyCell('setup2')">Copy</button>
+<a href="cells/setup2.py" target="_blank" class="raw-btn">Raw</a>
 </div>
-<div id="code-setup" class="cell-code" data-lines="116">
+<div id="code-setup2" class="cell-code" data-lines="115">
 <div class="highlight-with-lines">
-<div class="line-numbers" id="lines-setup">
-<a class="line-number" data-cell="setup" data-line="1" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 1, true);">1</a>
-<a class="line-number" data-cell="setup" data-line="2" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 2, true);">2</a>
-<a class="line-number" data-cell="setup" data-line="3" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 3, true);">3</a>
-<a class="line-number" data-cell="setup" data-line="4" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 4, true);">4</a>
-<a class="line-number" data-cell="setup" data-line="5" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 5, true);">5</a>
-<a class="line-number" data-cell="setup" data-line="6" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 6, true);">6</a>
-<a class="line-number" data-cell="setup" data-line="7" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 7, true);">7</a>
-<a class="line-number" data-cell="setup" data-line="8" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 8, true);">8</a>
-<a class="line-number" data-cell="setup" data-line="9" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 9, true);">9</a>
-<a class="line-number" data-cell="setup" data-line="10" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 10, true);">10</a>
-<a class="line-number" data-cell="setup" data-line="11" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 11, true);">11</a>
-<a class="line-number" data-cell="setup" data-line="12" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 12, true);">12</a>
-<a class="line-number" data-cell="setup" data-line="13" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 13, true);">13</a>
-<a class="line-number" data-cell="setup" data-line="14" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 14, true);">14</a>
-<a class="line-number" data-cell="setup" data-line="15" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 15, true);">15</a>
-<a class="line-number" data-cell="setup" data-line="16" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 16, true);">16</a>
-<a class="line-number" data-cell="setup" data-line="17" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 17, true);">17</a>
-<a class="line-number" data-cell="setup" data-line="18" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 18, true);">18</a>
-<a class="line-number" data-cell="setup" data-line="19" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 19, true);">19</a>
-<a class="line-number" data-cell="setup" data-line="20" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 20, true);">20</a>
-<a class="line-number" data-cell="setup" data-line="21" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 21, true);">21</a>
-<a class="line-number" data-cell="setup" data-line="22" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 22, true);">22</a>
-<a class="line-number" data-cell="setup" data-line="23" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 23, true);">23</a>
-<a class="line-number" data-cell="setup" data-line="24" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 24, true);">24</a>
-<a class="line-number" data-cell="setup" data-line="25" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 25, true);">25</a>
-<a class="line-number" data-cell="setup" data-line="26" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 26, true);">26</a>
-<a class="line-number" data-cell="setup" data-line="27" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 27, true);">27</a>
-<a class="line-number" data-cell="setup" data-line="28" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 28, true);">28</a>
-<a class="line-number" data-cell="setup" data-line="29" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 29, true);">29</a>
-<a class="line-number" data-cell="setup" data-line="30" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 30, true);">30</a>
-<a class="line-number" data-cell="setup" data-line="31" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 31, true);">31</a>
-<a class="line-number" data-cell="setup" data-line="32" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 32, true);">32</a>
-<a class="line-number" data-cell="setup" data-line="33" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 33, true);">33</a>
-<a class="line-number" data-cell="setup" data-line="34" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 34, true);">34</a>
-<a class="line-number" data-cell="setup" data-line="35" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 35, true);">35</a>
-<a class="line-number" data-cell="setup" data-line="36" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 36, true);">36</a>
-<a class="line-number" data-cell="setup" data-line="37" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 37, true);">37</a>
-<a class="line-number" data-cell="setup" data-line="38" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 38, true);">38</a>
-<a class="line-number" data-cell="setup" data-line="39" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 39, true);">39</a>
-<a class="line-number" data-cell="setup" data-line="40" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 40, true);">40</a>
-<a class="line-number" data-cell="setup" data-line="41" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 41, true);">41</a>
-<a class="line-number" data-cell="setup" data-line="42" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 42, true);">42</a>
-<a class="line-number" data-cell="setup" data-line="43" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 43, true);">43</a>
-<a class="line-number" data-cell="setup" data-line="44" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 44, true);">44</a>
-<a class="line-number" data-cell="setup" data-line="45" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 45, true);">45</a>
-<a class="line-number" data-cell="setup" data-line="46" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 46, true);">46</a>
-<a class="line-number" data-cell="setup" data-line="47" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 47, true);">47</a>
-<a class="line-number" data-cell="setup" data-line="48" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 48, true);">48</a>
-<a class="line-number" data-cell="setup" data-line="49" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 49, true);">49</a>
-<a class="line-number" data-cell="setup" data-line="50" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 50, true);">50</a>
-<a class="line-number" data-cell="setup" data-line="51" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 51, true);">51</a>
-<a class="line-number" data-cell="setup" data-line="52" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 52, true);">52</a>
-<a class="line-number" data-cell="setup" data-line="53" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 53, true);">53</a>
-<a class="line-number" data-cell="setup" data-line="54" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 54, true);">54</a>
-<a class="line-number" data-cell="setup" data-line="55" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 55, true);">55</a>
-<a class="line-number" data-cell="setup" data-line="56" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 56, true);">56</a>
-<a class="line-number" data-cell="setup" data-line="57" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 57, true);">57</a>
-<a class="line-number" data-cell="setup" data-line="58" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 58, true);">58</a>
-<a class="line-number" data-cell="setup" data-line="59" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 59, true);">59</a>
-<a class="line-number" data-cell="setup" data-line="60" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 60, true);">60</a>
-<a class="line-number" data-cell="setup" data-line="61" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 61, true);">61</a>
-<a class="line-number" data-cell="setup" data-line="62" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 62, true);">62</a>
-<a class="line-number" data-cell="setup" data-line="63" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 63, true);">63</a>
-<a class="line-number" data-cell="setup" data-line="64" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 64, true);">64</a>
-<a class="line-number" data-cell="setup" data-line="65" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 65, true);">65</a>
-<a class="line-number" data-cell="setup" data-line="66" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 66, true);">66</a>
-<a class="line-number" data-cell="setup" data-line="67" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 67, true);">67</a>
-<a class="line-number" data-cell="setup" data-line="68" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 68, true);">68</a>
-<a class="line-number" data-cell="setup" data-line="69" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 69, true);">69</a>
-<a class="line-number" data-cell="setup" data-line="70" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 70, true);">70</a>
-<a class="line-number" data-cell="setup" data-line="71" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 71, true);">71</a>
-<a class="line-number" data-cell="setup" data-line="72" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 72, true);">72</a>
-<a class="line-number" data-cell="setup" data-line="73" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 73, true);">73</a>
-<a class="line-number" data-cell="setup" data-line="74" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 74, true);">74</a>
-<a class="line-number" data-cell="setup" data-line="75" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 75, true);">75</a>
-<a class="line-number" data-cell="setup" data-line="76" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 76, true);">76</a>
-<a class="line-number" data-cell="setup" data-line="77" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 77, true);">77</a>
-<a class="line-number" data-cell="setup" data-line="78" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 78, true);">78</a>
-<a class="line-number" data-cell="setup" data-line="79" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 79, true);">79</a>
-<a class="line-number" data-cell="setup" data-line="80" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 80, true);">80</a>
-<a class="line-number" data-cell="setup" data-line="81" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 81, true);">81</a>
-<a class="line-number" data-cell="setup" data-line="82" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 82, true);">82</a>
-<a class="line-number" data-cell="setup" data-line="83" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 83, true);">83</a>
-<a class="line-number" data-cell="setup" data-line="84" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 84, true);">84</a>
-<a class="line-number" data-cell="setup" data-line="85" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 85, true);">85</a>
-<a class="line-number" data-cell="setup" data-line="86" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 86, true);">86</a>
-<a class="line-number" data-cell="setup" data-line="87" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 87, true);">87</a>
-<a class="line-number" data-cell="setup" data-line="88" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 88, true);">88</a>
-<a class="line-number" data-cell="setup" data-line="89" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 89, true);">89</a>
-<a class="line-number" data-cell="setup" data-line="90" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 90, true);">90</a>
-<a class="line-number" data-cell="setup" data-line="91" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 91, true);">91</a>
-<a class="line-number" data-cell="setup" data-line="92" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 92, true);">92</a>
-<a class="line-number" data-cell="setup" data-line="93" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 93, true);">93</a>
-<a class="line-number" data-cell="setup" data-line="94" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 94, true);">94</a>
-<a class="line-number" data-cell="setup" data-line="95" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 95, true);">95</a>
-<a class="line-number" data-cell="setup" data-line="96" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 96, true);">96</a>
-<a class="line-number" data-cell="setup" data-line="97" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 97, true);">97</a>
-<a class="line-number" data-cell="setup" data-line="98" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 98, true);">98</a>
-<a class="line-number" data-cell="setup" data-line="99" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 99, true);">99</a>
-<a class="line-number" data-cell="setup" data-line="100" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 100, true);">100</a>
-<a class="line-number" data-cell="setup" data-line="101" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 101, true);">101</a>
-<a class="line-number" data-cell="setup" data-line="102" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 102, true);">102</a>
-<a class="line-number" data-cell="setup" data-line="103" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 103, true);">103</a>
-<a class="line-number" data-cell="setup" data-line="104" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 104, true);">104</a>
-<a class="line-number" data-cell="setup" data-line="105" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 105, true);">105</a>
-<a class="line-number" data-cell="setup" data-line="106" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 106, true);">106</a>
-<a class="line-number" data-cell="setup" data-line="107" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 107, true);">107</a>
-<a class="line-number" data-cell="setup" data-line="108" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 108, true);">108</a>
-<a class="line-number" data-cell="setup" data-line="109" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 109, true);">109</a>
-<a class="line-number" data-cell="setup" data-line="110" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 110, true);">110</a>
-<a class="line-number" data-cell="setup" data-line="111" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 111, true);">111</a>
-<a class="line-number" data-cell="setup" data-line="112" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 112, true);">112</a>
-<a class="line-number" data-cell="setup" data-line="113" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 113, true);">113</a>
-<a class="line-number" data-cell="setup" data-line="114" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 114, true);">114</a>
-<a class="line-number" data-cell="setup" data-line="115" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 115, true);">115</a>
-<a class="line-number" data-cell="setup" data-line="116" href="#cell-setup" onclick="event.preventDefault(); selectCellLine('setup', 116, true);">116</a>
+<div class="line-numbers" id="lines-setup2">
+<a class="line-number" data-cell="setup2" data-line="1" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 1, true);">1</a>
+<a class="line-number" data-cell="setup2" data-line="2" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 2, true);">2</a>
+<a class="line-number" data-cell="setup2" data-line="3" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 3, true);">3</a>
+<a class="line-number" data-cell="setup2" data-line="4" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 4, true);">4</a>
+<a class="line-number" data-cell="setup2" data-line="5" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 5, true);">5</a>
+<a class="line-number" data-cell="setup2" data-line="6" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 6, true);">6</a>
+<a class="line-number" data-cell="setup2" data-line="7" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 7, true);">7</a>
+<a class="line-number" data-cell="setup2" data-line="8" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 8, true);">8</a>
+<a class="line-number" data-cell="setup2" data-line="9" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 9, true);">9</a>
+<a class="line-number" data-cell="setup2" data-line="10" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 10, true);">10</a>
+<a class="line-number" data-cell="setup2" data-line="11" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 11, true);">11</a>
+<a class="line-number" data-cell="setup2" data-line="12" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 12, true);">12</a>
+<a class="line-number" data-cell="setup2" data-line="13" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 13, true);">13</a>
+<a class="line-number" data-cell="setup2" data-line="14" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 14, true);">14</a>
+<a class="line-number" data-cell="setup2" data-line="15" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 15, true);">15</a>
+<a class="line-number" data-cell="setup2" data-line="16" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 16, true);">16</a>
+<a class="line-number" data-cell="setup2" data-line="17" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 17, true);">17</a>
+<a class="line-number" data-cell="setup2" data-line="18" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 18, true);">18</a>
+<a class="line-number" data-cell="setup2" data-line="19" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 19, true);">19</a>
+<a class="line-number" data-cell="setup2" data-line="20" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 20, true);">20</a>
+<a class="line-number" data-cell="setup2" data-line="21" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 21, true);">21</a>
+<a class="line-number" data-cell="setup2" data-line="22" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 22, true);">22</a>
+<a class="line-number" data-cell="setup2" data-line="23" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 23, true);">23</a>
+<a class="line-number" data-cell="setup2" data-line="24" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 24, true);">24</a>
+<a class="line-number" data-cell="setup2" data-line="25" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 25, true);">25</a>
+<a class="line-number" data-cell="setup2" data-line="26" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 26, true);">26</a>
+<a class="line-number" data-cell="setup2" data-line="27" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 27, true);">27</a>
+<a class="line-number" data-cell="setup2" data-line="28" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 28, true);">28</a>
+<a class="line-number" data-cell="setup2" data-line="29" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 29, true);">29</a>
+<a class="line-number" data-cell="setup2" data-line="30" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 30, true);">30</a>
+<a class="line-number" data-cell="setup2" data-line="31" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 31, true);">31</a>
+<a class="line-number" data-cell="setup2" data-line="32" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 32, true);">32</a>
+<a class="line-number" data-cell="setup2" data-line="33" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 33, true);">33</a>
+<a class="line-number" data-cell="setup2" data-line="34" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 34, true);">34</a>
+<a class="line-number" data-cell="setup2" data-line="35" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 35, true);">35</a>
+<a class="line-number" data-cell="setup2" data-line="36" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 36, true);">36</a>
+<a class="line-number" data-cell="setup2" data-line="37" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 37, true);">37</a>
+<a class="line-number" data-cell="setup2" data-line="38" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 38, true);">38</a>
+<a class="line-number" data-cell="setup2" data-line="39" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 39, true);">39</a>
+<a class="line-number" data-cell="setup2" data-line="40" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 40, true);">40</a>
+<a class="line-number" data-cell="setup2" data-line="41" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 41, true);">41</a>
+<a class="line-number" data-cell="setup2" data-line="42" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 42, true);">42</a>
+<a class="line-number" data-cell="setup2" data-line="43" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 43, true);">43</a>
+<a class="line-number" data-cell="setup2" data-line="44" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 44, true);">44</a>
+<a class="line-number" data-cell="setup2" data-line="45" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 45, true);">45</a>
+<a class="line-number" data-cell="setup2" data-line="46" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 46, true);">46</a>
+<a class="line-number" data-cell="setup2" data-line="47" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 47, true);">47</a>
+<a class="line-number" data-cell="setup2" data-line="48" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 48, true);">48</a>
+<a class="line-number" data-cell="setup2" data-line="49" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 49, true);">49</a>
+<a class="line-number" data-cell="setup2" data-line="50" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 50, true);">50</a>
+<a class="line-number" data-cell="setup2" data-line="51" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 51, true);">51</a>
+<a class="line-number" data-cell="setup2" data-line="52" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 52, true);">52</a>
+<a class="line-number" data-cell="setup2" data-line="53" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 53, true);">53</a>
+<a class="line-number" data-cell="setup2" data-line="54" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 54, true);">54</a>
+<a class="line-number" data-cell="setup2" data-line="55" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 55, true);">55</a>
+<a class="line-number" data-cell="setup2" data-line="56" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 56, true);">56</a>
+<a class="line-number" data-cell="setup2" data-line="57" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 57, true);">57</a>
+<a class="line-number" data-cell="setup2" data-line="58" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 58, true);">58</a>
+<a class="line-number" data-cell="setup2" data-line="59" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 59, true);">59</a>
+<a class="line-number" data-cell="setup2" data-line="60" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 60, true);">60</a>
+<a class="line-number" data-cell="setup2" data-line="61" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 61, true);">61</a>
+<a class="line-number" data-cell="setup2" data-line="62" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 62, true);">62</a>
+<a class="line-number" data-cell="setup2" data-line="63" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 63, true);">63</a>
+<a class="line-number" data-cell="setup2" data-line="64" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 64, true);">64</a>
+<a class="line-number" data-cell="setup2" data-line="65" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 65, true);">65</a>
+<a class="line-number" data-cell="setup2" data-line="66" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 66, true);">66</a>
+<a class="line-number" data-cell="setup2" data-line="67" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 67, true);">67</a>
+<a class="line-number" data-cell="setup2" data-line="68" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 68, true);">68</a>
+<a class="line-number" data-cell="setup2" data-line="69" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 69, true);">69</a>
+<a class="line-number" data-cell="setup2" data-line="70" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 70, true);">70</a>
+<a class="line-number" data-cell="setup2" data-line="71" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 71, true);">71</a>
+<a class="line-number" data-cell="setup2" data-line="72" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 72, true);">72</a>
+<a class="line-number" data-cell="setup2" data-line="73" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 73, true);">73</a>
+<a class="line-number" data-cell="setup2" data-line="74" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 74, true);">74</a>
+<a class="line-number" data-cell="setup2" data-line="75" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 75, true);">75</a>
+<a class="line-number" data-cell="setup2" data-line="76" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 76, true);">76</a>
+<a class="line-number" data-cell="setup2" data-line="77" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 77, true);">77</a>
+<a class="line-number" data-cell="setup2" data-line="78" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 78, true);">78</a>
+<a class="line-number" data-cell="setup2" data-line="79" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 79, true);">79</a>
+<a class="line-number" data-cell="setup2" data-line="80" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 80, true);">80</a>
+<a class="line-number" data-cell="setup2" data-line="81" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 81, true);">81</a>
+<a class="line-number" data-cell="setup2" data-line="82" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 82, true);">82</a>
+<a class="line-number" data-cell="setup2" data-line="83" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 83, true);">83</a>
+<a class="line-number" data-cell="setup2" data-line="84" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 84, true);">84</a>
+<a class="line-number" data-cell="setup2" data-line="85" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 85, true);">85</a>
+<a class="line-number" data-cell="setup2" data-line="86" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 86, true);">86</a>
+<a class="line-number" data-cell="setup2" data-line="87" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 87, true);">87</a>
+<a class="line-number" data-cell="setup2" data-line="88" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 88, true);">88</a>
+<a class="line-number" data-cell="setup2" data-line="89" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 89, true);">89</a>
+<a class="line-number" data-cell="setup2" data-line="90" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 90, true);">90</a>
+<a class="line-number" data-cell="setup2" data-line="91" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 91, true);">91</a>
+<a class="line-number" data-cell="setup2" data-line="92" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 92, true);">92</a>
+<a class="line-number" data-cell="setup2" data-line="93" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 93, true);">93</a>
+<a class="line-number" data-cell="setup2" data-line="94" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 94, true);">94</a>
+<a class="line-number" data-cell="setup2" data-line="95" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 95, true);">95</a>
+<a class="line-number" data-cell="setup2" data-line="96" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 96, true);">96</a>
+<a class="line-number" data-cell="setup2" data-line="97" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 97, true);">97</a>
+<a class="line-number" data-cell="setup2" data-line="98" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 98, true);">98</a>
+<a class="line-number" data-cell="setup2" data-line="99" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 99, true);">99</a>
+<a class="line-number" data-cell="setup2" data-line="100" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 100, true);">100</a>
+<a class="line-number" data-cell="setup2" data-line="101" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 101, true);">101</a>
+<a class="line-number" data-cell="setup2" data-line="102" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 102, true);">102</a>
+<a class="line-number" data-cell="setup2" data-line="103" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 103, true);">103</a>
+<a class="line-number" data-cell="setup2" data-line="104" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 104, true);">104</a>
+<a class="line-number" data-cell="setup2" data-line="105" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 105, true);">105</a>
+<a class="line-number" data-cell="setup2" data-line="106" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 106, true);">106</a>
+<a class="line-number" data-cell="setup2" data-line="107" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 107, true);">107</a>
+<a class="line-number" data-cell="setup2" data-line="108" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 108, true);">108</a>
+<a class="line-number" data-cell="setup2" data-line="109" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 109, true);">109</a>
+<a class="line-number" data-cell="setup2" data-line="110" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 110, true);">110</a>
+<a class="line-number" data-cell="setup2" data-line="111" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 111, true);">111</a>
+<a class="line-number" data-cell="setup2" data-line="112" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 112, true);">112</a>
+<a class="line-number" data-cell="setup2" data-line="113" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 113, true);">113</a>
+<a class="line-number" data-cell="setup2" data-line="114" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 114, true);">114</a>
+<a class="line-number" data-cell="setup2" data-line="115" href="#cell-setup2" onclick="event.preventDefault(); selectCellLine('setup2', 115, true);">115</a>
 </div>
 <div class="code-wrap">
 <div class="highlight"><pre><span></span><span class="c1"># /// script</span>
@@ -3918,8 +3916,7 @@ Cell: setup | 17.01s | FAILED
 
 <span class="kn">from</span><span class="w"> </span><span class="nn">transformers.models.gpt_oss.modeling_gpt_oss</span><span class="w"> </span><span class="kn">import</span> <span class="n">GptOssMLP</span><span class="p">,</span> <span class="n">GptOssRMSNorm</span>
 
-<span class="n">replace_kernel_forward_from_hub</span><span class="p">(</span><span class="n">GptOssMLP</span><span class="p">,</span> <span class="s2">&quot;Yamoe&quot;</span><span class="p">)</span>
-<span class="n">replace_kernel_forward_from_hub</span><span class="p">(</span><span class="n">GptOssRMSNorm</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+<span class="n">replace_kernel_forward_from_hub</span><span class="p">(</span><span class="n">GptOssRMSNorm</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>  <span class="c1"># direct, type-safe</span>
 <span class="n">custom_mapping</span> <span class="o">=</span> <span class="p">{</span>
     <span class="s2">&quot;Yamoe&quot;</span><span class="p">:</span> <span class="p">{</span>
         <span class="s2">&quot;cuda&quot;</span><span class="p">:</span> <span class="p">{</span>
@@ -3970,11 +3967,11 @@ Cell: setup | 17.01s | FAILED
 <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Generation took </span><span class="si">{</span><span class="n">end_time</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">start_time</span><span class="si">:</span><span class="s2">.2f</span><span class="si">}</span><span class="s2"> seconds&quot;</span><span class="p">)</span>
 </pre></div>
 
-<div class="code-line-highlight" id="line-highlight-setup"></div>
+<div class="code-line-highlight" id="line-highlight-setup2"></div>
 </div>
 </div>
 </div>
-<div id="output-setup" class="cell-output">
+<div id="output-setup2" class="cell-output">
 <div class="cell-stderr">warning: The requested interpreter resolved to Python 3.11.13, which is incompatible with the script&#x27;s Python requirement: `&gt;=3.12`
    Updating https://github.com/huggingface/transformers.git (HEAD)
     Updated https://github.com/huggingface/transformers.git (449533af73874470e914a203391635e04ac2ffc8)
@@ -3991,6 +3988,8 @@ Cell: setup | 17.01s | FAILED
 </div>
 </div>
 </div>
+
+<h2>Yamoe Kernel</h2>
     </div>
     
 </body>

moe_benchmarks/megablocks_yamoe/torch_profile.html

@@ -3720,7 +3720,7 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
 <span onclick="toggleOutput('utils')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-utils" onclick="toggleUvLogsFromHeader('utils')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: utils | deps: torch, numpy | 35.49s
+Cell: utils | deps: torch, numpy | 34.73s
  | <button class="run-btn" onclick="runCell('utils')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('utils')">Copy</button>
 <a href="cells/utils.py" target="_blank" class="raw-btn">Raw</a>
@@ -3795,22 +3795,22 @@ Cell: utils | deps: torch, numpy | 35.49s
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
 Downloading nvidia-curand-cu12 (60.7MiB)
+Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
 Downloading nvidia-cufile-cu12 (1.1MiB)
 Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
-Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading nvidia-cudnn-cu12 (674.0MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
-Downloading torch (846.9MiB)
-Downloading nvidia-nvjitlink-cu12 (37.4MiB)
+Downloading networkx (1.9MiB)
+Downloading sympy (6.0MiB)
 Downloading setuptools (1.1MiB)
 Downloading nvidia-cublas-cu12 (566.8MiB)
-Downloading networkx (1.9MiB)
-Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
+Downloading nvidia-cusparse-cu12 (274.9MiB)
 Downloading nvidia-cusolver-cu12 (255.1MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
 Downloading numpy (16.2MiB)
+Downloading nvidia-cudnn-cu12 (674.0MiB)
 Downloading nvidia-cufft-cu12 (184.2MiB)
 Downloading nvidia-nccl-cu12 (307.4MiB)
-Downloading sympy (6.0MiB)
+Downloading nvidia-nvjitlink-cu12 (37.4MiB)
+Downloading torch (846.9MiB)
 Downloading triton (148.3MiB)
  Downloading nvidia-cufile-cu12
  Downloading setuptools
@@ -3830,7 +3830,7 @@ Downloading triton (148.3MiB)
  Downloading nvidia-cublas-cu12
  Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 26 packages in 461ms
+Installed 26 packages in 456ms
 </div>
 </div>
 </div>
@@ -3843,7 +3843,7 @@ Installed 26 packages in 461ms
 <span onclick="toggleOutput('bench_utils')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-bench_utils" onclick="toggleUvLogsFromHeader('bench_utils')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: bench_utils | deps: torch, numpy | 34.17s
+Cell: bench_utils | deps: torch, numpy | 34.06s
  | <button class="run-btn" onclick="runCell('bench_utils')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('bench_utils')">Copy</button>
 <a href="cells/bench_utils.py" target="_blank" class="raw-btn">Raw</a>
@@ -4331,24 +4331,24 @@ Cell: bench_utils | deps: torch, numpy | 34.17s
 <div class="uv-install-logs" id="uv-logs-bench_utils">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading setuptools (1.1MiB)
+Downloading networkx (1.9MiB)
 Downloading nvidia-cufile-cu12 (1.1MiB)
-Downloading sympy (6.0MiB)
-Downloading nvidia-nvjitlink-cu12 (37.4MiB)
 Downloading nvidia-curand-cu12 (60.7MiB)
-Downloading nvidia-cublas-cu12 (566.8MiB)
-Downloading nvidia-cusolver-cu12 (255.1MiB)
-Downloading networkx (1.9MiB)
-Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
-Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading torch (846.9MiB)
-Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
-Downloading nvidia-nccl-cu12 (307.4MiB)
-Downloading numpy (16.2MiB)
 Downloading nvidia-cudnn-cu12 (674.0MiB)
+Downloading setuptools (1.1MiB)
 Downloading nvidia-cufft-cu12 (184.2MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading nvidia-cublas-cu12 (566.8MiB)
+Downloading nvidia-nvjitlink-cu12 (37.4MiB)
+Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+Downloading nvidia-cusparse-cu12 (274.9MiB)
+Downloading nvidia-cusolver-cu12 (255.1MiB)
+Downloading sympy (6.0MiB)
 Downloading triton (148.3MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading numpy (16.2MiB)
+Downloading torch (846.9MiB)
+Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
  Downloading nvidia-cufile-cu12
  Downloading setuptools
  Downloading networkx
@@ -4367,7 +4367,7 @@ Downloading triton (148.3MiB)
  Downloading nvidia-cublas-cu12
  Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 26 packages in 507ms
+Installed 26 packages in 448ms
 </div>
 </div>
 </div>
@@ -4381,7 +4381,7 @@ Installed 26 packages in 507ms
 <span onclick="toggleOutput('config')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-config" onclick="toggleUvLogsFromHeader('config')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: config | deps: torch, numpy | 34.91s
+Cell: config | deps: torch, numpy | 35.66s
  | <button class="run-btn" onclick="runCell('config')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('config')">Copy</button>
 <a href="cells/config.py" target="_blank" class="raw-btn">Raw</a>
@@ -4441,23 +4441,23 @@ Cell: config | deps: torch, numpy | 34.91s
 <div class="uv-install-logs" id="uv-logs-config">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading sympy (6.0MiB)
-Downloading nvidia-cufile-cu12 (1.1MiB)
-Downloading setuptools (1.1MiB)
+Downloading numpy (16.2MiB)
 Downloading nvidia-curand-cu12 (60.7MiB)
-Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading torch (846.9MiB)
+Downloading sympy (6.0MiB)
+Downloading nvidia-cublas-cu12 (566.8MiB)
 Downloading nvidia-cusolver-cu12 (255.1MiB)
-Downloading numpy (16.2MiB)
 Downloading nvidia-nvjitlink-cu12 (37.4MiB)
+Downloading setuptools (1.1MiB)
+Downloading nvidia-cusparse-cu12 (274.9MiB)
+Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
 Downloading nvidia-nccl-cu12 (307.4MiB)
 Downloading nvidia-cufft-cu12 (184.2MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
-Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+Downloading nvidia-cufile-cu12 (1.1MiB)
 Downloading networkx (1.9MiB)
+Downloading torch (846.9MiB)
 Downloading nvidia-cudnn-cu12 (674.0MiB)
-Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
-Downloading nvidia-cublas-cu12 (566.8MiB)
+Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
 Downloading triton (148.3MiB)
  Downloading nvidia-cufile-cu12
  Downloading setuptools
@@ -4474,10 +4474,10 @@ Downloading triton (148.3MiB)
  Downloading nvidia-cusparselt-cu12
  Downloading nvidia-cusparse-cu12
  Downloading nvidia-nccl-cu12
- Downloading nvidia-cudnn-cu12
  Downloading nvidia-cublas-cu12
+ Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 26 packages in 572ms
+Installed 26 packages in 456ms
 </div>
 </div>
 </div>
@@ -4490,7 +4490,7 @@ Installed 26 packages in 572ms
 <span onclick="toggleOutput('save_data')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-save_data" onclick="toggleUvLogsFromHeader('save_data')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: save_data | deps: torch, numpy | 39.37s
+Cell: save_data | deps: torch, numpy | 38.92s
  | <button class="run-btn" onclick="runCell('save_data')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('save_data')">Copy</button>
 <a href="cells/save_data.py" target="_blank" class="raw-btn">Raw</a>
@@ -4585,24 +4585,24 @@ Down sum: 206.729263
 <div class="uv-install-logs" id="uv-logs-save_data">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading nvidia-cufft-cu12 (184.2MiB)
 Downloading sympy (6.0MiB)
-Downloading nvidia-nvjitlink-cu12 (37.4MiB)
-Downloading triton (148.3MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
 Downloading numpy (16.2MiB)
-Downloading networkx (1.9MiB)
 Downloading torch (846.9MiB)
 Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading nvidia-cublas-cu12 (566.8MiB)
 Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
-Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+Downloading nvidia-nvjitlink-cu12 (37.4MiB)
 Downloading nvidia-curand-cu12 (60.7MiB)
-Downloading nvidia-cusolver-cu12 (255.1MiB)
 Downloading setuptools (1.1MiB)
+Downloading nvidia-cufft-cu12 (184.2MiB)
 Downloading nvidia-cudnn-cu12 (674.0MiB)
-Downloading nvidia-cublas-cu12 (566.8MiB)
-Downloading nvidia-cufile-cu12 (1.1MiB)
+Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+Downloading networkx (1.9MiB)
 Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading nvidia-cufile-cu12 (1.1MiB)
+Downloading nvidia-cusolver-cu12 (255.1MiB)
+Downloading triton (148.3MiB)
  Downloading nvidia-cufile-cu12
  Downloading setuptools
  Downloading networkx
@@ -4621,17 +4621,17 @@ Downloading nvidia-cusparselt-cu12 (273.9MiB)
  Downloading nvidia-cublas-cu12
  Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 26 packages in 455ms
+Installed 26 packages in 453ms
 </div>
 </div>
 <div class="cell-artifacts">
 <h4>Artifacts:</h4>
-<a href="artifacts/save_data/router_bias.pt" class="artifact" target="_blank">router_bias.pt</a>
+<a href="artifacts/save_data/down_proj.pt" class="artifact" target="_blank">down_proj.pt</a>
 <a href="artifacts/save_data/router_weight.pt" class="artifact" target="_blank">router_weight.pt</a>
 <a href="artifacts/save_data/down_proj_bias.pt" class="artifact" target="_blank">down_proj_bias.pt</a>
 <a href="artifacts/save_data/gate_up_proj.pt" class="artifact" target="_blank">gate_up_proj.pt</a>
-<a href="artifacts/save_data/down_proj.pt" class="artifact" target="_blank">down_proj.pt</a>
 <a href="artifacts/save_data/gate_up_proj_bias.pt" class="artifact" target="_blank">gate_up_proj_bias.pt</a>
+<a href="artifacts/save_data/router_bias.pt" class="artifact" target="_blank">router_bias.pt</a>
 </div>
 </div>
 </div>
@@ -4645,7 +4645,7 @@ Installed 26 packages in 455ms
 <span onclick="toggleOutput('yamoe_run')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-yamoe_run" onclick="toggleUvLogsFromHeader('yamoe_run')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: yamoe_run | deps: torch, kernels, numpy | 38.45s
+Cell: yamoe_run | deps: torch, kernels, numpy | 38.89s
  | <button class="run-btn" onclick="runCell('yamoe_run')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('yamoe_run')">Copy</button>
 <a href="cells/yamoe_run.py" target="_blank" class="raw-btn">Raw</a>
@@ -4938,10 +4938,10 @@ Input Variation: +0.001 * iteration (deterministic)
 
 Warming up (10 iterations)...
 Benchmarking (50 iterations)...
-  Progress: 20% complete (avg: 4.253 ms)
-  Progress: 40% complete (avg: 4.248 ms)
+  Progress: 20% complete (avg: 4.249 ms)
+  Progress: 40% complete (avg: 4.247 ms)
   Progress: 60% complete (avg: 4.248 ms)
-  Progress: 80% complete (avg: 4.250 ms)
+  Progress: 80% complete (avg: 4.248 ms)
 
 Output tensors:
   Primary: shape=(1, 100, 1152), dtype=torch.float32, device=cuda:0, range=[-0.049506, 0.054984], mean=0.000034, std=0.006508, norm=2.208791
@@ -4951,19 +4951,19 @@ Output tensors:
 Iterations: 50
 
 Latency Statistics:
-  Average: 4.250 ms
-  Min:     4.125 ms
-  Max:     4.300 ms
-  Std Dev: 0.023 ms
+  Average: 4.249 ms
+  Min:     4.132 ms
+  Max:     4.278 ms
+  Std Dev: 0.021 ms
 
 Percentiles:
-  P50 (median): 4.254 ms
-  P95:          4.268 ms
-  P99:          4.290 ms
+  P50 (median): 4.253 ms
+  P95:          4.263 ms
+  P99:          4.272 ms
 
 Throughput:
-  Tokens/sec: 23530.9
-  Std Dev:    131.3
+  Tokens/sec: 23537.0
+  Std Dev:    116.3
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
 Saved benchmark results to yamoe_results.json
@@ -4973,25 +4973,25 @@ Output sum: 3.971905
 <div class="uv-install-logs" id="uv-logs-yamoe_run">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading nvidia-curand-cu12 (60.7MiB)
-Downloading numpy (16.2MiB)
-Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
-Downloading hf-xet (3.0MiB)
-Downloading networkx (1.9MiB)
-Downloading nvidia-cufile-cu12 (1.1MiB)
+Downloading nvidia-cufft-cu12 (184.2MiB)
 Downloading setuptools (1.1MiB)
+Downloading nvidia-curand-cu12 (60.7MiB)
+Downloading nvidia-cudnn-cu12 (674.0MiB)
 Downloading sympy (6.0MiB)
-Downloading nvidia-cusparse-cu12 (274.9MiB)
 Downloading nvidia-cublas-cu12 (566.8MiB)
-Downloading torch (846.9MiB)
-Downloading nvidia-cufft-cu12 (184.2MiB)
+Downloading networkx (1.9MiB)
+Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
 Downloading triton (148.3MiB)
+Downloading hf-xet (3.0MiB)
 Downloading nvidia-cusolver-cu12 (255.1MiB)
-Downloading nvidia-cudnn-cu12 (674.0MiB)
-Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading numpy (16.2MiB)
 Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading nvidia-cusparse-cu12 (274.9MiB)
 Downloading nvidia-nvjitlink-cu12 (37.4MiB)
+Downloading nvidia-cufile-cu12 (1.1MiB)
+Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading torch (846.9MiB)
  Downloading nvidia-cufile-cu12
  Downloading hf-xet
  Downloading setuptools
@@ -5011,14 +5011,14 @@ Downloading nvidia-nvjitlink-cu12 (37.4MiB)
  Downloading nvidia-cublas-cu12
  Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 37 packages in 452ms
+Installed 37 packages in 450ms
 </div>
 </div>
 <div class="cell-stderr">Fetching 6 files:   0%|          | 0/6 [00:00&lt;?, ?it/s]
-Fetching 6 files:  17%|█▋        | 1/6 [00:00&lt;00:01,  3.53it/s]
-Fetching 6 files:  33%|███▎      | 2/6 [00:00&lt;00:01,  3.45it/s]
-Fetching 6 files:  50%|█████     | 3/6 [00:00&lt;00:01,  2.94it/s]
-Fetching 6 files: 100%|██████████| 6/6 [00:00&lt;00:00,  6.14it/s]</div>
+Fetching 6 files:  17%|█▋        | 1/6 [00:00&lt;00:01,  3.63it/s]
+Fetching 6 files:  33%|███▎      | 2/6 [00:00&lt;00:01,  3.84it/s]
+Fetching 6 files:  50%|█████     | 3/6 [00:00&lt;00:00,  3.44it/s]
+Fetching 6 files: 100%|██████████| 6/6 [00:00&lt;00:00,  7.04it/s]</div>
 <div class="cell-artifacts">
 <h4>Artifacts:</h4>
 <a href="artifacts/yamoe_run/yamoe_results.json" class="artifact" target="_blank">yamoe_results.json</a>
@@ -5035,7 +5035,7 @@ Fetching 6 files: 100%|██████████| 6/6 [00:00&lt;00:00,  6.1
 <span onclick="toggleOutput('binned_run')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-binned_run" onclick="toggleUvLogsFromHeader('binned_run')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: binned_run | deps: torch, numpy | 39.83s
+Cell: binned_run | deps: torch, numpy | 39.04s
  | <button class="run-btn" onclick="runCell('binned_run')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('binned_run')">Copy</button>
 <a href="cells/binned_run.py" target="_blank" class="raw-btn">Raw</a>
@@ -5449,10 +5449,10 @@ Input Variation: +0.001 * iteration (deterministic)
 
 Warming up (10 iterations)...
 Benchmarking (50 iterations)...
-  Progress: 20% complete (avg: 38.543 ms)
-  Progress: 40% complete (avg: 37.857 ms)
-  Progress: 60% complete (avg: 37.457 ms)
-  Progress: 80% complete (avg: 37.143 ms)
+  Progress: 20% complete (avg: 38.510 ms)
+  Progress: 40% complete (avg: 38.423 ms)
+  Progress: 60% complete (avg: 37.889 ms)
+  Progress: 80% complete (avg: 37.386 ms)
 
 Output tensors:
   Primary: shape=(1, 100, 1152), dtype=torch.float32, device=cuda:0, range=[-0.049506, 0.054984], mean=0.000034, std=0.006508, norm=2.208791
@@ -5462,19 +5462,19 @@ Output tensors:
 Iterations: 50
 
 Latency Statistics:
-  Average: 36.539 ms
-  Min:     32.831 ms
-  Max:     40.074 ms
-  Std Dev: 1.614 ms
+  Average: 36.727 ms
+  Min:     33.185 ms
+  Max:     39.167 ms
+  Std Dev: 1.747 ms
 
 Percentiles:
-  P50 (median): 36.870 ms
-  P95:          39.005 ms
-  P99:          39.950 ms
+  P50 (median): 36.949 ms
+  P95:          38.664 ms
+  P99:          38.973 ms
 
 Throughput:
-  Tokens/sec: 2736.8
-  Std Dev:    123.0
+  Tokens/sec: 2722.8
+  Std Dev:    133.0
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
 Saved benchmark results to binned_results.json
@@ -5484,24 +5484,24 @@ Output sum: 3.971905
 <div class="uv-install-logs" id="uv-logs-binned_run">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading nvidia-cudnn-cu12 (674.0MiB)
-Downloading nvidia-cublas-cu12 (566.8MiB)
-Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
-Downloading setuptools (1.1MiB)
-Downloading nvidia-cusolver-cu12 (255.1MiB)
 Downloading networkx (1.9MiB)
-Downloading torch (846.9MiB)
-Downloading nvidia-nvjitlink-cu12 (37.4MiB)
-Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading nvidia-cusolver-cu12 (255.1MiB)
 Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading numpy (16.2MiB)
+Downloading nvidia-cufft-cu12 (184.2MiB)
 Downloading nvidia-cufile-cu12 (1.1MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
-Downloading sympy (6.0MiB)
-Downloading triton (148.3MiB)
 Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
+Downloading numpy (16.2MiB)
+Downloading setuptools (1.1MiB)
+Downloading nvidia-nvjitlink-cu12 (37.4MiB)
 Downloading nvidia-curand-cu12 (60.7MiB)
-Downloading nvidia-cufft-cu12 (184.2MiB)
+Downloading sympy (6.0MiB)
+Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+Downloading nvidia-cublas-cu12 (566.8MiB)
+Downloading nvidia-cudnn-cu12 (674.0MiB)
+Downloading triton (148.3MiB)
+Downloading torch (846.9MiB)
  Downloading nvidia-cufile-cu12
  Downloading setuptools
  Downloading networkx
@@ -5520,7 +5520,7 @@ Downloading nvidia-cufft-cu12 (184.2MiB)
  Downloading nvidia-cublas-cu12
  Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 26 packages in 442ms
+Installed 26 packages in 450ms
 </div>
 </div>
 <div class="cell-artifacts">
@@ -5539,7 +5539,7 @@ Installed 26 packages in 442ms
 <span onclick="toggleOutput('gptoss_run')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-gptoss_run" onclick="toggleUvLogsFromHeader('gptoss_run')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: gptoss_run | deps: torch, numpy | 39.94s
+Cell: gptoss_run | deps: torch, numpy | 39.34s
  | <button class="run-btn" onclick="runCell('gptoss_run')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('gptoss_run')">Copy</button>
 <a href="cells/gptoss_run.py" target="_blank" class="raw-btn">Raw</a>
@@ -5857,10 +5857,10 @@ Input Variation: +0.001 * iteration (deterministic)
 
 Warming up (10 iterations)...
 Benchmarking (50 iterations)...
-  Progress: 20% complete (avg: 48.070 ms)
-  Progress: 40% complete (avg: 47.917 ms)
-  Progress: 60% complete (avg: 47.432 ms)
-  Progress: 80% complete (avg: 46.164 ms)
+  Progress: 20% complete (avg: 50.553 ms)
+  Progress: 40% complete (avg: 49.257 ms)
+  Progress: 60% complete (avg: 47.930 ms)
+  Progress: 80% complete (avg: 46.616 ms)
 
 Output tensors:
   Primary: shape=(1, 100, 1152), dtype=torch.float32, device=cuda:0, range=[-0.064982, 0.061193], mean=0.000100, std=0.013510, norm=4.585560
@@ -5870,19 +5870,19 @@ Output tensors:
 Iterations: 50
 
 Latency Statistics:
-  Average: 45.237 ms
-  Min:     39.776 ms
-  Max:     49.247 ms
-  Std Dev: 2.851 ms
+  Average: 45.423 ms
+  Min:     38.786 ms
+  Max:     51.683 ms
+  Std Dev: 3.671 ms
 
 Percentiles:
-  P50 (median): 46.461 ms
-  P95:          48.433 ms
-  P99:          49.077 ms
+  P50 (median): 45.482 ms
+  P95:          50.887 ms
+  P99:          51.415 ms
 
 Throughput:
-  Tokens/sec: 2210.6
-  Std Dev:    143.0
+  Tokens/sec: 2201.5
+  Std Dev:    179.5
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
 Saved benchmark results to gptoss_results.json
@@ -5892,24 +5892,24 @@ Output sum: 11.532237
 <div class="uv-install-logs" id="uv-logs-gptoss_run">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
+Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+Downloading nvidia-nvjitlink-cu12 (37.4MiB)
 Downloading numpy (16.2MiB)
-Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading nvidia-cudnn-cu12 (674.0MiB)
-Downloading nvidia-nccl-cu12 (307.4MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
 Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
-Downloading nvidia-curand-cu12 (60.7MiB)
-Downloading networkx (1.9MiB)
 Downloading setuptools (1.1MiB)
+Downloading triton (148.3MiB)
+Downloading nvidia-nccl-cu12 (307.4MiB)
 Downloading nvidia-cufft-cu12 (184.2MiB)
-Downloading nvidia-cufile-cu12 (1.1MiB)
-Downloading nvidia-nvjitlink-cu12 (37.4MiB)
-Downloading nvidia-cublas-cu12 (566.8MiB)
+Downloading nvidia-cudnn-cu12 (674.0MiB)
 Downloading sympy (6.0MiB)
+Downloading networkx (1.9MiB)
+Downloading nvidia-cublas-cu12 (566.8MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading nvidia-curand-cu12 (60.7MiB)
+Downloading nvidia-cufile-cu12 (1.1MiB)
+Downloading nvidia-cusparse-cu12 (274.9MiB)
 Downloading nvidia-cusolver-cu12 (255.1MiB)
-Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
 Downloading torch (846.9MiB)
-Downloading triton (148.3MiB)
  Downloading nvidia-cufile-cu12
  Downloading setuptools
  Downloading networkx
@@ -5947,7 +5947,7 @@ Installed 26 packages in 443ms
 <span onclick="toggleOutput('gptoss_training_run')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-gptoss_training_run" onclick="toggleUvLogsFromHeader('gptoss_training_run')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: gptoss_training_run | deps: torch, numpy | 41.85s
+Cell: gptoss_training_run | deps: torch, numpy | 40.47s
  | <button class="run-btn" onclick="runCell('gptoss_training_run')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('gptoss_training_run')">Copy</button>
 <a href="cells/gptoss_training_run.py" target="_blank" class="raw-btn">Raw</a>
@@ -6248,10 +6248,10 @@ Input Variation: +0.001 * iteration (deterministic)
 
 Warming up (10 iterations)...
 Benchmarking (50 iterations)...
-  Progress: 20% complete (avg: 49.277 ms)
-  Progress: 40% complete (avg: 48.351 ms)
-  Progress: 60% complete (avg: 47.557 ms)
-  Progress: 80% complete (avg: 46.750 ms)
+  Progress: 20% complete (avg: 50.806 ms)
+  Progress: 40% complete (avg: 50.412 ms)
+  Progress: 60% complete (avg: 49.460 ms)
+  Progress: 80% complete (avg: 48.277 ms)
 
 Output tensors:
   Primary: shape=(1, 100, 1152), dtype=torch.float32, device=cuda:0, range=[-0.064982, 0.061193], mean=0.000100, std=0.013510, norm=4.585560
@@ -6261,19 +6261,19 @@ Output tensors:
 Iterations: 50
 
 Latency Statistics:
-  Average: 45.751 ms
-  Min:     38.860 ms
-  Max:     50.817 ms
-  Std Dev: 2.834 ms
+  Average: 47.223 ms
+  Min:     40.875 ms
+  Max:     51.398 ms
+  Std Dev: 3.111 ms
 
 Percentiles:
-  P50 (median): 45.833 ms
-  P95:          50.540 ms
-  P99:          50.777 ms
+  P50 (median): 47.684 ms
+  P95:          51.259 ms
+  P99:          51.362 ms
 
 Throughput:
-  Tokens/sec: 2185.7
-  Std Dev:    141.0
+  Tokens/sec: 2117.6
+  Std Dev:    143.6
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
 Saved benchmark results to gptoss_training_results.json
@@ -6283,30 +6283,30 @@ Output sum: 11.532237
 <div class="uv-install-logs" id="uv-logs-gptoss_training_run">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading sympy (6.0MiB)
-Downloading nvidia-cufile-cu12 (1.1MiB)
-Downloading nvidia-cufft-cu12 (184.2MiB)
-Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+Downloading nvidia-nvjitlink-cu12 (37.4MiB)
 Downloading numpy (16.2MiB)
-Downloading nvidia-curand-cu12 (60.7MiB)
+Downloading sympy (6.0MiB)
 Downloading networkx (1.9MiB)
 Downloading setuptools (1.1MiB)
-Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading nvidia-cufft-cu12 (184.2MiB)
+Downloading torch (846.9MiB)
 Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
 Downloading nvidia-cudnn-cu12 (674.0MiB)
 Downloading nvidia-cublas-cu12 (566.8MiB)
 Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading nvidia-curand-cu12 (60.7MiB)
 Downloading nvidia-cusolver-cu12 (255.1MiB)
-Downloading nvidia-nvjitlink-cu12 (37.4MiB)
+Downloading nvidia-cufile-cu12 (1.1MiB)
+Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
 Downloading triton (148.3MiB)
-Downloading torch (846.9MiB)
+Downloading nvidia-cusparse-cu12 (274.9MiB)
  Downloading nvidia-cufile-cu12
  Downloading setuptools
  Downloading networkx
  Downloading nvidia-cuda-cupti-cu12
- Downloading numpy
  Downloading sympy
+ Downloading numpy
  Downloading nvidia-nvjitlink-cu12
  Downloading nvidia-curand-cu12
  Downloading nvidia-cuda-nvrtc-cu12
@@ -6319,7 +6319,7 @@ Downloading torch (846.9MiB)
  Downloading nvidia-cublas-cu12
  Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 26 packages in 544ms
+Installed 26 packages in 451ms
 </div>
 </div>
 <div class="cell-artifacts">
@@ -6338,7 +6338,7 @@ Installed 26 packages in 544ms
 <span onclick="toggleOutput('megablocks_run')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-megablocks_run" onclick="toggleUvLogsFromHeader('megablocks_run')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: megablocks_run | deps: torch, numpy, kernels | 47.50s
+Cell: megablocks_run | deps: torch, numpy, kernels | 47.67s
  | <button class="run-btn" onclick="runCell('megablocks_run')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('megablocks_run')">Copy</button>
 <a href="cells/megablocks_run.py" target="_blank" class="raw-btn">Raw</a>
@@ -6567,10 +6567,10 @@ Input Variation: +0.001 * iteration (deterministic)
 
 Warming up (10 iterations)...
 Benchmarking (50 iterations)...
-  Progress: 20% complete (avg: 0.852 ms)
-  Progress: 40% complete (avg: 0.837 ms)
-  Progress: 60% complete (avg: 0.835 ms)
-  Progress: 80% complete (avg: 2.704 ms)
+  Progress: 20% complete (avg: 0.875 ms)
+  Progress: 40% complete (avg: 0.853 ms)
+  Progress: 60% complete (avg: 0.859 ms)
+  Progress: 80% complete (avg: 2.680 ms)
 
 Output tensors:
   Primary: shape=(1, 100, 1152), dtype=torch.float32, device=cuda:0, range=[-0.061104, 0.055115], mean=0.000056, std=0.013535, norm=4.593927
@@ -6580,19 +6580,19 @@ Output tensors:
 Iterations: 50
 
 Latency Statistics:
-  Average: 3.870 ms
-  Min:     0.810 ms
-  Max:     8.541 ms
-  Std Dev: 3.728 ms
+  Average: 3.830 ms
+  Min:     0.819 ms
+  Max:     8.440 ms
+  Std Dev: 3.659 ms
 
 Percentiles:
-  P50 (median): 0.840 ms
-  P95:          8.540 ms
-  P99:          8.541 ms
+  P50 (median): 0.886 ms
+  P95:          8.437 ms
+  P99:          8.439 ms
 
 Throughput:
-  Tokens/sec: 25840.1
-  Std Dev:    53236.0
+  Tokens/sec: 26109.7
+  Std Dev:    51610.7
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
 Saved benchmark results to megablocks_results.json
@@ -6602,25 +6602,25 @@ Output sum: 6.473885
 <div class="uv-install-logs" id="uv-logs-megablocks_run">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading sympy (6.0MiB)
 Downloading nvidia-cufft-cu12 (184.2MiB)
+Downloading nvidia-cudnn-cu12 (674.0MiB)
+Downloading nvidia-cublas-cu12 (566.8MiB)
 Downloading nvidia-curand-cu12 (60.7MiB)
-Downloading nvidia-cufile-cu12 (1.1MiB)
 Downloading nvidia-cuda-nvrtc-cu12 (84.0MiB)
+Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
+Downloading nvidia-cufile-cu12 (1.1MiB)
 Downloading nvidia-nvjitlink-cu12 (37.4MiB)
 Downloading nvidia-cusparse-cu12 (274.9MiB)
-Downloading nvidia-cudnn-cu12 (674.0MiB)
 Downloading triton (148.3MiB)
 Downloading networkx (1.9MiB)
 Downloading hf-xet (3.0MiB)
-Downloading nvidia-cublas-cu12 (566.8MiB)
-Downloading setuptools (1.1MiB)
-Downloading nvidia-cusolver-cu12 (255.1MiB)
-Downloading sympy (6.0MiB)
-Downloading torch (846.9MiB)
 Downloading numpy (16.2MiB)
-Downloading nvidia-cuda-cupti-cu12 (9.8MiB)
+Downloading nvidia-nccl-cu12 (307.4MiB)
+Downloading setuptools (1.1MiB)
 Downloading nvidia-cusparselt-cu12 (273.9MiB)
+Downloading torch (846.9MiB)
+Downloading nvidia-cusolver-cu12 (255.1MiB)
  Downloading nvidia-cufile-cu12
  Downloading hf-xet
  Downloading setuptools
@@ -6640,19 +6640,19 @@ Downloading nvidia-cusparselt-cu12 (273.9MiB)
  Downloading nvidia-cublas-cu12
  Downloading nvidia-cudnn-cu12
  Downloading torch
-Installed 37 packages in 458ms
+Installed 37 packages in 453ms
 </div>
 </div>
 <div class="cell-stderr">Fetching 66 files:   0%|          | 0/66 [00:00&lt;?, ?it/s]
-Fetching 66 files:   2%|▏         | 1/66 [00:00&lt;00:12,  5.34it/s]
-Fetching 66 files:   3%|▎         | 2/66 [00:00&lt;00:16,  3.98it/s]
-Fetching 66 files:  26%|██▌       | 17/66 [00:01&lt;00:02, 17.77it/s]
-Fetching 66 files:  67%|██████▋   | 44/66 [00:01&lt;00:00, 43.54it/s]
-Fetching 66 files:  76%|███████▌  | 50/66 [00:01&lt;00:00, 43.86it/s]
-Fetching 66 files:  83%|████████▎ | 55/66 [00:01&lt;00:00, 42.86it/s]
-Fetching 66 files:  91%|█████████ | 60/66 [00:01&lt;00:00, 33.73it/s]
-Fetching 66 files:  98%|█████████▊| 65/66 [00:02&lt;00:00, 33.63it/s]
-Fetching 66 files: 100%|██████████| 66/66 [00:02&lt;00:00, 32.65it/s]</div>
+Fetching 66 files:   2%|▏         | 1/66 [00:00&lt;00:11,  5.69it/s]
+Fetching 66 files:   8%|▊         | 5/66 [00:00&lt;00:02, 20.65it/s]
+Fetching 66 files:  14%|█▎        | 9/66 [00:00&lt;00:04, 12.19it/s]
+Fetching 66 files:  26%|██▌       | 17/66 [00:01&lt;00:03, 13.56it/s]
+Fetching 66 files:  61%|██████    | 40/66 [00:01&lt;00:00, 41.01it/s]
+Fetching 66 files:  74%|███████▍  | 49/66 [00:01&lt;00:00, 41.96it/s]
+Fetching 66 files:  86%|████████▋ | 57/66 [00:01&lt;00:00, 43.75it/s]
+Fetching 66 files:  97%|█████████▋| 64/66 [00:01&lt;00:00, 42.69it/s]
+Fetching 66 files: 100%|██████████| 66/66 [00:02&lt;00:00, 31.94it/s]</div>
 <div class="cell-artifacts">
 <h4>Artifacts:</h4>
 <a href="artifacts/megablocks_run/megablocks_results.json" class="artifact" target="_blank">megablocks_results.json</a>
@@ -6669,7 +6669,7 @@ Fetching 66 files: 100%|██████████| 66/66 [00:02&lt;00:00, 3
 <span onclick="toggleOutput('visualization')" style="cursor: pointer;">▼ output</span>
  <span id="uv-indicator-visualization" onclick="toggleUvLogsFromHeader('visualization')" style="cursor: pointer;">▶ uv-logs</span>
 </span> | 
-Cell: visualization | deps: matplotlib | 3.14s
+Cell: visualization | deps: matplotlib | 3.13s
  | <button class="run-btn" onclick="runCell('visualization')">▶ run</button>
 <button class="copy-btn" onclick="copyCell('visualization')">Copy</button>
 <a href="cells/visualization.py" target="_blank" class="raw-btn">Raw</a>
@@ -6915,30 +6915,30 @@ Loaded /repo/moe_benchmarks/megablocks_yamoe/.uvnote/cache/0febdf3420999533bc2e1
 Performance Summary:
 Implementation                 Avg (ms)     P95 (ms)     Tokens/sec   Relative Speed 
 --------------------------------------------------------------------------------
-megablocks_results                 3.87        8.54       25840        1.00x
-yamoe_results                      4.25        4.27       23531        0.91x
-binned_results                    36.54       39.00        2737        0.11x
-gptoss_results                    45.24       48.43        2211        0.09x
-gptoss_training_results           45.75       50.54        2186        0.08x
-
-Fastest: megablocks_results (3.87ms avg)
-Slowest: gptoss_training_results (45.75ms avg)
-Max Speedup: 11.8x
+megablocks_results                 3.83        8.44       26110        1.00x
+yamoe_results                      4.25        4.26       23537        0.90x
+binned_results                    36.73       38.66        2723        0.10x
+gptoss_results                    45.42       50.89        2202        0.08x
+gptoss_training_results           47.22       51.26        2118        0.08x
+
+Fastest: megablocks_results (3.83ms avg)
+Slowest: gptoss_training_results (47.22ms avg)
+Max Speedup: 12.3x
 </div>
 <div class="uv-install-logs" id="uv-logs-visualization">
 <div class="uv-logs-header" onclick="toggleUvLogs(this)">▶ UV Install Logs</div>
 <div class="uv-logs-content" style="display: none;">
-Downloading pillow (6.3MiB)
-Downloading kiwisolver (1.4MiB)
-Downloading numpy (16.2MiB)
 Downloading matplotlib (8.3MiB)
+Downloading numpy (16.2MiB)
+Downloading pillow (6.3MiB)
 Downloading fonttools (4.7MiB)
+Downloading kiwisolver (1.4MiB)
  Downloading kiwisolver
  Downloading pillow
  Downloading fonttools
  Downloading matplotlib
  Downloading numpy
-Installed 11 packages in 48ms
+Installed 11 packages in 49ms
 </div>
 </div>
 <div class="cell-artifacts">