Zlib2/bonsai-8b-colab-prebuilt

Bonsai-8B (GGUF) – Run in 30 Seconds 🚀

⚡ Quick Start

🔧 Setup (run once)

import os
import tarfile

print("📦 Installing dependencies...")
!pip install -q huggingface_hub

from huggingface_hub import hf_hub_download

HF_REPO = "Zlib2/bonsai-8b-colab-prebuilt"

print("\n📥 1/3 Downloading pre-built llama.cpp (Fast)...")
llama_zip = hf_hub_download(repo_id=HF_REPO, filename="llama_cpp_prebuilt.tar.gz")

print("📥 2/3 Downloading Bonsai-8B.gguf model (Large file)...")
model_path = hf_hub_download(repo_id=HF_REPO, filename="Bonsai-8B.gguf")

print("📦 3/3 Extracting files...")
!mkdir -p /content/llama.cpp

with tarfile.open(llama_zip, "r:gz") as tar:
    tar.extractall(path="/content/llama.cpp")

!chmod +x /content/llama.cpp/build/bin/llama-cli

print("\n🎉 Setup complete!")

---

🤖 Run Inference

USER_PROMPT = "Explain quantum computing in simple terms."
SYSTEM_PROMPT = "You are a helpful assistant"

!/content/llama.cpp/build/bin/llama-cli \
    -m "{model_path}" \
    --system-prompt "{SYSTEM_PROMPT}" \
    -p "{USER_PROMPT}" \
    -n 4096 \
    --temp 0.5 \
    --top-p 0.85 \
    --top-k 20 \
    -ngl 99