app.py

import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama

# Download the Q3_K_M quantized GGUF from Unslo​th’s repo
model_path = hf_hub_download(
    repo_id="unsloth/Llama-3.2-1B-Instruct-GGUF",
    filename="Llama-3.2-1B-Instruct-Q3_K_M.gguf"
)
llm = Llama(model_path=model_path)

def generate(prompt: str, temperature: float = 0.7, max_tokens: int = 128):
    out = llm(prompt, temperature=temperature, max_tokens=max_tokens)
    return out["choices"][0]["text"]

iface = gr.Interface(
    fn=generate,
    inputs=[
        gr.Textbox(lines=4, label="Prompt"),
        gr.Slider(0.0, 1.0, 0.1, label="Temperature", value=0.7),
        gr.Slider(16, 512, 16, label="Max Tokens", value=128),
    ],
    outputs="text",
    title="unsloth Llama-3.2-1B (Q3_K_M, CPU)"
)

if __name__ == "__main__":
    iface.launch(server_name="0.0.0.0", server_port=7860)