Switch to ZeroGPU with llama-cpp for GGUF model

README.md

@@ -8,6 +8,11 @@ sdk_version: 5.29.0
 app_file: app.py
 pinned: false
 license: apache-2.0
+tags:
+  - qwen
+  - uncensored
+  - llama-cpp
+  - zerogpu
 ---
 
 # Qwen3.5-9B Uncensored API Interface

app.py

@@ -1,54 +1,44 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+import spaces
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
 
-MODEL_ID = "HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive"
+MODEL_REPO = "HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive"
+MODEL_FILE = "Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf"
 
-client = InferenceClient()
+llm = None
 
-DEFAULT_PARAMS = {
-    "temperature": 0.7,
-    "top_p": 0.8,
-    "top_k": 20,
-    "max_tokens": 2048,
-}
+def load_model():
+    global llm
+    if llm is None:
+        model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE)
+        llm = Llama(
+            model_path=model_path,
+            n_ctx=8192,
+            n_gpu_layers=-1,
+            verbose=False,
+        )
+    return llm
 
 
-def generate_response(
-    message: str,
-    history: list,
-    system_prompt: str = "",
-    temperature: float = 0.7,
-    top_p: float = 0.8,
-    top_k: int = 20,
-    max_tokens: int = 2048,
-) -> str:
-    messages = []
+def format_messages(message: str, history: list, system_prompt: str = "") -> str:
+    formatted = ""
     
     if system_prompt.strip():
-        messages.append({"role": "system", "content": system_prompt})
+        formatted += f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
     
     for user_msg, assistant_msg in history:
         if user_msg:
-            messages.append({"role": "user", "content": user_msg})
+            formatted += f"<|im_start|>user\n{user_msg}<|im_end|>\n"
         if assistant_msg:
-            messages.append({"role": "assistant", "content": assistant_msg})
+            formatted += f"<|im_start|>assistant\n{assistant_msg}<|im_end|>\n"
     
-    messages.append({"role": "user", "content": message})
-    
-    try:
-        response = client.chat_completion(
-            model=MODEL_ID,
-            messages=messages,
-            temperature=temperature,
-            top_p=top_p,
-            max_tokens=max_tokens,
-        )
-        return response.choices[0].message.content
-    except Exception as e:
-        return f"Error: {str(e)}"
+    formatted += f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
+    return formatted
 
 
-def generate_stream(
+@spaces.GPU
+def generate_response(
     message: str,
     history: list,
     system_prompt: str = "",
@@ -56,39 +46,23 @@ def generate_stream(
     top_p: float = 0.8,
     top_k: int = 20,
     max_tokens: int = 2048,
-):
-    messages = []
-    
-    if system_prompt.strip():
-        messages.append({"role": "system", "content": system_prompt})
-    
-    for user_msg, assistant_msg in history:
-        if user_msg:
-            messages.append({"role": "user", "content": user_msg})
-        if assistant_msg:
-            messages.append({"role": "assistant", "content": assistant_msg})
+) -> str:
+    model = load_model()
+    prompt = format_messages(message, history, system_prompt)
     
-    messages.append({"role": "user", "content": message})
+    output = model(
+        prompt,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
+        stop=["<|im_end|>", "<|im_start|>"],
+    )
     
-    try:
-        stream = client.chat_completion(
-            model=MODEL_ID,
-            messages=messages,
-            temperature=temperature,
-            top_p=top_p,
-            max_tokens=max_tokens,
-            stream=True,
-        )
-        
-        partial_message = ""
-        for chunk in stream:
-            if chunk.choices[0].delta.content:
-                partial_message += chunk.choices[0].delta.content
-                yield partial_message
-    except Exception as e:
-        yield f"Error: {str(e)}"
+    return output["choices"][0]["text"].strip()
 
 
+@spaces.GPU
 def api_generate(
     prompt: str,
     system_prompt: str = "",
@@ -109,22 +83,16 @@ def api_generate(
     Returns:
         Dictionary with 'response' key containing generated text
     """
-    messages = []
-    
-    if system_prompt.strip():
-        messages.append({"role": "system", "content": system_prompt})
-    
-    messages.append({"role": "user", "content": prompt})
-    
     try:
-        response = client.chat_completion(
-            model=MODEL_ID,
-            messages=messages,
+        response = generate_response(
+            message=prompt,
+            history=[],
+            system_prompt=system_prompt,
             temperature=temperature,
             top_p=top_p,
             max_tokens=max_tokens,
         )
-        return {"response": response.choices[0].message.content, "status": "success"}
+        return {"response": response, "status": "success"}
     except Exception as e:
         return {"response": None, "status": "error", "error": str(e)}
 
@@ -141,6 +109,7 @@ with gr.Blocks(title="Qwen3.5-9B Uncensored API", theme=gr.themes.Soft()) as dem
         - Fully uncensored (0/465 refusals)
         - Multimodal capable (text, image, video)
         - Supports 201 languages
+        - Running on ZeroGPU with Q4_K_M quantization
         
         Use the chat interface below or access via API.
         """
@@ -189,8 +158,8 @@ with gr.Blocks(title="Qwen3.5-9B Uncensored API", theme=gr.themes.Soft()) as dem
                 )
                 max_tokens = gr.Slider(
                     minimum=64,
-                    maximum=8192,
-                    value=2048,
+                    maximum=4096,
+                    value=1024,
                     step=64,
                     label="Max Tokens",
                 )
@@ -207,8 +176,7 @@ with gr.Blocks(title="Qwen3.5-9B Uncensored API", theme=gr.themes.Soft()) as dem
             message = history[-1][0]
             history_without_last = history[:-1]
             
-            response = ""
-            for partial in generate_stream(
+            response = generate_response(
                 message, 
                 history_without_last, 
                 system_prompt, 
@@ -216,10 +184,9 @@ with gr.Blocks(title="Qwen3.5-9B Uncensored API", theme=gr.themes.Soft()) as dem
                 top_p, 
                 top_k, 
                 max_tokens
-            ):
-                response = partial
-                history[-1][1] = response
-                yield history
+            )
+            history[-1][1] = response
+            return history
         
         msg.submit(
             user_submit, 
@@ -262,7 +229,7 @@ with gr.Blocks(title="Qwen3.5-9B Uncensored API", theme=gr.themes.Soft()) as dem
                 system_prompt="You are a helpful assistant",
                 temperature=0.7,
                 top_p=0.8,
-                max_tokens=2048,
+                max_tokens=1024,
                 api_name="/api_generate"
             )
             print(result)
@@ -279,7 +246,7 @@ with gr.Blocks(title="Qwen3.5-9B Uncensored API", theme=gr.themes.Soft()) as dem
                         "You are a helpful assistant",
                         0.7,
                         0.8,
-                        2048
+                        1024
                     ]
                 }'
             ```
@@ -301,7 +268,7 @@ with gr.Blocks(title="Qwen3.5-9B Uncensored API", theme=gr.themes.Soft()) as dem
                 with gr.Row():
                     api_temp = gr.Slider(0.0, 2.0, 0.7, step=0.1, label="Temperature")
                     api_top_p = gr.Slider(0.0, 1.0, 0.8, step=0.05, label="Top P")
-                api_max_tokens = gr.Slider(64, 8192, 2048, step=64, label="Max Tokens")
+                api_max_tokens = gr.Slider(64, 4096, 1024, step=64, label="Max Tokens")
                 api_submit = gr.Button("Generate", variant="primary")
             
             with gr.Column():

requirements.txt

@@ -1,2 +1,4 @@
 gradio>=4.0.0
 huggingface_hub>=0.20.0
+llama-cpp-python
+spaces