Update app.py

app.py

@@ -1,13 +1,16 @@
 import os
-import torch
 import gradio as gr
 import psutil
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from llama_cpp import Llama
 
-os.environ["HF_HOME"] = "/data/hf_cache"
+os.environ["HF_HOME"] = "/tmp/hf_cache"
 
-tokenizer = AutoTokenizer.from_pretrained("/data/model2")
-model = AutoModelForCausalLM.from_pretrained("/data/model2", device_map="cpu", offload_folder="/tmp/offload")
+model = Llama.from_pretrained(
+    repo_id="unsloth/Qwen3.5-35B-A3B-GGUF",
+    filename="Qwen3.5-35B-A3B-Q2_K.gguf",
+    n_ctx=2048,
+    n_threads=16,
+)
 
 def get_stats():
     process = psutil.Process(os.getpid())
@@ -18,27 +21,20 @@ def get_stats():
     return f"RAM: {ram:.2f} GB | /tmp: {disk_tmp:.2f} GB | /data: {disk_data:.2f} GB | CPU: {cpu}%"
 
 def chat(message, history):
-    messages = []
+    messages = [{"role": "system", "content": "Reply directly without any reasoning or thinking process."}]
     for user, assistant in history:
         messages.append({"role": "user", "content": user})
         messages.append({"role": "assistant", "content": assistant})
     messages.append({"role": "user", "content": message})
 
-    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-    inputs = tokenizer([text], return_tensors="pt").to(model.device)
-    
-    from transformers import TextIteratorStreamer
-    from threading import Thread
-    
-    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
-    generation_kwargs = dict(**inputs, max_new_tokens=512, streamer=streamer)
-    
-    thread = Thread(target=model.generate, kwargs=generation_kwargs)
-    thread.start()
-    
     output = ""
-    for token in streamer:
-        output += token
+    for chunk in model.create_chat_completion(
+        messages=messages,
+        max_tokens=2048,
+        stream=True
+    ):
+        delta = chunk["choices"][0]["delta"].get("content", "")
+        output += delta
         yield output
 
 with gr.Blocks() as demo: