Upload folder using huggingface_hub

README.md

@@ -9,6 +9,7 @@ app_file: app.py
 pinned: false
 license: apache-2.0
 short_description: Code assistant powered by fine-tuned Qwen 2.5 Coder
+suggested_hardware: t4-small
 ---
 
 # Qwen 2.5 Coder Assistant

app.py

@@ -1,9 +1,17 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from threading import Thread
 
 MODEL_ID = "GhostScientist/qwen25-coder-1.5b-codealpaca-sft"
 
-client = InferenceClient(MODEL_ID)
+# Load model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    torch_dtype=torch.float16,
+    device_map="auto",
+)
 
 def respond(message, history, system_message, max_tokens, temperature, top_p):
     """Generate response using the fine-tuned Qwen coder model."""
@@ -17,16 +25,35 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
 
     messages.append({"role": "user", "content": message})
 
-    response = ""
-    for token in client.chat_completion(
+    # Apply chat template
+    text = tokenizer.apply_chat_template(
         messages,
-        max_tokens=max_tokens,
-        stream=True,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    inputs = tokenizer([text], return_tensors="pt").to(model.device)
+
+    # Set up streaming
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+    generation_kwargs = dict(
+        **inputs,
+        streamer=streamer,
+        max_new_tokens=int(max_tokens),
         temperature=temperature,
         top_p=top_p,
-    ):
-        delta = token.choices[0].delta.content or ""
-        response += delta
+        do_sample=True,
+        pad_token_id=tokenizer.eos_token_id,
+    )
+
+    # Run generation in a thread
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
+
+    # Stream the response
+    response = ""
+    for new_text in streamer:
+        response += new_text
         yield response
 
 

requirements.txt

@@ -1,2 +1,4 @@
 gradio>=5.0.0
-huggingface_hub>=0.26.0
+torch
+transformers
+accelerate