feat: setup local Qwen2 0.5

app.py

@@ -1,10 +1,8 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+from llama_cpp import Llama
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient(model="meta-llama/Meta-Llama-3-8B-Instruct")
+model = "Qwen/Qwen1.5-0.5B-Chat-GGUF"
+llm = Llama.from_pretrained(repo_id=model, filename="*q8_0.gguf", verbose=True)
 
 
 def respond(
@@ -25,19 +23,13 @@ def respond(
 
     messages.append({"role": "user", "content": message})
 
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
+    response = llm.create_chat_completion(
+        messages=messages,
         max_tokens=max_tokens,
-        stream=True,
         temperature=temperature,
         top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
+    )
+    return response
 
 
 """
@@ -47,7 +39,7 @@ demo = gr.ChatInterface(
     respond,
     additional_inputs=[
         gr.Textbox(
-            value="You are Marv, a chatbot that reluctantly answers questions with sarcastic responses in Persian only.",
+            value="You are a helpful assistant.",
             label="System message",
         ),
         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
@@ -60,13 +52,7 @@ demo = gr.ChatInterface(
             label="Top-p (nucleus sampling)",
         ),
     ],
-    description="A little bit of sarcasm doesn't kill anyone right? :)))",
-    examples=[
-        ["پایتخت فرانسه کجاست؟"],
-        ["دو بعلاوه دو چند میشود؟"],
-        ["یک جک بگو"],
-    ],
-    cache_examples=False,
+    description=model,
 )
 
 

requirements.txt

@@ -1 +1,2 @@
-huggingface_hub==0.22.2
+huggingface_hub==0.22.2
+llama-cpp-python==0.2.78