v0.1

app.py

@@ -1,11 +1,11 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+from llama_cpp import Llama
 
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
+llm = Llama.from_pretrained("bartowski/starcoder2-15b-instruct-v0.1-GGUF", filename="starcoder2-15b-instruct-v0.1-Q6_K.gguf", n_gpu_layers=99, n_ctx=16384)
 
 def respond(
     message,
@@ -14,6 +14,7 @@ def respond(
     max_tokens,
     temperature,
     top_p,
+    top_k
 ):
     messages = [{"role": "system", "content": system_message}]
 
@@ -27,12 +28,13 @@ def respond(
 
     response = ""
 
-    for message in client.chat_completion(
+    for message in llm.create_chat_completion(
         messages,
         max_tokens=max_tokens,
         stream=True,
         temperature=temperature,
         top_p=top_p,
+        top_k=42,
     ):
         token = message.choices[0].delta.content
 

requirements.txt

@@ -1 +1,2 @@
-huggingface_hub==0.22.2
+huggingface_hub==0.22.2
+llama-cpp-python