File size: 1,827 Bytes
5fd0c28 01945bd 2936c26 01945bd 2936c26 01945bd def541d 01945bd def541d 01945bd def541d 01945bd def541d 01945bd def541d 01945bd def541d 01945bd 2af305a 01945bd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 |
import gradio as gr
from llama_cpp import Llama
from huggingface_hub import hf_hub_download
# Model identifier from Hugging Face
model_repo = "Mat17892/lora_llama_gguf_g14" # Hugging Face model ID
# Download the GGUF file from Hugging Face
model_path = hf_hub_download(repo_id=model_repo, filename="llama_lora_model.gguf")
# Load the GGUF model using llama-cpp-python
print("Loading model...")
llm = Llama(model_path=model_path, n_ctx=2048, n_threads=8) # Adjust threads as needed
print("Model loaded!")
# Chat function
def chat_with_model(user_input, chat_history):
"""
Process user input and generate a response from the model.
:param user_input: User's input string
:param chat_history: List of [user_message, ai_response] pairs
:return: Updated chat history
"""
# Combine chat history into a single prompt
prompt = ""
for user, ai in chat_history:
prompt += f"User: {user}\nAI: {ai}\n"
prompt += f"User: {user_input}\nAI:"
# Generate response from the model
response = llm(prompt)["choices"][0]["text"].strip()
# Update chat history as a list of tuples
chat_history.append((user_input, response))
return chat_history, chat_history
# Gradio UI
with gr.Blocks() as demo:
gr.Markdown("# 🦙 LLaMA GGUF Chatbot")
chatbot = gr.Chatbot(label="Chat with the GGUF Model")
with gr.Row():
with gr.Column(scale=4):
user_input = gr.Textbox(label="Your Message", placeholder="Type a message...")
with gr.Column(scale=1):
submit_btn = gr.Button("Send")
chat_history = gr.State([])
# Link components
submit_btn.click(
chat_with_model,
inputs=[user_input, chat_history],
outputs=[chatbot, chat_history],
show_progress=True,
)
# Launch the app
demo.launch()
|