|
import os |
|
import json |
|
import gradio as gr |
|
from llama_cpp import Llama |
|
|
|
|
|
model_id = os.getenv('MODEL') |
|
quant = os.getenv('QUANT') |
|
chat_template = os.getenv('CHAT_TEMPLATE') |
|
|
|
|
|
model_name = model_id.split('/')[1].split('-GGUF')[0] |
|
title = f"🐶 {model_name}" |
|
description = f"Chat with <a href=\"https://huggingface.co/{model_id}\">{model_name}</a> in GGUF format ({quant})!" |
|
|
|
|
|
llm = Llama(model_path="model.gguf", |
|
n_ctx=32768, |
|
n_threads=2, |
|
n_vocab=32002, |
|
n_gpu_layers=-1, |
|
chat_format=chat_template) |
|
|
|
|
|
def chat_stream_completion(message, history, system_prompt): |
|
messages_prompts = [{"role": "system", "content": system_prompt}] |
|
for human, assistant in history: |
|
messages_prompts.append({"role": "user", "content": human}) |
|
messages_prompts.append({"role": "assistant", "content": assistant}) |
|
messages_prompts.append({"role": "user", "content": message}) |
|
|
|
response = llm.create_chat_completion( |
|
messages=messages_prompts, |
|
stream=True |
|
) |
|
message_repl = "" |
|
for chunk in response: |
|
if len(chunk['choices'][0]["delta"]) != 0 and "content" in chunk['choices'][0]["delta"]: |
|
message_repl = message_repl + chunk['choices'][0]["delta"]["content"] |
|
yield message_repl |
|
|
|
|
|
gr.ChatInterface( |
|
fn=chat_stream_completion, |
|
title=title, |
|
description=description, |
|
additional_inputs=[gr.Textbox("You are helpful assistant.")], |
|
additional_inputs_accordion="📝 System prompt", |
|
examples=[ |
|
['Can you solve the equation 2x + 3 = 11 for x?'], |
|
['Write an epic poem about Ancient Rome.'], |
|
['Who was the first person to walk on the Moon?'], |
|
['Use a list comprehension to create a list of squares for numbers from 1 to 10.'], |
|
['Recommend some popular science fiction books.'], |
|
['Can you write a short story about a time-traveling detective?'] |
|
] |
|
).queue().launch(server_name="0.0.0.0") |
