Spaces:
Running
Running
import os | |
from groq import Groq | |
import gradio as gr | |
from transformers import AutoModel, AutoConfig | |
hf_token = os.getenv("HF_TOKEN") # Make sure you set this environment variable | |
try: | |
config = AutoConfig.from_pretrained("HusseinEid/llama-3-chatbot", config_file_name="config.json", use_auth_token=hf_token) | |
model = AutoModel.from_pretrained("HusseinEid/lora_model", config=config, use_auth_token=hf_token) | |
except OSError as e: | |
print(f"Error: {e}") | |
client = Groq(api_key = os.environ.get("GROQ_API_KEY"), ) | |
system_prompt = { | |
"role": "system", | |
"content": | |
"You are a useful assistant. You reply with detailed answers. " | |
} | |
async def chat_groq(message, history): | |
messages = [system_prompt] | |
for msg in history: | |
messages.append({"role": "user", "content": str(msg[0])}) | |
messages.append({"role": "assistant", "content": str(msg[1])}) | |
messages.append({"role": "user", "content": str (message)}) | |
response_content = '' | |
stream = client. chat.completions.create( | |
model=model, | |
messages=messages, | |
max_tokens=1024, | |
temperature=1.2, | |
stream=True | |
) | |
for chunk in stream: | |
content = chunk.choices[0].delta.content | |
if content: | |
response_content += chunk. choices[0].delta.content | |
yield response_content | |
with gr. Blocks(theme=gr.themes.Monochrome(), fill_height=True) as demo: | |
gr.ChatInterface( chat_groq, | |
clear_btn=None, | |
undo_btn=None, | |
retry_btn=None, | |
) | |
demo.queue() | |
demo.launch() |