File size: 3,079 Bytes
3daf494 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 |
import os
import copy
import gradio as gr
from typing import List, Tuple
from llama_cpp import Llama
from huggingface_hub import hf_hub_download
# Load the LLaMA model
llm = Llama(
model_path=hf_hub_download(
repo_id=os.environ.get("REPO_ID", "mradermacher/Atlas-Chat-2B-GGUF"),
filename=os.environ.get("MODEL_FILE", "Atlas-Chat-2B.Q8_0.gguf"),
),
n_ctx=2048, # context window size
)
# Training prompt template
training_prompt = """<start_of_turn>user
{}<end_of_turn>
<start_of_turn>model
{}<end_of_turn>"""
# Generate response function
def response(
user_message: str,
chat_history: List[Tuple[str, str]],
max_response_length: int,
temperature: float,
top_p: float,
):
if not user_message.strip():
return "تقدروا تكتبوا الرسالة مرة اخرى؟"
# Format chat history into the prompt
formatted_prompt = ""
for user_input, model_response in chat_history:
formatted_prompt += training_prompt.format(user_input, model_response)
# Add the current user message to the formatted prompt
formatted_prompt += training_prompt.format(user_message, "")
try:
output = llm(
formatted_prompt,
max_tokens=max_response_length,
temperature=temperature,
top_p=top_p,
top_k=40,
repeat_penalty=1.1,
stop=["<end_of_turn>", "<|endoftext|>"],
stream=True,
)
response_text = ""
for out in output:
stream = copy.deepcopy(out)
response_text += stream["choices"][0]["text"]
return response_text
except Exception as e:
return f"شي خطأ وقع: {str(e)}"
# Create the Gradio chat interface
demo = gr.ChatInterface(
response,
title="AtlasChat-mini",
description="""\
# AtlasChat-mini 2B
This is a demo of [`MBZUAI-Paris/Atlas-Chat-2B`](https://huggingface.co/mbzuai-paris/atlas-chat-2b).
For more details, please check [our paper](https://arxiv.org/pdf/2409.17912).
Looking for a larger and more powerful version? Try the 9B version in [Hugging Face](https://huggingface.co/mbzuai-paris/atlas-chat-9b).
This demo was done using the [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) library for efficient inference and is running the [`mradermacher/Atlas-Chat-2B-GGUF`](https://huggingface.co/mradermacher/Atlas-Chat-2B-GGUF) version with 8-bit Q8_0 quantization.
""",
examples=[
['What is the capital of Morocco?'],
['كيفاش نوجد شي طاجين ؟'],
['واش تقدر تعوض Google ؟'],
['عاود لي شي نكتة']
],
cache_examples=False,
additional_inputs=[
gr.Slider(minimum=1, maximum=1024, value=128, step=1, label="Max New Tokens"),
gr.Slider(minimum=0.1, maximum=3.0, value=0.5, step=0.1, label="Temperature"),
gr.Slider(minimum=0.1, maximum=1.0, value=0.90, step=0.05, label="Top-p (nucleus sampling)"),
],
)
# Launch the demo
if __name__ == "__main__":
demo.launch() |