LLaMa-2-ChatBot / app.py
ubermenchh's picture
Update app.py
407235e
import os
from typing import Iterator
from text_generation import Client
import gradio as gr
model_id = 'meta-llama/Llama-2-7b-chat-hf'
API_URL = "https://api-inference.huggingface.co/models/" + model_id
HF_TOKEN = os.environ.get('HF_READ_TOKEN', None)
client = Client(
API_URL,
headers={'Authorization': f"Bearer {HF_TOKEN}"}
)
EOS_STRING = "</s>"
EOT_STRING = "<EOT>"
def get_prompt(message, chat_history, system_prompt):
texts = [f"<s>[INST] <<SYS>>\n{system_prompt}\n<</SYS>>\n\n"]
do_strip = False
for user_input, response in chat_history:
user_input = user_input.strip() if do_strip else user_input
do_strip = True
texts.append(f"{user_input} [/INST] {response.strip()} </s><s>[INST] ")
message = message.strip() if do_strip else message
texts.append(f"{message} [/INST]")
return ''.join(texts)
def run(message, chat_history, system_prompt, max_new_tokens=1024, temperature=0.1, top_p=0.9, top_k=50):
prompt = get_prompt(message, chat_history, system_prompt)
generate_kwargs = dict(
max_new_tokens=max_new_tokens,
do_sample=True,
top_p=top_p,
top_k=top_k,
temperature=temperature
)
stream = client.generate_stream(prompt, **generate_kwargs)
output = ''
for response in stream:
if any([end_token in response.token.text for end_token in [EOS_STRING, EOT_STRING]]):
return output
else:
output += response.token.text
yield output
return output
DEFAULT_SYSTEM_PROMPT = """
You are LLaMa. You are an AI assistant, you are moderately-polite and give only true information.
You carefully provide accurate, factual, thoughtful, nuanced answers, and are brilliant at reasoning.
If you think there might not be a correct answer, you say so. Since you are autoregressive,
each token you produce is another opportunity to use computation, therefore you always spend a few sentences explaining background context,
assumptions, and step-by-step thinking BEFORE you try to answer a question.
"""
MAX_MAX_NEW_TOKENS = 4096
DEFAULT_MAX_NEW_TOKENS = 1024
MAX_INPUT_TOKEN_LENGTH = 4096
DESCRIPTION = """
# LLaMa-2-7b-chat-hf ChatBot
"""
def clear_and_save_textbox(message): return '', message
def display_input(message, history=[]):
history.append((message, ''))
return history
def delete_prev_fn(history=[]):
try:
message, _ = history.pop()
except IndexError:
message = ''
return history, message or ''
def generate(message, history_with_input, system_prompt, max_new_tokens, temperature, top_p, top_k):
if max_new_tokens > MAX_MAX_NEW_TOKENS:
raise ValueError
history = history_with_input[:-1]
generator = run(message, history, system_prompt, max_new_tokens, temperature, top_p, top_k)
try:
first_response = next(generator)
yield history + [(message, first_response)]
except StopIteration:
yield history + [(message, '')]
for response in generator:
yield history + [(message, response)]
def process_example(message):
generator = generate(message, [], DEFAULT_SYSTEM_PROMPT, 1024, 1, 0.95, 50)
for x in generator:
pass
return '', x
def check_input_token_length(message, chat_history, system_prompt):
input_token_length = len(message) + len(chat_history)
if input_token_length > MAX_INPUT_TOKEN_LENGTH:
raise gr.Error(f"The accumulated input is too long ({input_token_length} > {MAX_INPUT_TOKEN_LENGTH}). Clear your chat history and try again.")
with gr.Blocks() as demo:
gr.Markdown(DESCRIPTION)
with gr.Group():
chatbot = gr.Chatbot(label='Playground')
with gr.Row():
textbox = gr.Textbox(
container=False,
show_label=False,
placeholder='Hi, LLaMa',
scale=10
)
submit_button = gr.Button('Submit', variant='primary', scale=1, min_width=0)
with gr.Row():
retry_button = gr.Button('Retry', variant='secondary')
undo_button = gr.Button('Undo', variant='secondary')
clear_button = gr.Button('Clear', variant='secondary')
saved_input = gr.State()
with gr.Accordion(label='Advanced options', open=False):
system_prompt = gr.Textbox(label='System prompt', value=DEFAULT_SYSTEM_PROMPT, lines=5, interactive=False)
max_new_tokens = gr.Slider(label='Max New Tokens', minimum=1, maximum=MAX_MAX_NEW_TOKENS, step=1, value=DEFAULT_MAX_NEW_TOKENS)
temperature = gr.Slider(label='Temperature', minimum=0.1, maximum=4.0, step=0.1, value=0.1)
top_p = gr.Slider(label='Top-P (nucleus sampling)', minimum=0.05, maximum=1.0, step=0.05, value=0.9)
top_k = gr.Slider(label='Top-K', minimum=1, maximum=1000, step=1, value=10)
textbox.submit(
fn=clear_and_save_textbox,
inputs=textbox,
outputs=[textbox, saved_input],
api_name=False,
queue=False,
).then(
fn=display_input,
inputs=[saved_input, chatbot],
outputs=chatbot,
api_name=False,
queue=False,
).then(
fn=check_input_token_length,
inputs=[saved_input, chatbot, system_prompt],
api_name=False,
queue=False,
).success(
fn=generate,
inputs=[
saved_input,
chatbot,
system_prompt,
max_new_tokens,
temperature,
top_p,
top_k,
],
outputs=chatbot,
api_name=False,
)
button_event_preprocess = submit_button.click(
fn=clear_and_save_textbox,
inputs=textbox,
outputs=[textbox, saved_input],
api_name=False,
queue=False,
).then(
fn=display_input,
inputs=[saved_input, chatbot],
outputs=chatbot,
api_name=False,
queue=False,
).then(
fn=check_input_token_length,
inputs=[saved_input, chatbot, system_prompt],
api_name=False,
queue=False,
).success(
fn=generate,
inputs=[
saved_input,
chatbot,
system_prompt,
max_new_tokens,
temperature,
top_p,
top_k,
],
outputs=chatbot,
api_name=False,
)
retry_button.click(
fn=delete_prev_fn,
inputs=chatbot,
outputs=[chatbot, saved_input],
api_name=False,
queue=False,
).then(
fn=display_input,
inputs=[saved_input, chatbot],
outputs=chatbot,
api_name=False,
queue=False,
).then(
fn=generate,
inputs=[
saved_input,
chatbot,
system_prompt,
max_new_tokens,
temperature,
top_p,
top_k,
],
outputs=chatbot,
api_name=False,
)
undo_button.click(
fn=delete_prev_fn,
inputs=chatbot,
outputs=[chatbot, saved_input],
api_name=False,
queue=False,
).then(
fn=lambda x: x,
inputs=[saved_input],
outputs=textbox,
api_name=False,
queue=False,
)
clear_button.click(
fn=lambda: ([], ''),
outputs=[chatbot, saved_input],
queue=False,
api_name=False,
)
demo.queue(max_size=32).launch(show_api=False)