Spaces:
Runtime error
Runtime error
import argparse | |
from collections import defaultdict | |
import datetime | |
import json | |
import os | |
import time | |
import uuid | |
import gradio as gr | |
import requests | |
from fastchat.conversation import ( | |
Conversation, | |
compute_skip_echo_len, | |
SeparatorStyle, | |
) | |
from fastchat.constants import LOGDIR | |
from fastchat.utils import ( | |
build_logger, | |
server_error_msg, | |
violates_moderation, | |
moderation_msg, | |
) | |
from fastchat.serve.gradio_patch import Chatbot as grChatbot | |
from fastchat.serve.gradio_css import code_highlight_css | |
logger = build_logger("gradio_web_server", "gradio_web_server.log") | |
headers = {"User-Agent": "NeuralChat Client"} | |
no_change_btn = gr.Button.update() | |
enable_btn = gr.Button.update(interactive=True) | |
disable_btn = gr.Button.update(interactive=False) | |
controller_url = None | |
enable_moderation = False | |
conv_template_bf16 = Conversation( | |
system="A chat between a curious human and an artificial intelligence assistant. " | |
"The assistant gives helpful, detailed, and polite answers to the human's questions.", | |
roles=("Human", "Assistant"), | |
messages=(), | |
offset=0, | |
sep_style=SeparatorStyle.SINGLE, | |
sep="\n", | |
sep2="</s>", | |
) | |
def set_global_vars(controller_url_, enable_moderation_): | |
global controller_url, enable_moderation | |
controller_url = controller_url_ | |
enable_moderation = enable_moderation_ | |
def get_conv_log_filename(): | |
t = datetime.datetime.now() | |
name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json") | |
return name | |
def get_model_list(controller_url): | |
ret = requests.post(controller_url + "/refresh_all_workers") | |
assert ret.status_code == 200 | |
ret = requests.post(controller_url + "/list_models") | |
models = ret.json()["models"] | |
logger.info(f"Models: {models}") | |
return models | |
get_window_url_params = """ | |
function() { | |
const params = new URLSearchParams(window.location.search); | |
url_params = Object.fromEntries(params); | |
console.log("url_params", url_params); | |
return url_params; | |
} | |
""" | |
def load_demo_single(models, url_params): | |
dropdown_update = gr.Dropdown.update(visible=True) | |
if "model" in url_params: | |
model = url_params["model"] | |
if model in models: | |
dropdown_update = gr.Dropdown.update(value=model, visible=True) | |
state = None | |
return ( | |
state, | |
dropdown_update, | |
gr.Chatbot.update(visible=True), | |
gr.Textbox.update(visible=True), | |
gr.Button.update(visible=True), | |
gr.Row.update(visible=True), | |
gr.Accordion.update(visible=False), | |
) | |
def load_demo(url_params, request: gr.Request): | |
logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}") | |
return load_demo_single(models, url_params) | |
def vote_last_response(state, vote_type, model_selector, request: gr.Request): | |
with open(get_conv_log_filename(), "a") as fout: | |
data = { | |
"tstamp": round(time.time(), 4), | |
"type": vote_type, | |
"model": model_selector, | |
"state": state.dict(), | |
"ip": request.client.host, | |
} | |
fout.write(json.dumps(data) + "\n") | |
def upvote_last_response(state, model_selector, request: gr.Request): | |
logger.info(f"upvote. ip: {request.client.host}") | |
vote_last_response(state, "upvote", model_selector, request) | |
return ("",) + (disable_btn,) * 3 | |
def downvote_last_response(state, model_selector, request: gr.Request): | |
logger.info(f"downvote. ip: {request.client.host}") | |
vote_last_response(state, "downvote", model_selector, request) | |
return ("",) + (disable_btn,) * 3 | |
def flag_last_response(state, model_selector, request: gr.Request): | |
logger.info(f"flag. ip: {request.client.host}") | |
vote_last_response(state, "flag", model_selector, request) | |
return ("",) + (disable_btn,) * 3 | |
def regenerate(state, request: gr.Request): | |
logger.info(f"regenerate. ip: {request.client.host}") | |
state.messages[-1][-1] = None | |
state.skip_next = False | |
return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5 | |
def clear_history(request: gr.Request): | |
logger.info(f"clear_history. ip: {request.client.host}") | |
state = None | |
return (state, [], "") + (disable_btn,) * 5 | |
def add_text(state, text, request: gr.Request): | |
logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}") | |
if state is None: | |
state = conv_template_bf16.copy() | |
if len(text) <= 0: | |
state.skip_next = True | |
return (state, state.to_gradio_chatbot(), "") + (no_change_btn,) * 5 | |
if enable_moderation: | |
flagged = violates_moderation(text) | |
if flagged: | |
logger.info(f"violate moderation. ip: {request.client.host}. text: {text}") | |
state.skip_next = True | |
return (state, state.to_gradio_chatbot(), moderation_msg) + ( | |
no_change_btn, | |
) * 5 | |
text = text[:1536] # Hard cut-off | |
state.append_message(state.roles[0], text) | |
state.append_message(state.roles[1], None) | |
state.skip_next = False | |
return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5 | |
def post_process_code(code): | |
sep = "\n```" | |
if sep in code: | |
blocks = code.split(sep) | |
if len(blocks) % 2 == 1: | |
for i in range(1, len(blocks), 2): | |
blocks[i] = blocks[i].replace("\\_", "_") | |
code = sep.join(blocks) | |
return code | |
def http_bot(state, model_selector, temperature, max_new_tokens, request: gr.Request): | |
logger.info(f"http_bot. ip: {request.client.host}") | |
start_tstamp = time.time() | |
model_name = model_selector | |
temperature = float(temperature) | |
max_new_tokens = int(max_new_tokens) | |
if state.skip_next: | |
# This generate call is skipped due to invalid inputs | |
yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5 | |
return | |
if len(state.messages) == state.offset + 2: | |
# First round of conversation | |
new_state = conv_template_bf16.copy() | |
new_state.conv_id = uuid.uuid4().hex | |
new_state.model_name = state.model_name or model_selector | |
new_state.append_message(new_state.roles[0], state.messages[-2][1]) | |
new_state.append_message(new_state.roles[1], None) | |
state = new_state | |
# Query worker address | |
ret = requests.post( | |
controller_url + "/get_worker_address", json={"model": model_name} | |
) | |
worker_addr = ret.json()["address"] | |
logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}") | |
# No available worker | |
if worker_addr == "": | |
state.messages[-1][-1] = server_error_msg | |
yield ( | |
state, | |
state.to_gradio_chatbot(), | |
disable_btn, | |
disable_btn, | |
disable_btn, | |
enable_btn, | |
enable_btn, | |
) | |
return | |
# Construct prompt | |
prompt = state.get_prompt() | |
skip_echo_len = compute_skip_echo_len(model_name, state, prompt) | |
# Make requests | |
pload = { | |
"model": model_name, | |
"prompt": prompt, | |
"temperature": temperature, | |
"max_new_tokens": max_new_tokens, | |
"stop": "</s>" | |
} | |
logger.info(f"==== request ====\n{pload}") | |
start_time = time.time() | |
state.messages[-1][-1] = "▌" | |
yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5 | |
try: | |
# Stream output | |
response = requests.post( | |
worker_addr + "/worker_generate_stream", | |
headers=headers, | |
json=pload, | |
stream=True, | |
timeout=20, | |
) | |
for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): | |
if chunk: | |
data = json.loads(chunk.decode()) | |
if data["error_code"] == 0: | |
output = data["text"][skip_echo_len:].strip() | |
output = post_process_code(output) | |
state.messages[-1][-1] = output + "▌" | |
yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5 | |
else: | |
output = data["text"] + f" (error_code: {data['error_code']})" | |
state.messages[-1][-1] = output | |
yield (state, state.to_gradio_chatbot()) + ( | |
disable_btn, | |
disable_btn, | |
disable_btn, | |
enable_btn, | |
enable_btn, | |
) | |
return | |
time.sleep(0.02) | |
except requests.exceptions.RequestException as e: | |
state.messages[-1][-1] = server_error_msg + f" (error_code: 4)" | |
yield (state, state.to_gradio_chatbot()) + ( | |
disable_btn, | |
disable_btn, | |
disable_btn, | |
enable_btn, | |
enable_btn, | |
) | |
return | |
finish_tstamp = time.time() - start_time | |
elapsed_time = "\n✅generation elapsed time: {}s".format(round(finish_tstamp, 4)) | |
state.messages[-1][-1] = state.messages[-1][-1][:-1] + elapsed_time | |
yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5 | |
logger.info(f"{output}") | |
with open(get_conv_log_filename(), "a") as fout: | |
data = { | |
"tstamp": round(finish_tstamp, 4), | |
"type": "chat", | |
"model": model_name, | |
"gen_params": { | |
"temperature": temperature, | |
"max_new_tokens": max_new_tokens, | |
}, | |
"start": round(start_tstamp, 4), | |
"finish": round(start_tstamp, 4), | |
"state": state.dict(), | |
"ip": request.client.host, | |
} | |
fout.write(json.dumps(data) + "\n") | |
block_css = ( | |
code_highlight_css | |
+ """ | |
pre { | |
white-space: pre-wrap; /* Since CSS 2.1 */ | |
white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ | |
white-space: -pre-wrap; /* Opera 4-6 */ | |
white-space: -o-pre-wrap; /* Opera 7 */ | |
word-wrap: break-word; /* Internet Explorer 5.5+ */ | |
} | |
#notice_markdown th { | |
display: none; | |
} | |
#notice_markdown { | |
text-align: center; | |
background: #874bec; | |
padding: 1%; | |
} | |
#notice_markdown h1, #notice_markdown h4 { | |
color: #fff; | |
margin-top: 0; | |
} | |
gradio-app { | |
background: linear-gradient(to bottom, #ba97d8, #5400ff) !important; | |
padding: 3%; | |
} | |
.gradio-container { | |
margin: 0 auto !important; | |
width: 70% !important; | |
padding: 0 !important; | |
} | |
#chatbot { | |
border-style: solid; | |
overflow: visible; | |
margin: 1% 4%; | |
width: 90%; | |
box-shadow: 0 15px 15px -5px rgba(0, 0, 0, 0.2); | |
border: 1px solid #ddd; | |
} | |
#text-box-style, #btn-style { | |
width: 90%; | |
margin: 1% 4%; | |
} | |
.user, .bot { | |
width: 80% !important; | |
} | |
.bot { | |
white-space: pre-wrap !important; | |
line-height: 1.3 !important; | |
display: flex; | |
flex-direction: column; | |
justify-content: flex-start; | |
} | |
#btn-send-style { | |
background: rgb(0, 180, 50); | |
color: #fff; | |
} | |
#btn-list-style { | |
background: #eee0; | |
border: 1px solid #691ef7; | |
} | |
""" | |
) | |
def build_single_model_ui(models): | |
notice_markdown = """ | |
# 🤖 NeuralChat | |
#### deployed on 4th Gen Intel Xeon Scalable Processors codenamed Sapphire Rapids | |
""" | |
learn_more_markdown = """ | |
""" | |
state = gr.State() | |
notice = gr.Markdown(notice_markdown, elem_id="notice_markdown") | |
with gr.Row(elem_id="model_selector_row", visible=False): | |
model_selector = gr.Dropdown( | |
choices=models, | |
value=models[0] if len(models) > 0 else "", | |
interactive=True, | |
show_label=False, | |
).style(container=False) | |
chatbot = grChatbot(elem_id="chatbot", visible=False).style(height=550) | |
with gr.Row(elem_id="text-box-style"): | |
with gr.Column(scale=20): | |
textbox = gr.Textbox( | |
show_label=False, | |
placeholder="Enter text and press ENTER", | |
visible=False, | |
).style(container=False) | |
with gr.Column(scale=1, min_width=50): | |
send_btn = gr.Button(value="Send", visible=False, elem_id="btn-send-style") | |
with gr.Row(visible=False, elem_id="btn-style") as button_row: | |
upvote_btn = gr.Button(value="👍 Upvote", interactive=False, elem_id="btn-list-style") | |
downvote_btn = gr.Button(value="👎 Downvote", interactive=False, elem_id="btn-list-style") | |
flag_btn = gr.Button(value="⚠️ Flag", interactive=False, visible=False, elem_id="btn-list-style") | |
# stop_btn = gr.Button(value="⏹️ Stop Generation", interactive=False) | |
regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=False, elem_id="btn-list-style") | |
clear_btn = gr.Button(value="🗑️ Clear history", interactive=False, elem_id="btn-list-style") | |
with gr.Accordion("Parameters", open=False, visible=False) as parameter_row: | |
temperature = gr.Slider( | |
minimum=0.0, | |
maximum=1.0, | |
value=0.95, | |
step=0.1, | |
interactive=True, | |
label="Temperature", | |
) | |
max_output_tokens = gr.Slider( | |
minimum=0, | |
maximum=1024, | |
value=512, | |
step=64, | |
interactive=True, | |
label="Max output tokens", | |
) | |
gr.Markdown(learn_more_markdown) | |
# Register listeners | |
btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn] | |
upvote_btn.click( | |
upvote_last_response, | |
[state, model_selector], | |
[textbox, upvote_btn, downvote_btn, flag_btn], | |
) | |
downvote_btn.click( | |
downvote_last_response, | |
[state, model_selector], | |
[textbox, upvote_btn, downvote_btn, flag_btn], | |
) | |
flag_btn.click( | |
flag_last_response, | |
[state, model_selector], | |
[textbox, upvote_btn, downvote_btn, flag_btn], | |
) | |
regenerate_btn.click(regenerate, state, [state, chatbot, textbox] + btn_list).then( | |
http_bot, | |
[state, model_selector, temperature, max_output_tokens], | |
[state, chatbot] + btn_list, | |
) | |
clear_btn.click(clear_history, None, [state, chatbot, textbox] + btn_list) | |
model_selector.change(clear_history, None, [state, chatbot, textbox] + btn_list) | |
textbox.submit( | |
add_text, [state, textbox], [state, chatbot, textbox] + btn_list | |
).then( | |
http_bot, | |
[state, model_selector, temperature, max_output_tokens], | |
[state, chatbot] + btn_list, | |
) | |
send_btn.click( | |
add_text, [state, textbox], [state, chatbot, textbox] + btn_list | |
).then( | |
http_bot, | |
[state, model_selector, temperature, max_output_tokens], | |
[state, chatbot] + btn_list, | |
) | |
return state, model_selector, chatbot, textbox, send_btn, button_row, parameter_row | |
def build_demo(models): | |
with gr.Blocks( | |
title="NeuralChat · Intel", | |
theme=gr.themes.Base(), | |
css=block_css, | |
) as demo: | |
url_params = gr.JSON(visible=False) | |
( | |
state, | |
model_selector, | |
chatbot, | |
textbox, | |
send_btn, | |
button_row, | |
parameter_row, | |
) = build_single_model_ui(models) | |
if model_list_mode == "once": | |
demo.load( | |
load_demo, | |
[url_params], | |
[ | |
state, | |
model_selector, | |
chatbot, | |
textbox, | |
send_btn, | |
button_row, | |
parameter_row, | |
], | |
_js=get_window_url_params, | |
) | |
else: | |
raise ValueError(f"Unknown model list mode: {model_list_mode}") | |
return demo | |
if __name__ == "__main__": | |
controller_url = "http://mlp-dgx-01.sh.intel.com:21001" | |
host = "mlp-dgx-01.sh.intel.com" | |
# port = "mlp-dgx-01.sh.intel.com" | |
concurrency_count = 10 | |
model_list_mode = "once" | |
share = True | |
moderate = False | |
set_global_vars(controller_url, moderate) | |
models = get_model_list(controller_url) | |
demo = build_demo(models) | |
demo.queue( | |
concurrency_count=concurrency_count, status_update_rate=10, api_open=False | |
).launch( | |
server_name=host, share=share, max_threads=200 | |
) | |