|
import time |
|
import gradio as gr |
|
|
|
API_URL = "https://joi-20b.ngrok.io/generate_stream" |
|
|
|
def predict(inputs, history=[], top_p, temperature, top_k, repetition_penalty): |
|
if not inputs.startswith("User: "): |
|
inputs = "User: " + inputs + "\n" |
|
payload = { |
|
"inputs": inputs, |
|
"parameters": { |
|
"details": True, |
|
"do_sample": True, |
|
"max_new_tokens": 20, |
|
"repetition_penalty": 1.03, |
|
"seed": 0, |
|
"stop": ["photographer"], |
|
"temperature": 0.5, |
|
"top_k": 10, |
|
"top_p": 0.95 |
|
} |
|
} |
|
|
|
headers = { |
|
'accept': 'text/event-stream', |
|
'Content-Type': 'application/json' |
|
} |
|
|
|
history.append(inputs) |
|
response = requests.post(API_URL, headers=headers, json=payload) |
|
responses = response.text.split("\n\n") |
|
|
|
partial_words = "" |
|
for idx, resp in enumerate(responses): |
|
if resp[:4] == 'data': |
|
partial_words = partial_words + json.loads(resp[5:])['token']['text'] |
|
|
|
time.sleep(0.05) |
|
if idx == 0: |
|
history.append(" " + partial_words) |
|
else: |
|
history[-1] = partial_words |
|
|
|
chat = [(history[i], history[i + 1]) for i in range(0, len(history) - 1, 2) ] |
|
|
|
yield chat, history |
|
|
|
title = """<h1 align="center">Gradio Streaming</h1>""" |
|
description = """Language models can be conditioned to act like dialogue agents through a conversational prompt that typically takes the form: |
|
``` |
|
User: <utterance> |
|
Assistant: <utterance> |
|
User: <utterance> |
|
Assistant: <utterance> |
|
... |
|
``` |
|
In this app, you can explore the outputs of the Joi alpha language models. |
|
""" |
|
|
|
with gr.Blocks(css = "#chatbot {height: 400px, overflow: auto;}") as demo: |
|
gr.HTML(title) |
|
inputs = gr.Textbox(placeholder= "Hi my name is Joe.", label= "Type an input and press Enter") |
|
chatbot = gr.Chatbot(elem_id='chatbot') |
|
state = gr.State([]) |
|
b1 = gr.Button() |
|
|
|
|
|
with gr.Accordion("Parameters", open=False): |
|
top_p = gr.Slider( minimum=-0, maximum=1.0, value=0.95, step=0.05, interactive=True, label="Top-p (nucleus sampling)",) |
|
temperature = gr.Slider( minimum=-0, maximum=5.0, value=0.5, step=0.1, interactive=True, label="Temperature",) |
|
top_k = gr.Slider( minimum=1, maximum=50, value=4, step=1, interactive=True, label="Top-k",) |
|
repetition_penalty = gr.Slider( minimum=0.1, maximum=3.0, value=1.03, step=0.01, interactive=True, label="Repetition Penalty", ) |
|
|
|
|
|
|
|
inputs.submit( inference_chat, [inputs, state, top_p, temperature, top_k, repetition_penalty,], [chatbot, state],) |
|
b1.click( inference_chat, [inputs, state, top_p, temperature, top_k, repetition_penalty,], [chatbot, state],) |
|
|
|
gr.HTML(description) |
|
demo.queue().launch(debug=True) |
|
|
|
|
