app.py

import gradio as gr
from huggingface_hub import InferenceClient
from models import LlmBot



llm_bot = LlmBot("mistralai/Mixtral-8x7B-Instruct-v0.1")


        
def generate(prompt, history, name, description, user_name, max_new_tokens):
    stream = llm_bot.call(prompt, history, name, description, user_name, max_new_tokens)
    output = ""
    for response in stream:
        output += response.token.text
        yield output
    return output


additional_inputs = [
    gr.Textbox(
        label="Name",
        max_lines=1,
        interactive=True,
    ),
    gr.Textbox(
        label="Description",
        max_lines=1,
        interactive=True,
    ),
    gr.Textbox(
        label="How to call you",
        max_lines=1,
        interactive=True,
    ),
    gr.Slider(
        label="Max new tokens",
        value=256,
        minimum=0,
        maximum=1048,
        step=64,
        interactive=True,
        info="The maximum numbers of new tokens",
    ),
]

examples = [
    ["tell me about your day", "Messmer, the Impaler", "Messmer, son of Queen Marika the Eternal, was born with a dark serpent inside him, called the Abyssal Serpent, contained by Marika through a seal in place of one of his eyes. Constantly accompanied by winged snakes, Messmer and Commander Gaius acted as 'older brothers' to General Radahn. Before the Shattering, Marika tasked Messmer with purging the tower-dwelling people of the Land of Shadow. Even after being abandoned by Marika, he continued this purge with zeal. Among his army, Black Knight Commander Andreas and Black Knight Captain Huw, initially considered brothers-in-arms, eventually rebelled upon discovering his serpentine nature.","Marika",256],
    ["tell me about your day","Ada Wong","Ada Wong's early life is shrouded in mystery, with unconfirmed details about her ethnicity, nationality, and birth. She has mentioned a possibly fabricated story of being born around 1974 in Saigon, Vietnam, to a wealthy family that fled to the United States post-Vietnam War. As a young adult, she engaged in criminal activities and was eventually recruited by Albert Wesker for a bioweapons corporation known as 'the Organization.' Around 1997, she infiltrated the Umbrella Corporation's Arklay Laboratory, forming a relationship with Dr. John Clemens to gather information. During the 1998 t-Virus outbreak, Wong managed to escape undetected despite being present at the facility.","Leon",128]    
]


iface = gr.ChatInterface(theme='upsatwal/mlsc_tiet',
    fn=generate,
    chatbot=gr.Chatbot(show_label=False, show_share_button=False, show_copy_button=True, likeable=True, layout="panel"),
    additional_inputs=additional_inputs,
    title="Mixtral 46.7B",
    examples=examples,
    concurrency_limit=20,
    description="This application has mandatory fields in the additional inputs such as name, description and your name. You can adjust the length of the response in token length. There are also two example of inputs"
)

iface.queue(api_open=True)
iface.launch(share=True, debug=True, inbrowser=True)