app.py

from huggingface_hub import InferenceClient
import gradio as gr
import random
client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
     
from prompts import GAME_MASTER, COMPRESS_HISTORY
def format_prompt(message, history):
    prompt=""
    '''
    prompt = "<s>"
    
    for user_prompt, bot_response in history:
        prompt += f"[INST] {user_prompt} [/INST]"
        prompt += f" {bot_response}</s> "
    '''
    prompt += f"[INST] {message} [/INST]"
    return prompt


temperature=0.9
top_p=0.95
repetition_penalty=1.0


def compress_history(history,temperature=temperature,top_p=top_p,repetition_penalty=repetition_penalty):
    formatted_prompt=f"{COMPRESS_HISTORY.format(history=history)}"
    generate_kwargs = dict(
        temperature=temperature,
        max_new_tokens=1024,
        top_p=top_p,
        repetition_penalty=repetition_penalty,
        do_sample=True,
        seed=random.randint(1,99999999999)
        #seed=42,
    )
    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
    output = ""
    for response in stream:
        output += response.token.text
    return output

MAX_HISTORY=100

def generate(prompt, history,max_new_tokens,temperature=temperature,top_p=top_p,repetition_penalty=repetition_penalty):
    temperature = float(temperature)
    if temperature < 1e-2:
        temperature = 1e-2
    top_p = float(top_p)

    generate_kwargs = dict(
        temperature=temperature,
        max_new_tokens=max_new_tokens,
        top_p=top_p,
        repetition_penalty=repetition_penalty,
        do_sample=True,
        seed=random.randint(1,99999999999)
        #seed=42,
    )
    cnt=0
    for ea in history:
        print (ea)
        for l in ea:
            print (l)
            cnt+=len(l.split("\n"))
    print(f'cnt:: {cnt}')
    if cnt > MAX_HISTORY:
        history = compress_history(str(history), temperature, max_new_tokens, top_p, repetition_penalty)
    formatted_prompt = format_prompt(f"{GAME_MASTER.format(history=history)}, {prompt}", history)
    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
    output = ""

    for response in stream:
        output += response.token.text
        if history:
            yield "",[(prompt,output)]
        else:
            yield "",[(prompt,output)]

    lines = output.strip().strip("\n").split("\n")
    #history=""
    for i,line in enumerate(lines):
        if line.startswith("1. "):
            print(line)
        if line.startswith("2. "):
            print(line)
        if line.startswith("3. "):
            print(line)
        if line.startswith("4. "):
            print(line)
        if line.startswith("5. "):
            print(line)            
    
    if history:
        history.append((prompt,output))
        return "",history
    else:
        return "",[(prompt,output)]

def clear_fn():
    return None,None
with gr.Blocks() as app:
    gr.HTML("""<center><h1>Mixtral 8x7B RPG</h1><h3>Role Playing Game Master</h3>""")
    chatbot = gr.Chatbot(label="Mixtral 8x7B Chatbot",show_copy_button=True)
    with gr.Row():
        with gr.Column(scale=3):
            prompt=gr.Textbox(label = "Prompt")
        with gr.Column(scale=1):
            button=gr.Button()
        
        #models_dd=gr.Dropdown(choices=[m for m in return_list],interactive=True)
    with gr.Row():
        stop_button=gr.Button("Stop")
        clear_btn = gr.Button("Clear")
    with gr.Row():
        tokens = gr.Slider(label="Max new tokens",value=1048,minimum=0,maximum=1048*10,step=64,interactive=True,info="The maximum numbers of new tokens")
    json_out=gr.JSON()
    e_box=gr.Textbox()
    #text=gr.JSON()
    #inp_query.change(search_models,inp_query,models_dd)
    #test_b=test_btn.click(itt,url,e_box)
    clear_btn.click(clear_fn,None,[prompt,chatbot])
    go=button.click(generate,[prompt,chatbot,tokens],[prompt,chatbot])
    stop_button.click(None,None,None,cancels=[go])
app.launch(show_api=False) 



'''
examples=[["Start the Game", None, None, None, None, None, ],
          ["Start a Game based in the year 1322", None, None, None, None, None,],
         ]

gr.ChatInterface(
    fn=generate,
    chatbot=gr.Chatbot(show_label=False, show_share_button=False, show_copy_button=True, likeable=True, layout="panel"),
    additional_inputs=additional_inputs,
    title="Mixtral RPG Game Master",
    examples=examples,
    concurrency_limit=20,
).launch(share=True,show_api=True)
'''