app.py

import gradio as gr
import itertools
from transformers import AutoTokenizer
import ctranslate2

generator = ctranslate2.Generator("./ct2-model" )
tokenizer = AutoTokenizer.from_pretrained( "./models", use_fast=False)

static_prompt="""ユーザー: We will now start chatting. If spoken to in English, answer in English; if spoken to in Japanese, answer in Japanese. Please take a deep breath and calm down and have a conversation.
システム: I'll try to keep calm and have a conversation.
ユーザー: その調子で頑張ってください。
システム: 分かりました。
"""
system_prompt_tokens=tokenizer.convert_ids_to_tokens(tokenizer.encode(static_prompt, add_special_tokens=False))

def inference_func(prompt, max_length=128, sampling_topk=40, sampling_topp=0.75, sampling_temperature=0.7, repetition_penalty=1.4):
    tokens = tokenizer.convert_ids_to_tokens( tokenizer.encode(prompt, add_special_tokens=False))
    results = generator.generate_batch(
        [tokens],
        static_prompt=system_prompt_tokens,
        max_length=max_length,
        sampling_topk=sampling_topk,
        sampling_topp=sampling_topp,  
        sampling_temperature=sampling_temperature,
        repetition_penalty=repetition_penalty,
        include_prompt_in_result=False,
    )
    output = tokenizer.decode(results[0].sequences_ids[0])
    return output

def make_prompt(message, chat_history, max_context_size: int = 10):
    contexts = chat_history + [[message, ""]]
    contexts = list(itertools.chain.from_iterable(contexts))
    if max_context_size > 0:
        context_size = max_context_size - 1
    else:
        context_size = 100000
    contexts = contexts[-context_size:]
    prompt = []
    for idx, context in enumerate(reversed(contexts)):
        if idx % 2 == 0:
            prompt = [f"システム: {context}"] + prompt
        else:
            prompt = [f"ユーザー: {context}"] + prompt
    prompt = "\n".join(prompt)
    return prompt

def interact_func(message, chat_history, max_context_size, max_length, sampling_topk, sampling_topp, sampling_temperature, repetition_penalty ):
    prompt = make_prompt(message, chat_history, max_context_size)
    print(f"prompt: {prompt}")
    generated = inference_func(prompt, max_length, sampling_topk, sampling_topp, sampling_temperature, repetition_penalty )
    print(f"generated: {generated}")
    chat_history.append((message, generated))
    return "", chat_history

with gr.Blocks( theme="monochrome" ) as demo:
    with gr.Accordion("Parameters", open=False):
        # max_context_size = the number of turns * 2
        max_context_size = gr.Number(value=10, label="max_context_size", precision=0)
        max_length = gr.Number(value=128, label="max_length", precision=0)
        sampling_topk = gr.Slider(0, 1000, value=40, step=0.1, label="top_k")
        sampling_topp = gr.Slider(0.1, 1.0, value=0.75, step=0.1, label="top_p")
        sampling_temperature = gr.Slider(0.0, 10.0, value=0.7, step=0.1, label="temperature")
        repetition_penalty = gr.Slider(0.0, 10.0, value=1.4, step=0.1, label="repetition_penalty")
    chatbot = gr.Chatbot( show_copy_button=True, show_share_button="RETRY" )
    msg = gr.Textbox()
    clear = gr.Button("RESET")
    msg.submit(
        interact_func,
        [msg, chatbot, max_context_size, max_length, sampling_topk, sampling_topp, sampling_temperature, repetition_penalty],
        [msg, chatbot],
    )
    clear.click(lambda: None, None, chatbot, queue=False)

if __name__ == "__main__":
    demo.launch(debug=True)