app.py

from transformers import AutoTokenizer, AutoModelForCausalLM
import gradio as gr

prologue = """quality: high

[System]
Assistant is a distilled language model trained by the community.<|STK_SP|>

[System]
<|STK_SP|>

[User]"""

tokenizer = AutoTokenizer.from_pretrained("./openchatgpt-neo-r1/", use_fast=True)
model = AutoModelForCausalLM.from_pretrained("./openchatgpt-neo-r1/")

def chat(inpt, max_new_tokens, top_k, top_p, temperature, repetition_penalty):
    inputs = tokenizer(f"{prologue}\n{inpt}<|STK_SP|>\n\n[Assistant]\n", return_tensors="pt").input_ids
    outputs = model.generate(inputs, max_new_tokens=max_new_tokens, do_sample=True, top_k=top_k, top_p=top_p, eos_token_id=tokenizer.sep_token_id, temperature=temperature)
    return tokenizer.batch_decode([i[len(inputs[0]):] for i in outputs], skip_special_tokens=True)[0]

gr.Interface(
    fn=chat,
    inputs=["textbox", gr.Slider(767, 2048 + 1), gr.Slider(0, 100, value=50), gr.Slider(0, 1, step=0.01, value=0.95), gr.Slider(0.01, 1, step=0.01, value=1), gr.Slider(1, 100, step=0.5)],
    outputs=[gr.Textbox(label="Assistant says")],
    examples=[
        ["Hi!"],
        ["Hello, I am trying to use the fft function in Python, but I am not sure how to interpret the results. Can you explain how to interpret the output of the fft function?"],
        ["Hello, I have a question about American history. Who is the current Vice President of the United States?"],
        ["Hello, I have a question about quantum computing. Can quantum computers solve NP-complete problems in polynomial time?"],
        ["I'm wondering how to make an apple pie?"],
        ["Hey, what are some pros and cons of using a neural network for image recognition?"],
        ["Hi! I want to build a website using python and Flask. I just want to know what are the requirements for building a website using Flask?"],
        ["Hi, I want to know about the GPT-3 model. Could you provide me some information about it?"],
        ["Please, help me with GPT-2 training"],
        ["Please, help me understand LLMs!"],
        ["What are the health benefits of ginger?"],
        ["What is the fundamental theorem of algebra?"],
        ["What is the meaning of life?"],
        ["What is the origin of the word 'sushi'?"],
        ["What's the difference between a chatbot and an AI?"],
        ["What's the difference between a monad and a functor in functional programming?"],
    ]
).launch()