app.py

import gradio as gr
from transformers import pipeline, AutoTokenizer

def load_model(model_name):
    return pipeline("text-generation", model=model_name, device="cpu")

def generate(
    model_name,
    system_input,
    user_initial_message,
    assistant_initial_message,
    user_input,
):
    pipe = load_model(model_name)
    
    message_template = [
        {"role": "system", "content": system_input},
        {"role": "user", "content": user_initial_message},
        {"role": "assistant", "content": assistant_initial_message},
        {"role": "user", "content": user_input},
    ]
    
    prompt = pipe.tokenizer.apply_chat_template(message_template, tokenize=False, add_generation_prompt=True)
    
    if model_name == "Felladrin/Pythia-31M-Chat-v1":
        outputs = pipe(prompt, max_new_tokens=1024, do_sample=True, temperature=0.4, top_k=7, top_p=0.25, repetition_penalty=1.0016)
    elif model_name == "Felladrin/Llama-160M-Chat-v1":
        outputs = pipe(prompt, max_new_tokens=1024, penalty_alpha=0.5, top_k=5, repetition_penalty=1.01)
    elif model_name == "Felladrin/TinyMistral-248M-SFT-v4":
        outputs = pipe(prompt, max_new_tokens=1024, penalty_alpha=0.5, top_k=5, repetition_penalty=1.0)
    elif model_name == "Felladrin/Smol-Llama-101M-Chat-v1":
        outputs = pipe(prompt, max_new_tokens=1024, penalty_alpha=0.5, top_k=5, repetition_penalty=1.0, add_special_tokens=True)
    else:
        outputs = pipe(prompt, max_new_tokens=1024, do_sample=True, temperature=0.9, top_k=50, top_p=0.95, repetition_penalty=1.2)
        
    return outputs[0]["generated_text"]

model_choices = ["Felladrin/Llama-160M-Chat-v1", "Felladrin/Smol-Llama-101M-Chat-v1", "Felladrin/TinyMistral-248M-SFT-v4", "Felladrin/Pythia-31M-Chat-v1"]

g = gr.Interface(
    fn=generate,
    inputs=[
        gr.components.Dropdown(choices=model_choices, label="Model", value=model_choices[0], interactive=True),
        gr.components.Textbox(lines=2, label="System Message", value="You are a highly knowledgeable and friendly assistant. Your goal is to understand and respond to user inquiries with accuracy and clarity. You're adept at providing detailed explanations, concise summaries, and insightful responses. Your interactions are always respectful, helpful, and focused on delivering the most relevant information to the user."),
        gr.components.Textbox(lines=2, label="User Initial Message", value="Hi!"),
        gr.components.Textbox(lines=2, label="Assistant Initial Message", value="Hello there. How can I assist you today?"),
        gr.components.Textbox(lines=2, label="User Message", value="Tell me something curious about the Earth!"),
    ],
    outputs=[gr.Textbox(lines=10, label="Output")],
    title="A place to try out text-generation models fine-tuned by Felladrin",
    concurrency_limit=1
)

g.launch(max_threads=2)