app_gpu.py

import gradio as gr
from vllm import LLM, SamplingParams

model_name = "ShynBui/OU_student_book_v5_1.5"
llm = LLM(model=model_name)

def generate_text(prompt):
    sampling_params = SamplingParams(max_tokens=2048, temperature=0.05, top_p=0.7, top_k=10)
    outputs = llm.generate([prompt], sampling_params)
    return outputs[0].outputs[0].text

demo = gr.Interface(fn=generate_text, inputs="text", outputs="text")
demo.launch()