YOLO

app.py

@@ -1,3 +1,35 @@
 import gradio as gr
+from transformers import AutoModel, AutoTokenizer
 
-gr.Interface.load("models/fnlp/moss-moon-003-sft-plugin").launch()
+tokenizer = AutoTokenizer.from_pretrained("fnlp/moss-moon-003-sft-plugin", trust_remote_code=True)
+model = AutoModel.from_pretrained("fnlp/moss-moon-003-sft-plugin", trust_remote_code=True).half().cuda()
+model = model.eval()
+
+def predict(input, history=None):
+    if history is None:
+        history = []
+    prompt = ""
+    for (q, r) in history:
+        prompt += "<|Human|>: " + q + "<eoh>\n"
+        prompt += "<|MOSS|>: " + r + "\n"
+    prompt += "<|Human|>: " + input + "<eoh>\n<|MOSS|>:"
+    inputs = tokenizer(prompt, return_tensors="pt")
+    outputs = model.generate(**inputs, do_sample=True, temperature=0.7, top_p=0.8, repetition_penalty=1.1, max_new_tokens=512)
+    response = tokenizer.decode(outputs[0])
+    completion = response[len(prompt)+2:]
+    history = history + [(input, completion)]
+    return history, history
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("moss-moon-003-sft-plugin")
+    state = gr.State([])
+    chatbot = gr.Chatbot([], elem_id="chatbot").style(height=400)
+    with gr.Row():
+        with gr.Column(scale=4):
+            txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter").style(container=False)
+        with gr.Column(scale=1):
+            button = gr.Button("Generate")
+    txt.submit(predict, [txt, state], [chatbot, state])
+    button.click(predict, [txt, state], [chatbot, state])
+demo.queue().launch()