Update app.py

app.py

@@ -1,3 +1,33 @@
 import gradio as gr
+import torch
+from transformers import pipeline
 
-gr.load("models/HuggingFaceH4/zephyr-7b-beta").launch()
+pipe = pipeline("text-generation", model="HuggingFaceH4/zephyr-7b-beta", torch_dtype=torch.bfloat16, device_map="auto")
+
+messages = [
+    {
+        "role": "system",
+        "content": "You are a friendly chatbot who always responds in the style of a pirate",
+    },
+    # {"role": "user", "content": "How many helicopters can a human eat in one sitting?"},
+]
+
+
+def random_response(message, history):
+    msg = messages.copy()
+    for m in history:
+        q, a = m
+        msg.append({"role": "user", "content": q})
+        msg.append({"role": "assistant", "content": a})
+    msg.append({"role": "user", "content": message})
+    prompt = pipe.tokenizer.apply_chat_template(msg, tokenize=False, add_generation_prompt=True)
+    outputs = pipe(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+    output = outputs[0]["generated_text"]
+    messages.append({"role": "assistant", "content": output})
+    response_start = output.rfind('<|assistant|>')
+    return output[response_start + len('<|assistant|>'):]
+
+
+demo = gr.ChatInterface(random_response)
+
+demo.launch()