Update app.py

app.py

@@ -1,18 +1,25 @@
 import gradio as gr
+import os
 import spaces
+import torch
+from transformers import AutoTokenizer, AutoModelForChatGPT
 
-from vllm import LLM, SamplingParams
-
-llm = LLM(model="meta-llama/Llama-2-7B-Chat-hf")
-sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
+model_path = "cognitivecomputations/dolphin-2.7-mixtral-8x7b"
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = AutoModelForChatGPT.from_pretrained(model_path)
 
 @spaces.GPU
-def pipe(text: str):
-    prompt = [text]
-    tokens = llm.generate(prompt, sampling_params)
-    output = (output.outputs[0].text for output in tokens)
-    return output[0]
+def chat(prompt):
+    input_ids = tokenizer.encode(prompt, return_tensors="pt")
+    output = model.generate(input_ids, max_length=1024, num_return_sequences=1, top_p=0.9, top_k=50, num_beams=2, early_stopping=True)
+    response = tokenizer.decode(output[0], skip_special_tokens=True)
+    return response
+
+demo = gr.Interface(
+    fn=chat,
+    inputs=gr.Textbox(value="Hello!", lines=5),
+    outputs=gr.Textbox(label="Bot's Response", lines=5)
+)
 
 if __name__ == "__main__":
-    interface = gr.Interface(pipe, gr.Textbox(label="Prompt"), gr.Textbox(label="Response"), title="Text Completion")
-    interface.launch()
+    demo.launch()