Update main.py

main.py

@@ -1,57 +1,49 @@
-from fastapi import FastAPI, Request, Form
-from fastapi.responses import HTMLResponse, FileResponse
-from fastapi.staticfiles import StaticFiles
+from fastapi import FastAPI, WebSocket
+from fastapi.responses import HTMLResponse
+from pydantic import BaseModel
 from huggingface_hub import InferenceClient
-import logging
-
-# Initialize the logger
-logging.basicConfig(level=logging.INFO)  # Adjust the logging level as needed
-logger = logging.getLogger(__name__)
-
-# Hugging Face Inference Client
-client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")
+import json
 
 app = FastAPI()
 
-# Format the prompt for the model
-def format_prompt(message, history):
-    prompt = "<s>"
-    for user_prompt, bot_response in history:
-        prompt += f"[INST] {user_prompt} [/INST]"
-        prompt += f" {bot_response}</s> "
-    prompt += f"[INST] {message} [/INST]"
-    return prompt
-
-# Generate response from the model
-def generate(prompt: str, history: list, temperature: float = 0.9, max_new_tokens: int = 512, top_p: float = 0.95, top_k: int = 50, repetition_penalty: float = 1.0) -> str:
-    try:
-        formatted_prompt = format_prompt(prompt, history)
-        logger.info(f"Formatted prompt: {formatted_prompt}")
-        bot_response = client.text_generation(
-            formatted_prompt, temperature=temperature, max_new_tokens=max_new_tokens,
-            top_p=top_p, top_k=top_k, repetition_penalty=repetition_penalty, stream=True,
-            details=True, return_full_text=False
-        )
-        output = [response.token.text.strip() for response in bot_response if response.token.text.strip()]
-        logger.info(f"Bot response tokens: {output}")
-        return " ".join(output)
-    except Exception as e:
-        logger.error(f"Error generating text: {e}")
-        return ""
-
-@app.post("/generate/")
-async def generate_chat(request: Request, prompt: str = Form(...), history: str = Form(...), temperature: float = Form(0.9), max_new_tokens: int = Form(512), top_p: float = Form(0.95), top_k: int = Form(50), repetition_penalty: float = Form(1.0)):
-    history = eval(history)  # Convert history string back to list
-    response = generate(prompt, history, temperature, max_new_tokens, top_p, top_k, repetition_penalty)
-    
-    # Remove any HTML tags from the response
-    import re
-    response = re.sub('<[^<]+?>', '', response)
-    
-    return {"response": response}
+client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")
 
-app.mount("/", StaticFiles(directory="static", html=True), name="static")
+class MessageRequest(BaseModel):
+    message: str
+    history: list[tuple[str, str]]
+    system_message: str
+    max_tokens: int
+    temperature: float
+    top_p: float
 
 @app.get("/")
-def index() -> FileResponse:
-    return FileResponse(path="static/index.html", media_type="text/html")
+async def get():
+    with open("index.html", "r") as file:
+        return HTMLResponse(content=file.read(), media_type="text/html")
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    await websocket.accept()
+    while True:
+        data = await websocket.receive_text()
+        request = MessageRequest(**json.loads(data))
+        messages = [{"role": "system", "content": request.system_message}]
+        for val in request.history:
+            if val[0]:
+                messages.append({"role": "user", "content": val[0]})
+            if val[1]:
+                messages.append({"role": "assistant", "content": val[1]})
+        messages.append({"role": "user", "content": request.message})
+
+        response = ""
+        for message in client.chat_completion(
+            messages,
+            max_tokens=request.max_tokens,
+            stream=True,
+            temperature=request.temperature,
+            top_p=request.top_p,
+        ):
+            token = message.choices[0].delta.content
+            response += token
+            await websocket.send_text(json.dumps({"token": token}))
+