main.py

from fastapi import FastAPI, Request, Form
from fastapi.responses import HTMLResponse, FileResponse
from fastapi.staticfiles import StaticFiles
from huggingface_hub import InferenceClient
import logging

# Initialize the logger
logging.basicConfig(level=logging.INFO)  # Adjust the logging level as needed
logger = logging.getLogger(__name__)

# Hugging Face Inference Client
client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")

app = FastAPI()

# Format the prompt for the model
def format_prompt(message, history):
    prompt = "<s>"
    for user_prompt, bot_response in history:
        prompt += f"[INST] {user_prompt} [/INST]"
        prompt += f" {bot_response}</s> "
    prompt += f"[INST] {message} [/INST]"
    return prompt

# Generate response from the model
def generate(prompt: str, history: list, temperature: float = 0.9, max_new_tokens: int = 512, top_p: float = 0.95, repetition_penalty: float = 1.0) -> str:
    try:
        formatted_prompt = format_prompt(prompt, history)
        logger.info(f"Formatted prompt: {formatted_prompt}")
        bot_response = client.text_generation(
            formatted_prompt, temperature=temperature, max_new_tokens=max_new_tokens,
            top_p=top_p, repetition_penalty=repetition_penalty, stream=True,
            details=True, return_full_text=False
        )
        output = [response.token.text.strip() for response in bot_response if response.token.text.strip()]
        logger.info(f"Bot response tokens: {output}")
        return " ".join(output)
    except Exception as e:
        logger.error(f"Error generating text: {e}")
        return ""

@app.post("/generate/")
async def generate_chat(request: Request, prompt: str = Form(...), history: str = Form(...), temperature: float = Form(0.9), max_new_tokens: int = Form(512), top_p: float = Form(0.95), repetition_penalty: float = Form(1.0)):
    history = eval(history)  # Convert history string back to list
    response = generate(prompt, history, temperature, max_new_tokens, top_p, repetition_penalty)
    
    # Remove any HTML tags from the response
    import re
    response = re.sub('<[^<]+?>', '', response)
    
    return {"response": response}

app.mount("/", StaticFiles(directory="static", html=True), name="static")

@app.get("/")
def index() -> FileResponse:
    return FileResponse(path="static/index.html", media_type="text/html")