app.py

import gradio as gr
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig

base_model_id = "mistralai/Mistral-7B-v0.1"
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

# Load the fine-tuned model "Tonic/mistralmed"
model = AutoModelForCausalLM.from_pretrained("Tonic/mistralmed", quantization_config=bnb_config)

tokenizer = AutoTokenizer.from_pretrained("Tonic/mistralmed", trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = 'left'

class ChatBot:
    def __init__(self):
        self.history = []

    def predict(self, input):
        new_user_input_ids = tokenizer.encode(input + tokenizer.eos_token, return_tensors="pt")
        flat_history = [item for sublist in self.history for item in sublist]
        flat_history_tensor = torch.tensor(flat_history).unsqueeze(dim=0)
        bot_input_ids = torch.cat([flat_history_tensor, new_user_input_ids], dim=-1) if self.history else new_user_input_ids
        chat_history_ids = model.generate(bot_input_ids, max_length=2000, pad_token_id=tokenizer.eos_token_id)
        self.history.append(chat_history_ids[:, bot_input_ids.shape[-1]:].tolist()[0])
        response = tokenizer.decode(chat_history_ids[:, bot_input_ids.shape[-1]:][0], skip_special_tokens=True)
        return response

bot = ChatBot()

title = "👋🏻Welcome to Tonic's EZ Chat🚀"
description = "You can use this Space to test out the current model (MistralMed) or duplicate this Space and use it for any other model on 🤗HuggingFace. Join me on [Discord](https://discord.gg/fpEPNZGsbt) to build together."
examples = [["What is the boiling point of nitrogen"]]

iface = gr.Interface(
    fn=bot.predict,
    title=title,
    description=description,
    examples=examples,
    inputs="text",
    outputs="text",
    theme="ParityError/Anime"
)

iface.launch()