from transformers import AutoConfig, AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForCausalLM, MistralForCausalLM
from peft import PeftModel, PeftConfig
import torch
import gradio as gr
import random
from textwrap import wrap
# Functions to Wrap the Prompt Correctly
def wrap_text(text, width=90):
lines = text.split('\n')
wrapped_lines = [textwrap.fill(line, width=width) for line in lines]
wrapped_text = '\n'.join(wrapped_lines)
return wrapped_text
def multimodal_prompt(user_input, system_prompt="You are an expert medical analyst:"):
"""
Generates text using a large language model, given a user input and a system prompt.
Args:
user_input: The user's input text to generate a response for.
system_prompt: Optional system prompt.
Returns:
A string containing the generated text.
"""
# Combine user input and system prompt
formatted_input = f"[INST]{system_prompt} {user_input}[/INST]"
# Encode the input text
encodeds = tokenizer(formatted_input, return_tensors="pt", add_special_tokens=False)
model_inputs = encodeds.to(device)
# Generate a response using the model
output = model.generate(
**model_inputs,
max_length=max_length,
use_cache=True,
early_stopping=True,
bos_token_id=model.config.bos_token_id,
eos_token_id=model.config.eos_token_id,
pad_token_id=model.config.eos_token_id,
temperature=0.1,
do_sample=False
)
# Decode the response
response_text = tokenizer.decode(output[0], skip_special_tokens=True)
return response_text
# Define the device
device = "cuda" if torch.cuda.is_available() else "cpu"
# Use the base model's ID
base_model_id = "mistralai/Mistral-7B-v0.1"
model_directory = "Tonic/mistralmed"
# Instantiate the Tokenizer
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1", trust_remote_code=True, padding_side="left")
# tokenizer = AutoTokenizer.from_pretrained("Tonic/mistralmed", trust_remote_code=True, padding_side="left")
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = 'left'
# Specify the configuration class for the model
#model_config = AutoConfig.from_pretrained(base_model_id)
# Load the PEFT model with the specified configuration
#peft_model = AutoModelForCausalLM.from_pretrained(base_model_id, config=model_config)
# Load the PEFT model
peft_config = PeftConfig.from_pretrained("Tonic/mistralmed")
peft_model = MistralForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1", trust_remote_code=True)
peft_model = PeftModel.from_pretrained(peft_model, "Tonic/mistralmed")
class ChatBot:
def __init__(self):
self.history = []
class ChatBot:
def __init__(self):
# Initialize the ChatBot class with an empty history
self.history = []
def predict(self, user_input, system_prompt="You are an expert medical analyst:"):
# Combine the user's input with the system prompt
formatted_input = f"[INST]{system_prompt} {user_input}[/INST]"
# Encode the formatted input using the tokenizer
user_input_ids = tokenizer.encode(formatted_input, return_tensors="pt")
# Generate a response using the PEFT model
response = peft_model.generate(input_ids=user_input_ids, max_length=512, pad_token_id=tokenizer.eos_token_id)
# Decode the generated response to text
response_text = tokenizer.decode(response[0], skip_special_tokens=True)
return response_text # Return the generated response
bot = ChatBot()
title = "๐๐ปํ ๋์ ๋ฏธ์คํธ๋๋ฉ๋ ์ฑํ
์ ์ค์ ๊ฒ์ ํ์ํฉ๋๋ค๐๐๐ปWelcome to Tonic's MistralMed Chat๐"
description = "์ด ๊ณต๊ฐ์ ์ฌ์ฉํ์ฌ ํ์ฌ ๋ชจ๋ธ์ ํ
์คํธํ ์ ์์ต๋๋ค. [(Tonic/MistralMed)](https://huggingface.co/Tonic/MistralMed) ๋๋ ์ด ๊ณต๊ฐ์ ๋ณต์ ํ๊ณ ๋ก์ปฌ ๋๋ ๐คHuggingFace์์ ์ฌ์ฉํ ์ ์์ต๋๋ค. [Discord์์ ํจ๊ป ๋ง๋ค๊ธฐ ์ํด Discord์ ๊ฐ์
ํ์ญ์์ค](https://discord.gg/VqTxc76K3u). You can use this Space to test out the current model [(Tonic/MistralMed)](https://huggingface.co/Tonic/MistralMed) or duplicate this Space and use it locally or on ๐คHuggingFace. [Join me on Discord to build together](https://discord.gg/VqTxc76K3u)."
examples = [["[Question:] What is the proper treatment for buccal herpes?", "You are a medicine and public health expert, you will receive a question, answer the question, and provide a complete answer"]]
iface = gr.Interface(
fn=bot.predict,
title=title,
description=description,
examples=examples,
inputs=["text", "text"], # Take user input and system prompt separately
outputs="text",
theme="ParityError/Anime"
)
iface.launch()