stormy-7b-10ep / app.py
masanorihirano's picture
update
b491e93
import datetime
import json
import os
import shutil
from typing import Optional
from typing import Tuple
from typing import Union
import gradio as gr
import requests
import torch
from fastchat.conversation import Conversation
from fastchat.conversation import SeparatorStyle
from fastchat.conversation import get_conv_template
from fastchat.conversation import register_conv_template
from fastchat.model.model_adapter import BaseAdapter
from fastchat.model.model_adapter import load_model
from fastchat.model.model_adapter import model_adapters
from fastchat.serve.cli import SimpleChatIO
from fastchat.serve.inference import generate_stream
from huggingface_hub import Repository
from huggingface_hub import snapshot_download
from peft import LoraConfig
from peft import PeftModel
from peft import get_peft_model
from peft import set_peft_model_state_dict
from transformers import AutoModelForCausalLM
from transformers import AutoTokenizer
from transformers import PreTrainedModel
from transformers import PreTrainedTokenizerBase
class FastTokenizerAvailableBaseAdapter(BaseAdapter):
def load_model(self, model_path: str, from_pretrained_kwargs: dict):
try:
tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
except ValueError:
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForCausalLM.from_pretrained(
model_path, low_cpu_mem_usage=True, **from_pretrained_kwargs
)
return model, tokenizer
model_adapters[-1] = FastTokenizerAvailableBaseAdapter()
def load_lora_model(
model_path: str,
lora_weight: str,
device: str,
num_gpus: int,
max_gpu_memory: Optional[str] = None,
load_8bit: bool = False,
cpu_offloading: bool = False,
debug: bool = False,
) -> Tuple[Union[PreTrainedModel, PeftModel], PreTrainedTokenizerBase]:
model: Union[PreTrainedModel, PeftModel]
tokenizer: PreTrainedTokenizerBase
model, tokenizer = load_model(
model_path=model_path,
device=device,
num_gpus=num_gpus,
max_gpu_memory=max_gpu_memory,
load_8bit=load_8bit,
cpu_offloading=cpu_offloading,
debug=debug,
)
if lora_weight is not None:
# model = PeftModelForCausalLM.from_pretrained(model, model_path, **kwargs)
config = LoraConfig.from_pretrained(lora_weight)
model = get_peft_model(model, config)
# Check the available weights and load them
checkpoint_name = os.path.join(
lora_weight, "pytorch_model.bin"
) # Full checkpoint
if not os.path.exists(checkpoint_name):
checkpoint_name = os.path.join(
lora_weight, "adapter_model.bin"
) # only LoRA model - LoRA config above has to fit
# The two files above have a different name depending on how they were saved,
# but are actually the same.
if os.path.exists(checkpoint_name):
adapters_weights = torch.load(checkpoint_name)
set_peft_model_state_dict(model, adapters_weights)
else:
raise IOError(f"Checkpoint {checkpoint_name} not found")
if debug:
print(model)
return model, tokenizer
print(datetime.datetime.now())
NUM_THREADS = 1
print(NUM_THREADS)
print("starting server ...")
BASE_MODEL = "cyberagent/open-calm-7b"
LORA_WEIGHTS_HF = "izumi-lab/stormy-7b-10ep"
HF_TOKEN = os.environ.get("HF_TOKEN", None)
DATASET_REPOSITORY = os.environ.get("DATASET_REPOSITORY", None)
SLACK_WEBHOOK = os.environ.get("SLACK_WEBHOOK", None)
LORA_WEIGHTS = snapshot_download(LORA_WEIGHTS_HF)
repo = None
LOCAL_DIR = "/home/user/data/"
if HF_TOKEN and DATASET_REPOSITORY:
try:
shutil.rmtree(LOCAL_DIR)
except Exception:
pass
repo = Repository(
local_dir=LOCAL_DIR,
clone_from=DATASET_REPOSITORY,
use_auth_token=HF_TOKEN,
repo_type="dataset",
)
repo.git_pull()
if torch.cuda.is_available():
device = "cuda"
else:
device = "cpu"
model, tokenizer = load_lora_model(
model_path=BASE_MODEL,
lora_weight=LORA_WEIGHTS,
device=device,
num_gpus=1,
max_gpu_memory="16GiB",
load_8bit=False,
cpu_offloading=False,
debug=False,
)
register_conv_template(
Conversation(
name="japanese",
system="ไปฅไธ‹ใฏใ‚ฟใ‚นใ‚ฏใ‚’่ชฌๆ˜Žใ™ใ‚‹ๆŒ‡็คบใงใ™ใ€‚่ฆๆฑ‚ใ‚’้ฉๅˆ‡ใซๆบ€ใŸใ™ใ‚ˆใ†ใช่ฟ”็ญ”ใ‚’ๆ›ธใ„ใฆใใ ใ•ใ„ใ€‚\n\n",
roles=("### ๆŒ‡็คบ", "### ่ฟ”็ญ”"),
messages=(),
offset=0,
sep_style=SeparatorStyle.ADD_COLON_SINGLE,
sep="\n###",
stop_str="###",
)
)
Conversation._get_prompt = Conversation.get_prompt
Conversation._append_message = Conversation.append_message
def conversation_append_message(cls, role: str, message: str):
cls.offset = -2
return cls._append_message(role, message)
def conversation_get_prompt_overrider(cls: Conversation) -> str:
cls.messages = cls.messages[-2:]
return cls._get_prompt()
def save_inputs_and_outputs(now, inputs, outputs, generate_kwargs):
current_hour = now.strftime("%Y-%m-%d_%H")
file_name = f"prompts_{LORA_WEIGHTS_HF.split('/')[-1]}_{current_hour}.jsonl"
if repo is not None:
repo.git_pull(rebase=True)
with open(os.path.join(LOCAL_DIR, file_name), "a", encoding="utf-8") as f:
json.dump(
{
"inputs": inputs,
"outputs": outputs,
"generate_kwargs": generate_kwargs,
},
f,
ensure_ascii=False,
)
f.write("\n")
repo.push_to_hub()
# we cant add typing now
# https://github.com/gradio-app/gradio/issues/3514
def evaluate(
instruction,
temperature=0.7,
max_tokens=256,
repetition_penalty=1.0,
):
try:
conv_template = "japanese"
inputs = tokenizer(instruction, return_tensors="pt")
if len(inputs["input_ids"][0]) > max_tokens - 40:
if HF_TOKEN and DATASET_REPOSITORY:
try:
now = datetime.datetime.now()
current_time = now.strftime("%Y-%m-%d %H:%M:%S")
print(f"[{current_time}] Pushing prompt and completion to the Hub")
save_inputs_and_outputs(
now,
instruction,
"",
{
"temperature": temperature,
"max_tokens": max_tokens,
"repetition_penalty": repetition_penalty,
},
)
except Exception as e:
print(e)
return (
f"please reduce the input length. Currently, {len(inputs['input_ids'][0])} ( > {max_tokens - 40}) tokens are used.",
gr.update(interactive=True),
gr.update(interactive=True),
)
conv = get_conv_template(conv_template)
conv.append_message(conv.roles[0], instruction)
conv.append_message(conv.roles[1], None)
generate_stream_func = generate_stream
prompt = conv.get_prompt()
gen_params = {
"model": BASE_MODEL,
"prompt": prompt,
"temperature": temperature,
"max_new_tokens": max_tokens - len(inputs["input_ids"][0]) - 30,
"stop": conv.stop_str,
"stop_token_ids": conv.stop_token_ids,
"echo": False,
"repetition_penalty": repetition_penalty,
}
chatio = SimpleChatIO()
chatio.prompt_for_output(conv.roles[1])
output_stream = generate_stream_func(model, tokenizer, gen_params, device)
output = chatio.stream_output(output_stream)
if HF_TOKEN and DATASET_REPOSITORY:
try:
now = datetime.datetime.now()
current_time = now.strftime("%Y-%m-%d %H:%M:%S")
print(f"[{current_time}] Pushing prompt and completion to the Hub")
save_inputs_and_outputs(
now,
prompt,
output,
{
"temperature": temperature,
"max_tokens": max_tokens,
"repetition_penalty": repetition_penalty,
},
)
except Exception as e:
print(e)
return output, gr.update(interactive=True), gr.update(interactive=True)
except Exception as e:
print(e)
import traceback
if SLACK_WEBHOOK:
payload_dic = {
"text": f"BASE_MODEL: {BASE_MODEL}\n LORA_WEIGHTS: {LORA_WEIGHTS_HF}\n"
+ f"instruction: {instruction}\ninput: {input}\ntemperature: {temperature}\n"
+ f"max_tokens: {max_tokens}\nrepetition_penalty: {repetition_penalty}\n\n"
+ str(traceback.format_exc()),
"username": "Hugging Face Space",
"channel": "#monitor",
}
try:
requests.post(SLACK_WEBHOOK, data=json.dumps(payload_dic))
except Exception:
pass
return (
"Error happend. Please return later.",
gr.update(interactive=True),
gr.update(interactive=True),
)
def reset_textbox():
return gr.update(value=""), gr.update(value=""), gr.update(value="")
def no_interactive() -> Tuple[gr.Request, gr.Request]:
return gr.update(interactive=False), gr.update(interactive=False)
title = """<h1 align="center">stormy 7B 10epochs</h1>"""
theme = gr.themes.Default(primary_hue="green")
description = (
"The official demo for **[izumi-lab/stormy-7b-10ep](https://huggingface.co/izumi-lab/stormy-7b-10ep)**. "
"It is a 7B-parameter CALM model finetuned to follow instructions. "
"It is trained on the dataset specially extracted from [izumi-lab/llm-japanese-dataset](https://huggingface.co/datasets/izumi-lab/llm-japanese-dataset) dataset. "
"For more information, please visit [the project's website](https://llm.msuzuki.me). "
"This model can output up to 256 tokens. "
"It takes about **1 minute** to output. When access is concentrated, the operation may become slow."
)
with gr.Blocks(
css="""#col_container { margin-left: auto; margin-right: auto;}""",
theme=theme,
) as demo:
gr.HTML(title)
gr.Markdown(description)
with gr.Column(elem_id="col_container", visible=False) as main_block:
with gr.Row():
with gr.Column():
instruction = gr.Textbox(
lines=3, label="Instruction", placeholder="ใ“ใ‚“ใซใกใฏ"
)
with gr.Row():
with gr.Column(scale=3):
clear_button = gr.Button("Clear").style(full_width=True)
with gr.Column(scale=5):
submit_button = gr.Button("Submit").style(full_width=True)
outputs = gr.Textbox(lines=4, label="Output")
# inputs, top_p, temperature, top_k, repetition_penalty
with gr.Accordion("Parameters", open=True):
temperature = gr.Slider(
minimum=0,
maximum=1.0,
value=0.0,
step=0.05,
interactive=True,
label="Temperature",
)
max_tokens = gr.Slider(
minimum=20,
maximum=256,
value=128,
step=1,
interactive=True,
label="Max length (Pre-prompt + instruction + input + output)",
)
repetition_penalty = gr.Slider(
minimum=0.0,
maximum=5.0,
value=1.0,
step=0.1,
interactive=True,
label="Repetition penalty",
)
with gr.Column(elem_id="user_consent_container") as user_consent_block:
# Get user consent
gr.Markdown(
"""
## User Consent for Data Collection, Use, and Sharing:
By using our app, you acknowledge and agree to the following terms regarding the data you provide:
- **Collection**: We may collect inputs you type into our app.
- **Use**: We may use the collected data for research purposes, to improve our services, and to develop new products or services, including commercial applications.
- **Sharing and Publication**: Your input data may be published, shared with third parties, or used for analysis and reporting purposes.
- **Data Retention**: We may retain your input data for as long as necessary.
By continuing to use our app, you provide your explicit consent to the collection, use, and potential sharing of your data as described above. If you do not agree with our data collection, use, and sharing practices, please do not use our app.
## ใƒ‡ใƒผใ‚ฟๅŽ้›†ใ€ๅˆฉ็”จใ€ๅ…ฑๆœ‰ใซ้–ขใ™ใ‚‹ใƒฆใƒผใ‚ถใƒผใฎๅŒๆ„๏ผš
ๆœฌใ‚ขใƒ—ใƒชใ‚’ไฝฟ็”จใ™ใ‚‹ใ“ใจใซใ‚ˆใ‚Šใ€ๆไพ›ใ™ใ‚‹ใƒ‡ใƒผใ‚ฟใซ้–ขใ™ใ‚‹ไปฅไธ‹ใฎๆกไปถใซๅŒๆ„ใ™ใ‚‹ใ‚‚ใฎใจใ—ใพใ™๏ผš
- **ๅŽ้›†**: ๆœฌใ‚ขใƒ—ใƒชใซๅ…ฅๅŠ›ใ•ใ‚Œใ‚‹ใƒ†ใ‚ญใ‚นใƒˆใƒ‡ใƒผใ‚ฟใฏๅŽ้›†ใ•ใ‚Œใ‚‹ๅ ดๅˆใŒใ‚ใ‚Šใพใ™ใ€‚
- **ๅˆฉ็”จ**: ๅŽ้›†ใ•ใ‚ŒใŸใƒ‡ใƒผใ‚ฟใฏ็ ”็ฉถใ‚„ใ€ๅ•†็”จใ‚ขใƒ—ใƒชใ‚ฑใƒผใ‚ทใƒงใƒณใ‚’ๅซใ‚€ใ‚ตใƒผใƒ“ใ‚นใฎ้–‹็™บใซไฝฟ็”จใ•ใ‚Œใ‚‹ๅ ดๅˆใŒใ‚ใ‚Šใพใ™ใ€‚
- **ๅ…ฑๆœ‰ใŠใ‚ˆใณๅ…ฌ้–‹**: ๅ…ฅๅŠ›ใƒ‡ใƒผใ‚ฟใฏ็ฌฌไธ‰่€…ใจๅ…ฑๆœ‰ใ•ใ‚ŒใŸใ‚Šใ€ๅˆ†ๆžใ‚„ๅ…ฌ้–‹ใฎ็›ฎ็š„ใงไฝฟ็”จใ•ใ‚Œใ‚‹ๅ ดๅˆใŒใ‚ใ‚Šใพใ™ใ€‚
- **ใƒ‡ใƒผใ‚ฟไฟๆŒ**: ๅ…ฅๅŠ›ใƒ‡ใƒผใ‚ฟใฏๅฟ…่ฆใช้™ใ‚ŠไฟๆŒใ•ใ‚Œใพใ™ใ€‚
ๆœฌใ‚ขใƒ—ใƒชใ‚’ๅผ•ใ็ถšใไฝฟ็”จใ™ใ‚‹ใ“ใจใซใ‚ˆใ‚Šใ€ไธŠ่จ˜ใฎใ‚ˆใ†ใซใƒ‡ใƒผใ‚ฟใฎๅŽ้›†ใƒปๅˆฉ็”จใƒปๅ…ฑๆœ‰ใซใคใ„ใฆๅŒๆ„ใ—ใพใ™ใ€‚ใƒ‡ใƒผใ‚ฟใฎๅˆฉ็”จๆ–นๆณ•ใซๅŒๆ„ใ—ใชใ„ๅ ดๅˆใฏใ€ๆœฌใ‚ขใƒ—ใƒชใ‚’ไฝฟ็”จใ—ใชใ„ใงใใ ใ•ใ„ใ€‚
"""
)
accept_button = gr.Button("I Agree")
def enable_inputs():
return user_consent_block.update(visible=False), main_block.update(
visible=True
)
accept_button.click(
fn=enable_inputs,
inputs=[],
outputs=[user_consent_block, main_block],
queue=False,
)
submit_button.click(no_interactive, [], [submit_button, clear_button])
submit_button.click(
evaluate,
[instruction, temperature, max_tokens, repetition_penalty],
[outputs, submit_button, clear_button],
)
clear_button.click(reset_textbox, [], [instruction, outputs], queue=False)
demo.queue(max_size=20, concurrency_count=NUM_THREADS, api_open=False).launch(
server_name="0.0.0.0", server_port=7860
)