Spaces:
Runtime error
Runtime error
"""Test various models.""" | |
# pylint: disable=invalid-name, line-too-long,broad-exception-caught, protected-access | |
import os | |
import time | |
from pathlib import Path | |
import gradio as gr | |
import pendulum | |
import torch | |
from loguru import logger | |
from transformers import AutoModel, AutoTokenizer | |
# ruff: noqa: E402 | |
# os.system("pip install --upgrade torch transformers sentencepiece scipy cpm_kernels accelerate bitsandbytes loguru") | |
# os.system("pip install torch transformers sentencepiece loguru") | |
# fix timezone in Linux | |
os.environ["TZ"] = "Asia/Shanghai" | |
try: | |
time.tzset() # type: ignore # pylint: disable=no-member | |
except Exception: | |
# Windows | |
logger.warning("Windows, cant run time.tzset()") | |
model_name = "THUDM/chatglm2-6b-int4" # 3.9G | |
tokenizer = AutoTokenizer.from_pretrained( | |
"THUDM/chatglm2-6b-int4", trust_remote_code=True | |
) | |
has_cuda = torch.cuda.is_available() | |
# has_cuda = False # force cpu | |
logger.debug("load") | |
if has_cuda: | |
if model_name.endswith("int4"): | |
model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda() | |
else: | |
model = ( | |
AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half() | |
) | |
else: | |
model = ( | |
AutoModel.from_pretrained(model_name, trust_remote_code=True).float() | |
) # .float() .half().float(): must use float for cpu | |
model = model.eval() | |
logger.debug("done load") | |
# tokenizer = AutoTokenizer.from_pretrained("openchat/openchat_v2_w") | |
# model = AutoModelForCausalLM.from_pretrained("openchat/openchat_v2_w", load_in_8bit_fp32_cpu_offload=True, load_in_8bit=True) | |
# locate model file cache | |
cache_loc = Path("~/.cache/huggingface/hub").expanduser() | |
model_cache_path = [ | |
elm | |
for elm in Path(cache_loc).rglob("*") | |
if Path(model_name).name in elm.as_posix() and "pytorch_model.bin" in elm.as_posix() | |
] | |
logger.debug(f"{model_cache_path=}") | |
if model_cache_path: | |
model_size_gb = model_cache_path[0].stat().st_size / 2**30 | |
logger.info(f"{model_name=} {model_size_gb=:.2f} GB") | |
def get_time(): | |
# return datetime.now().time() | |
return pendulum.now().format('HH:mm:ss zz') | |
def respond(message, chat_history): | |
"""Gen a response.""" | |
message = message.strip() | |
response, chat_history = model.chat( | |
tokenizer, | |
message, | |
history=chat_history, | |
temperature=0.7, | |
repetition_penalty=1.2, | |
max_length=128, | |
) | |
chat_history.append((message, response)) | |
return message, chat_history | |
theme = gr.themes.Soft(text_size="sm") | |
with gr.Blocks(theme=theme) as block: | |
chatbot = gr.Chatbot() | |
with gr.Row(): | |
with gr.Column(scale=12): | |
msg = gr.Textbox() | |
_ = """ | |
with gr.Column(scale=1, min_width=16): | |
btn = gr.Button("Send") | |
with gr.Column(scale=1, min_width=8): | |
clear = gr.ClearButton([msg, chatbot]) | |
with gr.Column(scale=1, min_width=25): | |
dt = gr.Textbox(label="Current time") | |
# """ | |
with gr.Column(scale=1, min_width=100): | |
with gr.Row(): | |
with gr.Column(scale=1, min_width=50): | |
btn = gr.Button("Send") | |
with gr.Column(scale=1, min_width=50): | |
clear = gr.ClearButton([msg, chatbot]) | |
with gr.Row(): | |
dt = gr.Textbox(label="Current time") | |
# do not clear prompt | |
msg.submit(respond, [msg, chatbot], [msg, chatbot]) | |
btn.click(lambda x, y: ("",) + respond(x, y)[1:], [msg, chatbot], [msg, chatbot]) | |
with gr.Accordion("Example inputs", open=True): | |
etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """ | |
examples = gr.Examples( | |
examples=[ | |
["Explain the plot of Cinderella in a sentence."], | |
[ | |
"How long does it take to become proficient in French, and what are the best methods for retaining information?" | |
], | |
["What are some common mistakes to avoid when writing code?"], | |
["Build a prompt to generate a beautiful portrait of a horse"], | |
["Suggest four metaphors to describe the benefits of AI"], | |
["Write a pop song about leaving home for the sandy beaches."], | |
["Write a summary demonstrating my ability to tame lions"], | |
["鲁迅和周树人什么关系"], | |
["从前有一头牛,这头牛后面有什么?"], | |
["正无穷大加一大于正无穷大吗?"], | |
["正无穷大加正无穷大大于正无穷大吗?"], | |
["-2的平方根等于什么"], | |
["树上有5只鸟,猎人开枪打死了一只。树上还有几只鸟?"], | |
["树上有11只鸟,猎人开枪打死了一只。树上还有几只鸟?提示:需考虑鸟可能受惊吓飞走。"], | |
["鲁迅和周树人什么关系 用英文回答"], | |
["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"], | |
[f"{etext} 翻成中文,列出3个版本"], | |
[f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本"], | |
["js 判断一个数是不是质数"], | |
["js 实现python 的 range(10)"], | |
["js 实现python 的 [*(range(10)]"], | |
["假定 1 + 2 = 4, 试求 7 + 8"], | |
["Erkläre die Handlung von Cinderella in einem Satz."], | |
["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"], | |
], | |
inputs=[msg], | |
examples_per_page=60, | |
) | |
block.load(get_time, inputs=[], outputs=dt, every=1) | |
block.queue().launch() | |