Change layout
history blame contribute delete
No virus
5.92 kB
"""Test various models."""
# pylint: disable=invalid-name, line-too-long,broad-exception-caught, protected-access
import os
import time
from pathlib import Path
import gradio as gr
import pendulum
import torch
from loguru import logger
from transformers import AutoModel, AutoTokenizer
# ruff: noqa: E402
# os.system("pip install --upgrade torch transformers sentencepiece scipy cpm_kernels accelerate bitsandbytes loguru")
# os.system("pip install torch transformers sentencepiece loguru")
# fix timezone in Linux
os.environ["TZ"] = "Asia/Shanghai"
time.tzset() # type: ignore # pylint: disable=no-member
except Exception:
# Windows
logger.warning("Windows, cant run time.tzset()")
model_name = "THUDM/chatglm2-6b-int4" # 3.9G
tokenizer = AutoTokenizer.from_pretrained(
"THUDM/chatglm2-6b-int4", trust_remote_code=True
has_cuda = torch.cuda.is_available()
# has_cuda = False # force cpu
if has_cuda:
if model_name.endswith("int4"):
model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda()
model = (
AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half()
model = (
AutoModel.from_pretrained(model_name, trust_remote_code=True).float()
) # .float() .half().float(): must use float for cpu
model = model.eval()
logger.debug("done load")
# tokenizer = AutoTokenizer.from_pretrained("openchat/openchat_v2_w")
# model = AutoModelForCausalLM.from_pretrained("openchat/openchat_v2_w", load_in_8bit_fp32_cpu_offload=True, load_in_8bit=True)
# locate model file cache
cache_loc = Path("~/.cache/huggingface/hub").expanduser()
model_cache_path = [
for elm in Path(cache_loc).rglob("*")
if Path(model_name).name in elm.as_posix() and "pytorch_model.bin" in elm.as_posix()
if model_cache_path:
model_size_gb = model_cache_path[0].stat().st_size / 2**30
logger.info(f"{model_name=} {model_size_gb=:.2f} GB")
def get_time():
# return datetime.now().time()
return pendulum.now().format('HH:mm:ss zz')
def respond(message, chat_history):
"""Gen a response."""
message = message.strip()
response, chat_history = model.chat(
chat_history.append((message, response))
return message, chat_history
theme = gr.themes.Soft(text_size="sm")
with gr.Blocks(theme=theme) as block:
chatbot = gr.Chatbot()
with gr.Column():
with gr.Column(scale=12):
msg = gr.Textbox()
_ = """
with gr.Column(scale=1, min_width=16):
btn = gr.Button("Send")
with gr.Column(scale=1, min_width=8):
clear = gr.ClearButton([msg, chatbot])
with gr.Column(scale=1, min_width=25):
dt = gr.Textbox(label="Current time")
# """
with gr.Column(scale=1, min_width=100):
with gr.Column():
with gr.Column(scale=1, min_width=50):
btn = gr.Button("Send")
with gr.Column(scale=1, min_width=50):
clear = gr.ClearButton([msg, chatbot])
# with gr.Row():
dt = gr.Textbox(label="Current time")
# do not clear prompt
msg.submit(respond, [msg, chatbot], [msg, chatbot])
btn.click(lambda x, y: ("",) + respond(x, y)[1:], [msg, chatbot], [msg, chatbot])
with gr.Accordion("Example inputs", open=True):
etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
examples = gr.Examples(
["Explain the plot of Cinderella in a sentence."],
"How long does it take to become proficient in French, and what are the best methods for retaining information?"
["What are some common mistakes to avoid when writing code?"],
["Build a prompt to generate a beautiful portrait of a horse"],
["Suggest four metaphors to describe the benefits of AI"],
["Write a pop song about leaving home for the sandy beaches."],
["Write a summary demonstrating my ability to tame lions"],
["鲁迅和周树人什么关系 用英文回答"],
[f"{etext} 翻成中文,列出3个版本"],
[f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本"],
["js 判断一个数是不是质数"],
["js 实现python 的 range(10)"],
["js 实现python 的 [*(range(10)]"],
["假定 1 + 2 = 4, 试求 7 + 8"],
["Erkläre die Handlung von Cinderella in einem Satz."],
["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"],
block.load(get_time, inputs=[], outputs=dt, every=1)