Spaces:
Runtime error
Runtime error
| import os | |
| os.environ['CUDA_LAUNCH_BLOCKING'] = '1' | |
| import torch | |
| import gradio as gr | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| def init_model(): | |
| model = AutoModelForCausalLM.from_pretrained("Linly-AI/Chinese-LLaMA-2-7B-hf", device_map="cuda:0", | |
| torch_dtype=torch.bfloat16, trust_remote_code=True) | |
| tokenizer = AutoTokenizer.from_pretrained("Linly-AI/Chinese-LLaMA-2-7B-hf", use_fast=False, trust_remote_code=True) | |
| return model, tokenizer | |
| def process(message, history): | |
| input_prompt = "" | |
| for interaction in history: | |
| input_prompt = f"{input_prompt} User: {str(interaction[0]).strip(' ')} Bot: {str(interaction[1]).strip(' ')}" | |
| input_prompt = f"{input_prompt} ### Instruction:{message.strip()} ### Response:" | |
| inputs = tokenizer(input_prompt, return_tensors="pt").to("cuda:0") | |
| try: | |
| generate_ids = model.generate(inputs.input_ids, max_new_tokens=2048, do_sample=True, top_k=20, top_p=0.84, | |
| temperature=1, repetition_penalty=1.15, eos_token_id=2, bos_token_id=1, | |
| pad_token_id=0) | |
| response = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] | |
| print('log:', response) | |
| response = response.split("### Response:")[-1] | |
| return response | |
| except: | |
| return "Error: 会话超长，请重试！" | |
| if __name__ == '__main__': | |
| examples = ["Python和JavaScript编程语言的主要区别是什么？", "影响消费者行为的主要因素是什么？", "请用pytorch实现一个带ReLU激活函数的全连接层的代码", | |
| "请用C++编程语言实现“给你两个字符串haystack和needle，在haystack字符串中找出needle字符串的第一个匹配项的下标（下标从 0 开始）。如果needle不是haystack的一部分，则返回-1。", | |
| "如何使用ssh -L，请用具体例子说明", | |
| "应对压力最有效的方法是什么？"] | |
| model, tokenizer = init_model() | |
| demo = gr.ChatInterface( | |
| process, | |
| chatbot=gr.Chatbot(height=600), | |
| textbox=gr.Textbox(placeholder="Input", container=False, scale=7), | |
| title="Linly ChatFlow", | |
| description="", | |
| theme="soft", | |
| examples=examples, | |
| cache_examples=True, | |
| retry_btn="Retry", | |
| undo_btn="Delete Previous", | |
| clear_btn="Clear", | |
| ) | |
| demo.queue(concurrency_count=75).launch() |