import argparse import json import os import pprint import json5 import jsonlines from rouge_score import rouge_scorer from tqdm import tqdm from transformers import Agent, AutoModelForCausalLM, AutoTokenizer from transformers.generation import GenerationConfig from transformers.tools.evaluate_agent import evaluate_agent from transformers.trainer_utils import set_seed data_root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') def is_callable(response, golden): return response['action'].strip().lower() == golden['action'].strip( ).lower() def process_res(response): # parse response response += '\n' # fix not-find bug thought = response[:response.find('Action:')].strip() action = response[response.find('Action:') + len('Action:'):response.find('Action Input:')].strip() action_input = response[response.find('Action Input:') + len('Action Input:'):response.find('Observation:' )].strip() #TODO: This parsing result is incorrect if the response contains multiple Actions. To be fixed in the future. observation = response[response.find('Observation:') + len('Observation:'):response.rfind('Thought:' )].strip() thought_last = response[response.rfind('Thought:') + len('Thought:'):response.find('Final Answer:' )].strip() final_answer = response[response.find('Final Answer:') + len('Final Answer:'):].strip() try: action_input = json.dumps(json5.loads(action_input), ensure_ascii=False, sort_keys=True) except: # print("JSON Load Error:", action_input) pass res_dict = { 'thought': thought, 'action': action, 'action_input': action_input, 'observation': observation, 'thought_last': thought_last, 'final_answer': final_answer } return res_dict class _DummyTokenizer: def tokenize(self, text: str): return text.split() def _get_tokenized_string(tokenizer, text_list): token_ids_list, tokenized_string_list = [], [] for text in text_list: assert tokenizer is not None token_ids = tokenizer.encode(text) tokens_bytes = tokenizer.convert_ids_to_tokens(token_ids) tokens = [ token.decode('utf-8', errors='replace') for token in tokens_bytes ] tokenized_string = ' '.join(tokens) token_ids_list.append(token_ids) tokenized_string_list.append(tokenized_string) return token_ids_list, tokenized_string_list def eval_action(job): response = job['gen'][0] golden = job['response'] if 'Action:' in response: response, golden = process_res(response), process_res(golden) if is_callable(response, golden): return True return False def eval_action_input(job, tokenizer): response = job['gen'][0] golden = job['response'] response, golden = process_res(response), process_res(golden) query = job['prompt'] job = {} job['prompt'] = query job['gen'] = response['action_input'] job['response'] = golden['action_input'] job['_gen_tok'], job['_gen_tok_str'] = _get_tokenized_string( tokenizer, [response['action_input']]) job['_reference_tok'], job['_reference_tok_str'] = _get_tokenized_string( tokenizer, [golden['action_input']]) scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], tokenizer=_DummyTokenizer()) score = scorer.score(job['_reference_tok_str'][0], job['_gen_tok_str'][0]) rouge = score['rougeL'].fmeasure return rouge class QWenAgent(Agent): """ Agent that uses QWen model and tokenizer to generate code. Example: ```py agent = QWenAgent() agent.run("Draw me a picture of rivers and lakes.") ``` """ def __init__(self, chat_prompt_template=None, run_prompt_template=None, additional_tools=None, tokenizer=None, model=None): if tokenizer and model: self.tokenizer = tokenizer self.model = model else: checkpoint = 'Qwen/Qwen-7B-Chat' self.tokenizer = AutoTokenizer.from_pretrained( checkpoint, trust_remote_code=True) self.model = AutoModelForCausalLM.from_pretrained( checkpoint, device_map='auto', trust_remote_code=True).cuda().eval() self.model.generation_config = GenerationConfig.from_pretrained( checkpoint, trust_remote_code=True) # 可指定不同的生成长度、top_p等相关超参 self.model.generation_config.do_sample = False # greedy super().__init__( chat_prompt_template=chat_prompt_template, run_prompt_template=run_prompt_template, additional_tools=additional_tools, ) def generate_one(self, prompt, stop): # "Human:" 和 "Assistant:" 曾为通义千问的特殊保留字，需要替换为 "_HUMAN_:" 和 "_ASSISTANT_:"。这一问题将在未来版本修复。 prompt = prompt.replace('Human:', '_HUMAN_:').replace('Assistant:', '_ASSISTANT_:') stop = [ item.replace('Human:', '_HUMAN_:').replace('Assistant:', '_ASSISTANT_:') for item in stop ] result, _ = self.model.chat(self.tokenizer, prompt, history=None) for stop_seq in stop: if result.endswith(stop_seq): result = result[:-len(stop_seq)] result = result.replace('_HUMAN_:', 'Human:').replace('_ASSISTANT_:', 'Assistant:') return result def load_models_tokenizer(args): tokenizer = AutoTokenizer.from_pretrained(args.checkpoint_path, trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained(args.checkpoint_path, device_map='auto', trust_remote_code=True, bf16=True, use_flash_attn=True).eval() model.generation_config = GenerationConfig.from_pretrained( args.checkpoint_path, trust_remote_code=True) model.generation_config.do_sample = False # use greedy decoding return model, tokenizer def load_jobs(filename): jobs = [] with jsonlines.open(os.path.join(data_root_path, filename), mode='r') as reader: for job in reader: jobs.append(job) return jobs def react_inference(filename, model, tokenizer): filename_cache = filename + '.cache' if os.path.exists(os.path.join(data_root_path, filename_cache)): jobs = load_jobs(filename=filename_cache) print('Loaded from', filename_cache) else: with open(os.path.join(data_root_path, filename_cache), 'w') as f: jobs = load_jobs(filename=filename) print('Inference:', filename) for job in tqdm(jobs): response, history = model.chat(tokenizer, job['prompt'], history=None) job['gen'] = [response] f.writelines(json.dumps(job, ensure_ascii=False) + '\n') print(filename_cache, 'is saved.') return jobs def main(args): print('loading model weights') if args.checkpoint_path is not None: model, tokenizer = load_models_tokenizer(args) else: model, tokenizer = None, None print('model loaded') result = {} # eval react positive if args.eval_react_positive: print('eval react positive ...') acc_count = 0 rouge_mean = 0 jobs = react_inference(filename=args.eval_react_positive_filename, model=model, tokenizer=tokenizer) for job in jobs: if eval_action(job): acc_count += 1 rouge = eval_action_input(job, tokenizer) rouge_mean += (rouge / len(jobs)) scores = { 'action_right_rate': acc_count / len(jobs), 'action_input_rouge': rouge_mean, } result.update({'react_positive': scores}) # eval react negative if args.eval_react_negative: print('eval react negative ...') bad_count = 0 jobs = react_inference(filename=args.eval_react_negative_filename, model=model, tokenizer=tokenizer) for job in jobs: if '\nAction:' in job['gen'][0]: bad_count += 1 scores = {'bad_rate': bad_count / len(jobs)} result.update({'react_negative': scores}) # eval hfagent if args.eval_hfagent: print('eval hfagent ...') agent = QWenAgent(model=model, tokenizer=tokenizer) scores = evaluate_agent(agent, verbose=False, return_errors=False) result.update({'hfagent': scores}) pp = pprint.PrettyPrinter(indent=4) pp.pprint(result) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Test HF checkpoint.') parser.add_argument('-c', '--checkpoint-path', type=str, help='Checkpoint path', default='Qwen/Qwen-7B-Chat') parser.add_argument('-s', '--seed', type=int, default=1234, help='Random seed') """Provide extra arguments required for tasks.""" group = parser.add_argument_group(title='Evaluation options') group.add_argument('--eval-react-positive', action='store_true', default=False, help='Eval react positive.') group.add_argument('--eval-react-positive-filename', type=str, default='exam_plugin_v1_react_positive.jsonl', help='Eval react positive filename.') group.add_argument('--eval-react-negative', action='store_true', default=False, help='Eval react negative.') group.add_argument('--eval-react-negative-filename', type=str, default='exam_plugin_v1_react_negative.jsonl', help='Eval react negative filename.') group.add_argument('--eval-hfagent', action='store_true', default=False, help='Eval hfagent.') args = parser.parse_args() set_seed(args.seed) main(args)