| generation_config = GenerationConfig( | |
| temperature=0.1, | |
| top_p=0.75, | |
| num_beams=4, | |
| ) | |
| def evaluate(instruction, input=None): | |
| prompt = generate_prompt(instruction, input) | |
| inputs = tokenizer(prompt, return_tensors="pt") | |
| input_ids = inputs["input_ids"].cuda() | |
| generation_output = model.generate( | |
| input_ids=input_ids, | |
| generation_config=generation_config, | |
| return_dict_in_generate=True, | |
| output_scores=True, | |
| max_new_tokens=256 | |
| ) | |
| for s in generation_output.sequences: | |
| output = tokenizer.decode(s) | |
| print("Response:", output.split("### Response:")[1].strip()) |