|
--- |
|
|
|
|
|
{} |
|
--- |
|
to use : code: |
|
``` |
|
PEFT_MODEL = "Bepitic/DM-falcon-7b-shared" |
|
config = PeftConfig.from_pretrained(PEFT_MODEL) |
|
model = AutoModelForCausalLM.from_pretrained( |
|
config.base_model_name_or_path, |
|
return_dict=True, |
|
quantization_config=bnb_config, |
|
device_map="auto", |
|
trust_remote_code=True |
|
) |
|
|
|
tokenizer=AutoTokenizer.from_pretrained(config.base_model_name_or_path) |
|
tokenizer.pad_token = tokenizer.eos_token |
|
|
|
model = PeftModel.from_pretrained(model, PEFT_MODEL) |
|
|
|
generation_config = model.generation_config |
|
generation_config.max_new_tokens = 200 |
|
generation_config.temperature = 0.7 |
|
generation_config.top_p = 0.7 |
|
generation_config.num_return_sequences = 1 |
|
generation_config.pad_token_id = tokenizer.eos_token_id |
|
generation_config.eos_token_id = tokenizer.eos_token_id |
|
|
|
# new cell colab |
|
%%time |
|
device = "cuda:0" |
|
|
|
prompt = """ |
|
<human>: Explain how Aelar Windrider a Human tried to Attack with a melee weapon of Extremely Easy difficulty and got an Catastrophic Failure. |
|
<assistant>: |
|
""".strip() |
|
|
|
encoding = tokenizer(prompt, return_tensors="pt").to(device) |
|
with torch.inference_mode(): |
|
outputs = model.generate( |
|
input_ids = encoding.input_ids, |
|
attention_mask = encoding.attention_mask, |
|
generation_config = generation_config |
|
) |
|
|
|
print(tokenizer.decode(outputs[0], skip_special_tokens=True)) |
|
``` |
|
|
|
|
|
|