---
library_name: peft
---
## Training procedure


The following `bitsandbytes` quantization config was used during training:
- quant_method: bitsandbytes
- load_in_8bit: False
- load_in_4bit: True
- llm_int8_threshold: 6.0
- llm_int8_skip_modules: None
- llm_int8_enable_fp32_cpu_offload: False
- llm_int8_has_fp16_weight: False
- bnb_4bit_quant_type: nf4
- bnb_4bit_use_double_quant: False
- bnb_4bit_compute_dtype: float16
### Framework versions


- PEFT 0.5.0

## Get it started
```python
import torch
from datasets import Dataset
from huggingface_hub import login
from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer, AddedToken

# load model and tokenizer
login("[YOUR HF TOKEN HERE FOR USING LLAMA]")
config = PeftConfig.from_pretrained("ChangeIsKey/llama-7b-lexical-substitution")
base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf", device_map='auto')

tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf", use_fast=False, trust_remote_code=True)
tokenizer.add_special_tokens({ "additional_special_tokens":[AddedToken("<|s|>"), AddedToken("<|answer|>"), AddedToken("<|end|>")]})
if tokenizer.pad_token is None:
    tokenizer.add_special_tokens({'pad_token': '[PAD]'})
tokenizer.padding_side = 'left'
base_model.resize_token_embeddings(len(tokenizer))

model = PeftModel.from_pretrained(base_model, "ChangeIsKey/llama-7b-lexical-substitution")
model.eval()

# let's use this model
def formatting_func(records):
    text_batch = []
    
    for i in range(len(records['example'])):
        example = records[i]['example']
        start, end = records[i]['start'], records[i]['end']
        
        target = f'**{example[start:end]}**'
        input_text = f'{example[:start]} {target} {example[end:]}'
        text_batch.append(f"{input_text}<|answer|>")
    
    return text_batch

def tokenization(dataset):
    return tokenizer(formatting_func(dataset),
                       truncation=True,
                       max_length=512,
                       padding=True,
                       return_tensors="pt").to("cuda")


# a toy example
examples = [{'example': 'The traffic jam on the highway made everyone late for work.', 'start': 12, 'end': 15},
            {'example': 'I spread a generous layer of strawberry jam on my toast this morning', 'start': 40, 'end': 43}]
dataset = Dataset.from_list(examples)


batch_size = 32
output = list()

with torch.no_grad():
    for i in range(0, len(dataset), batch_size):
        model_input = tokenization(dataset.select(range(i, min(dataset.num_rows, i + batch_size))))

        output_ids = model.generate(**model_input,
                                    do_sample=True,
                                    num_return_sequences=1, 
                                    max_new_tokens=30,
                                    temperature=0.00001, 
                                    repetition_penalty=1/0.85,
                                    top_k=40,
                                    top_p=0.1)

        answers = tokenizer.batch_decode(output_ids, skip_special_tokens=False)

        for answer in answers:
            answer = " ".join(answer.split('<|answer|>')[1:])
            substitutes = [s.strip() for s in answer.split('<|end|>')[:-1] if s.strip() != ""]
            output.append(", ".join(substitutes))

# output
dataset = dataset.add_column('substitutes', output)
for row in dataset:
    target = row['example'][row['start']:row['end']]
    print(f"Target: {target}\nExample: {row['example']}\nSubstitutes: {row['substitutes']}\n")
```