Aspik101/polish-alpaca7B-lora

This repo contains a low-rank adapter for LLaMA-7b fit on the Stanford Alpaca dataset translated to Polish language.

How to use (8-bit)

import torch
from peft import PeftModel
from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig

tokenizer = LLaMATokenizer.from_pretrained("decapoda-research/llama-13b-hf")

model = LLaMAForCausalLM.from_pretrained(
    "decapoda-research/llama-7b-hf",
    load_in_8bit=True,
    device_map="auto",
)


model = PeftModel.from_pretrained(model, "Aspik101/polish-alpaca7B-lora")


def get_answer(question, model_version =  model):
    PROMPT =f'''Poniżej znajduje się instrukcja opisująca zadanie. Napisz odpowiedź, która odpowiednio rozwiąrze zadanie.

    ### Instruction:
    {question}

    ### Response:
    '''

    inputs = tokenizer(
        PROMPT,
        return_tensors="pt",
    )
    input_ids = inputs["input_ids"].cuda()

    generation_config = GenerationConfig(
        temperature=0.2,
        top_p=0.95,
        repetition_penalty=1.15,
    )
    print("Generating...")
    generation_output = model_version.generate(
        input_ids=input_ids,
        generation_config=generation_config,
        return_dict_in_generate=True,
        output_scores=True,
        max_new_tokens=128,
    )

    sentences = " ".join([tokenizer.decode(s) for s in generation_output.sequences])
    print(sentences.split("Response:\n")[1])