Usage

from unsloth import FastLanguageModel
import torch
import json


HUGGINGFACE_TOKEN = "your token here"

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name = "solaoi/gemma2-9b-magpie-sft-v1-v11",
    max_seq_length = 1024,
    dtype = None,
    load_in_4bit = True,
    token = HUGGINGFACE_TOKEN,
)
FastLanguageModel.for_inference(model)

system_prompt = "あなたは知識豊富な人工知能アシスタントです。ユーザーは日本語であなたに質問をし、あなたは自身の知識に基づいて日本語で誠実に回答します。"
prompt = tokenizer.apply_chat_template([
    {"role": "system", "content": system_prompt},
    {"role": "user", "content": input}
], tokenize=False, add_generation_prompt=True)

inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
outputs = model.generate(
    **inputs,
    max_new_tokens     = 304,
    use_cache          = True,
    temperature=0.7,
    top_k=50,
    top_p=0.85,
    repetition_penalty = 1.15,
    eos_token_id       = tokenizer.eos_token_id,
    pad_token_id       = tokenizer.pad_token_id
)

answer = tokenizer.decode(outputs[0], skip_special_tokens=True)[len(prompt):].strip()