metadata
base_model: google/gemma-2-9b
tags:
- text-generation-inference
- transformers
- unsloth
- gemma2
- trl
license: apache-2.0
language:
- ja
datasets:
- llm-jp/magpie-sft-v1.0
- Aratako/Magpie-Tanuki-8B-annotated-96k
Uploaded model
- Developed by: Chrom256
- License: apache-2.0
- Finetuned from model : google/gemma-2-9b
This gemma2 model was trained 2x faster with Unsloth and Huggingface's TRL library.
訓練用データ 以下のデータでInstruction finetuningを実施した
- https://huggingface.co/datasets/llm-jp/magpie-sft-v1.0 (Apache license 2.0)
- https://huggingface.co/datasets/Aratako/Magpie-Tanuki-8B-annotated-96k (Apache license 2.0) データをサンプリングして活用
実行コード Google Colab用 リンク先：編集中 ＊リンク先のGoogle Colabノートを実行してください
以下に同じノートを同じコードを掲載します
!pip install -q transformers==4.46.3 accelerate bitsandbytes
!pip install -q tqdm
!pip install flash-attn --no-build-isolation
import os
import torch
import json
from tqdm import tqdm
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from torch.cuda.amp import autocast
from concurrent.futures import ThreadPoolExecutor
import threading
print("【重要】以下の手順でHugging Faceトークンを設定しておいてください")
print("1. 左メニューの'シークレット'タブを開く")
print("2. '新しいシークレット'をクリック")
print("3. 名前に'HF_TOKEN'を入力")
print("4. 値にHugging Faceトークンを入力して保存")
print("ファイルタブ内にelyza-tasks-100-TV_0.jsonlを配置しておいてください")
print("出力物は、新規に作成されるOutputファイルの中に格納されます")
# シークレットからHF_TOKENを取得
from google.colab import userdata
HF_TOKEN = userdata.get('HF_TOKEN')
if HF_TOKEN is None:
raise ValueError("HF_TOKENが設定されていません。上記の手順でトークンを設定してください。")
quantization_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.bfloat16
)
def load_model_and_tokenizer():
model_id = "Chrom256/gemma-2-9b-it-lora_20241216_033631"
base_model_id = "google/gemma-2-9b"
downloaded_components = {"model": None, "tokenizer": None}
download_lock = threading.Lock()
def download_base_model():
quantization_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.bfloat16
)
model = AutoModelForCausalLM.from_pretrained(
base_model_id,
quantization_config=quantization_config,
device_map="auto",
trust_remote_code=True,
torch_dtype=torch.bfloat16,
attn_implementation="eager",
low_cpu_mem_usage=True,
token=HF_TOKEN
)
with download_lock:
downloaded_components["model"] = model
def download_tokenizer():
tokenizer = AutoTokenizer.from_pretrained(
model_id,
trust_remote_code=True,
token=HF_TOKEN
)
with download_lock:
downloaded_components["tokenizer"] = tokenizer
torch.cuda.empty_cache()
# ThreadPoolExecutorを使用して並列ダウンロード
with ThreadPoolExecutor(max_workers=2) as executor:
model_future = executor.submit(download_base_model)
tokenizer_future = executor.submit(download_tokenizer)
model_future.result()
tokenizer_future.result()
model = downloaded_components["model"]
tokenizer = downloaded_components["tokenizer"]
torch.cuda.empty_cache()
try:
adapter_path = model_id
print(f"Loading adapter from {adapter_path}")
model.load_adapter(adapter_path, "default", token=HF_TOKEN)
print("Adapter loaded successfully")
except Exception as e:
print(f"Error loading adapter: {e}")
raise
model.config.use_cache = True
model.eval()
torch.cuda.empty_cache()
return model, tokenizer
def run_inference(model, tokenizer, tokenized_inputs, generation_config, batch_size=4):
results = []
for i in tqdm(range(0, len(tokenized_inputs), batch_size)):
batch = tokenized_inputs[i:i+batch_size]
prompts = [
f"""<start_of_turn>system
簡潔に回答してください。装飾や特殊記号は使用しないでください。
<end_of_turn>
<start_of_turn>user
{item["input"]}
<end_of_turn>
<start_of_turn>model
""" for item in batch
]
inputs = tokenizer(
prompts,
padding=True,
truncation=True,
return_tensors="pt"
).to(model.device)
with torch.no_grad(), autocast(dtype=torch.bfloat16):
outputs = model.generate(
**inputs,
pad_token_id=tokenizer.pad_token_id,
eos_token_id=tokenizer.eos_token_id,
**generation_config
)
for idx, output in enumerate(outputs):
response = tokenizer.decode(output, skip_special_tokens=True)
if 'model\n' in response:
response = response.split('model\n')[-1].strip()
elif 'model' in response:
response = response.split('model')[-1].strip()
response = post_process_output(response)
results.append({
"task_id": batch[idx]["task_id"],
"input": batch[idx]["input"],
"output": response
})
del outputs, inputs
torch.cuda.empty_cache()
return results
def post_process_output(response):
response = response.strip()
symbols_to_replace = ['**', '`', '|', '```', '---', '===']
for symbol in symbols_to_replace:
response = response.replace(symbol, ' ')
return ' '.join(response.split())
GENERATION_CONFIG = {
"max_new_tokens": 512,
"use_cache": True,
"do_sample": False,
"num_beams": 4,
"repetition_penalty": 1.2,
"length_penalty": 1.0,
"early_stopping": False
}
def load_input_data(file_path):
tokenized_inputs = []
with open(file_path, "r") as f:
for line in f:
if line.strip():
dt = json.loads(line)
tokenized_inputs.append({
"task_id": dt["task_id"],
"input": dt["input"]
})
return tokenized_inputs
def save_results(results, output_dir):
os.makedirs(output_dir, exist_ok=True)
jsonl_path = os.path.join(output_dir, "Output.jsonl")
with open(jsonl_path, 'w', encoding='utf-8') as f:
for item in results:
json.dump(item, f, ensure_ascii=False)
f.write('\n')
print(f"Saved results to: {jsonl_path}")
def main():
model, tokenizer = load_model_and_tokenizer()
tokenized_inputs = load_input_data("/content/elyza-tasks-100-TV_0.jsonl")
results = run_inference(model, tokenizer, tokenized_inputs, GENERATION_CONFIG)
save_results(results, "output")
if __name__ == "__main__":
main()