Update README.md
Browse files
README.md
CHANGED
@@ -75,6 +75,56 @@ Users (both direct and downstream) should be made aware of the risks, biases and
|
|
75 |
## How to Get Started with the Model
|
76 |
|
77 |
Use the code below to get started with the model.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
78 |
|
79 |
[More Information Needed]
|
80 |
|
|
|
75 |
## How to Get Started with the Model
|
76 |
|
77 |
Use the code below to get started with the model.
|
78 |
+
from transformers import AutoModelForCausalLM,AutoTokenizer,BitsAndBytesConfig
|
79 |
+
from peft import PeftModel,PeftConfig
|
80 |
+
import torch
|
81 |
+
|
82 |
+
HF_TOKEN = "your token"
|
83 |
+
model_name = "llm-jp/llm-jp-3-13b"
|
84 |
+
adapter_name = "yossy0125/llm-jp-3-13b-it_lora/"
|
85 |
+
|
86 |
+
#QLoRaの量子化に合わせる
|
87 |
+
bnb_config = BitsAndBytesConfig(
|
88 |
+
load_in_4bit=True,
|
89 |
+
bnb_4bit_quant_type= "nf4",
|
90 |
+
bnb_4bit_compute_dtype=torch.bfloat16,
|
91 |
+
|
92 |
+
)
|
93 |
+
|
94 |
+
#BaseModel
|
95 |
+
model = AutoModelForCausalLM.from_pretrained(
|
96 |
+
model_name,
|
97 |
+
quantization_config = bnb_config,
|
98 |
+
device_map="auto",
|
99 |
+
token=HF_TOKEN
|
100 |
+
)
|
101 |
+
tokenizer = AutoTokenizer.from_pretrained(model_name,trust_remote_code=True,token=HF_TOKEN)
|
102 |
+
|
103 |
+
#adapterをBaseModelに統合
|
104 |
+
model = PeftModel.from_pretrained(model,adapter_name,token=HF_TOKEN)
|
105 |
+
|
106 |
+
input = "カレーの具材は何ですか?"
|
107 |
+
prompt = f"""以下はタスクを説明する指示です。
|
108 |
+
要求を適切に満たす応答を出力しなさい。
|
109 |
+
|
110 |
+
### 指示:{input}
|
111 |
+
|
112 |
+
### 応答:
|
113 |
+
"""
|
114 |
+
|
115 |
+
tokenized_input = tokenizer.encode(prompt,add_special_tokens=False,return_tensors="pt").to(model.device)
|
116 |
+
attention_mask = torch.ones_like(tokenized_input)
|
117 |
+
outputs = None
|
118 |
+
with torch.no_grad():
|
119 |
+
outputs = model.generate(
|
120 |
+
tokenized_input,
|
121 |
+
attention_mask=attention_mask,
|
122 |
+
max_new_tokens=2048, #生成するトークン数
|
123 |
+
do_sample=False,
|
124 |
+
repetition_penalty=1.2,
|
125 |
+
pad_token_id=tokenizer.eos_token_id
|
126 |
+
)[0]
|
127 |
+
output = tokenizer.decode(outputs[tokenized_input.size(1):],skip_special_tokens=True)
|
128 |
|
129 |
[More Information Needed]
|
130 |
|