yossy0125 commited on
Commit
f746113
1 Parent(s): 2f113bf

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +50 -0
README.md CHANGED
@@ -75,6 +75,56 @@ Users (both direct and downstream) should be made aware of the risks, biases and
75
  ## How to Get Started with the Model
76
 
77
  Use the code below to get started with the model.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
 
79
  [More Information Needed]
80
 
 
75
  ## How to Get Started with the Model
76
 
77
  Use the code below to get started with the model.
78
+ from transformers import AutoModelForCausalLM,AutoTokenizer,BitsAndBytesConfig
79
+ from peft import PeftModel,PeftConfig
80
+ import torch
81
+
82
+ HF_TOKEN = "your token"
83
+ model_name = "llm-jp/llm-jp-3-13b"
84
+ adapter_name = "yossy0125/llm-jp-3-13b-it_lora/"
85
+
86
+ #QLoRaの量子化に合わせる
87
+ bnb_config = BitsAndBytesConfig(
88
+ load_in_4bit=True,
89
+ bnb_4bit_quant_type= "nf4",
90
+ bnb_4bit_compute_dtype=torch.bfloat16,
91
+
92
+ )
93
+
94
+ #BaseModel
95
+ model = AutoModelForCausalLM.from_pretrained(
96
+ model_name,
97
+ quantization_config = bnb_config,
98
+ device_map="auto",
99
+ token=HF_TOKEN
100
+ )
101
+ tokenizer = AutoTokenizer.from_pretrained(model_name,trust_remote_code=True,token=HF_TOKEN)
102
+
103
+ #adapterをBaseModelに統合
104
+ model = PeftModel.from_pretrained(model,adapter_name,token=HF_TOKEN)
105
+
106
+ input = "カレーの具材は何ですか?"
107
+ prompt = f"""以下はタスクを説明する指示です。
108
+ 要求を適切に満たす応答を出力しなさい。
109
+
110
+ ### 指示:{input}
111
+
112
+ ### 応答:
113
+ """
114
+
115
+ tokenized_input = tokenizer.encode(prompt,add_special_tokens=False,return_tensors="pt").to(model.device)
116
+ attention_mask = torch.ones_like(tokenized_input)
117
+ outputs = None
118
+ with torch.no_grad():
119
+ outputs = model.generate(
120
+ tokenized_input,
121
+ attention_mask=attention_mask,
122
+ max_new_tokens=2048, #生成するトークン数
123
+ do_sample=False,
124
+ repetition_penalty=1.2,
125
+ pad_token_id=tokenizer.eos_token_id
126
+ )[0]
127
+ output = tokenizer.decode(outputs[tokenized_input.size(1):],skip_special_tokens=True)
128
 
129
  [More Information Needed]
130