File size: 6,445 Bytes
b56a22e 72c862f 953a4bb 72c862f 953a4bb b56a22e 72c862f f4091e9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 |
"""
cmd example
You need a file called "sample.txt" (default path) with text to take tokens for prompts or supply --text_file "path/to/text.txt" as an argument to a text file.
You can use our attached "sample.txt" file with one of Deci's blogs as a prompt.
# Run this and record tokens per second (652 tokens per second on A10 for DeciLM-6b)
python hf_benchmark_example.py --model Deci/DeciLM-6b
# Run this and record tokens per second (136 tokens per second on A10 for meta-llama/Llama-2-7b-hf), CUDA OOM above batch size 8
python hf_benchmark_example.py --model meta-llama/Llama-2-7b-hf --batch_size 8
"""
import json
import datasets
import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser
from argparse import ArgumentParser
def parse_args():
parser = ArgumentParser()
parser.add_argument(
"--model",
required=True,
help="Model to evaluate, provide a repo name in Hugging Face hub or a local path",
)
parser.add_argument(
"--temperature",
default=0.2,
type=float
)
parser.add_argument(
"--top_p",
default=0.95,
type=float
)
parser.add_argument(
"--top_k",
default=0,
type=float
)
parser.add_argument(
"--revision",
default=None,
help="Model revision to use",
)
parser.add_argument(
"--iterations",
type=int,
default=6,
help="Model revision to use",
)
parser.add_argument(
"--batch_size",
type=int,
default=64,
help="Batch size for evaluation on each worker, can be larger for HumanEval",
)
parser.add_argument(
"--prompt_length",
type=int,
default=512,
)
parser.add_argument(
"--max_new_tokens",
type=int,
default=512,
help="Maximum length of generated sequence (prompt+generation)",
)
parser.add_argument(
"--precision",
type=str,
default="bf16",
help="Model precision, from: fp32, fp16 or bf16",
)
parser.add_argument(
"--text_file",
type=str,
default="sample.txt",
help="text file that will be used to generate tokens for prompts",
)
parser.add_argument(
"--load_in_8bit",
action="store_true",
help="Load model in 8bit",
)
parser.add_argument(
"--load_in_4bit",
action="store_true",
help="Load model in 4bit",
)
return parser.parse_args()
def main():
args = parse_args()
transformers.logging.set_verbosity_error()
datasets.logging.set_verbosity_error()
results = {}
dict_precisions = {
"fp32": torch.float32,
"fp16": torch.float16,
"bf16": torch.bfloat16,
}
if args.precision not in dict_precisions:
raise ValueError(
f"Non valid precision {args.precision}, choose from: fp16, fp32, bf16"
)
if args.load_in_8bit:
print("Loading model in 8bit")
# the model needs to fit in one GPU
model = AutoModelForCausalLM.from_pretrained(
args.model,
revision=args.revision,
load_in_8bit=args.load_in_8bit,
trust_remote_code=args.trust_remote_code,
use_auth_token=args.use_auth_token,
device_map={"": 'cuda'},
)
elif args.load_in_4bit:
print("Loading model in 4bit")
# the model needs to fit in one GPU
model = AutoModelForCausalLM.from_pretrained(
args.model,
revision=args.revision,
load_in_4bit=args.load_in_4bit,
trust_remote_code=args.trust_remote_code,
use_auth_token=args.use_auth_token,
device_map={"": 'cuda'},
)
else:
print(f"Loading model in {args.precision}")
model = AutoModelForCausalLM.from_pretrained(
args.model,
torch_dtype=torch.bfloat16,
trust_remote_code=True,
use_auth_token=True
)
tokenizer = AutoTokenizer.from_pretrained(
args.model,
revision=args.revision,
trust_remote_code=True,
use_auth_token=True,
)
starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
model.cuda()
model.eval()
with open(args.text_file, "r") as f:
prompt = f.read()
prompt = torch.tensor(tokenizer.encode(prompt))[:args.prompt_length].cuda()
results = {'prefill': [], 'gen': [], 'max_new_tokens': args.max_new_tokens, 'prompt_length': args.prompt_length, 'model': args.model, 'batch_size': args.batch_size}
inputs = prompt.repeat(args.batch_size, 1)
#warmup
print('start warmup')
for _ in range(10):
with torch.no_grad():
_ = model.generate(
input_ids=inputs,
max_new_tokens=1,
do_sample=False,
)
print('finish warmup')
torch.cuda.synchronize()
for prefill_iter in range(args.iterations):
starter.record()
with torch.no_grad():
_ = model.generate(
input_ids=inputs,
max_new_tokens=1,
do_sample=False,
)
ender.record()
torch.cuda.synchronize()
t = starter.elapsed_time(ender) / 1000
results['prefill'].append(t)
print(f'{args.batch_size} prefill iter {prefill_iter} took: {t}')
for gen_iter in range(args.iterations):
starter.record()
with torch.no_grad():
_ = model.generate(
input_ids=inputs,
max_new_tokens=args.max_new_tokens,
do_sample=False,
)
ender.record()
torch.cuda.synchronize()
t = starter.elapsed_time(ender) / 1000
results['gen'].append(t)
print(f'{args.batch_size} total generation iter {gen_iter} took: {t}')
print(f'{args.batch_size * args.max_new_tokens / t} tokens per seconds')
model_str = args.model.split('/')[-1]
with open(f'timing_{model_str}_{args.batch_size}.json', 'w') as f:
json.dump(results, f)
if __name__ == "__main__":
main()
|