Spaces:
Runtime error
Runtime error
File size: 2,119 Bytes
e72aedf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 |
"""Test command line interface for model inference."""
import argparse
import os
from fastchat.utils import run_cmd
def test_single_gpu():
models = [
"lmsys/vicuna-7b-v1.3",
"lmsys/longchat-7b-16k",
"lmsys/fastchat-t5-3b-v1.0",
"THUDM/chatglm-6b",
"THUDM/chatglm2-6b",
"mosaicml/mpt-7b-chat",
"project-baize/baize-v2-7b",
"h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b",
"tiiuae/falcon-7b-instruct",
"~/model_weights/RWKV-4-Raven-7B-v11x-Eng99%-Other1%-20230429-ctx8192.pth",
]
for model_path in models:
if "model_weights" in model_path and not os.path.exists(
os.path.expanduser(model_path)
):
continue
cmd = (
f"python3 -m fastchat.serve.cli --model-path {model_path} "
f"--style programmatic < test_cli_inputs.txt"
)
ret = run_cmd(cmd)
if ret != 0:
return
print("")
def test_multi_gpu():
models = [
"lmsys/vicuna-13b-v1.3",
]
for model_path in models:
cmd = (
f"python3 -m fastchat.serve.cli --model-path {model_path} "
f"--style programmatic --num-gpus 2 < test_cli_inputs.txt"
)
ret = run_cmd(cmd)
if ret != 0:
return
print("")
def test_8bit():
models = [
"lmsys/vicuna-13b-v1.3",
]
for model_path in models:
cmd = (
f"python3 -m fastchat.serve.cli --model-path {model_path} "
f"--style programmatic --load-8bit < test_cli_inputs.txt"
)
ret = run_cmd(cmd)
if ret != 0:
return
print("")
def test_hf_api():
models = [
"lmsys/vicuna-7b-v1.3",
"lmsys/fastchat-t5-3b-v1.0",
]
for model_path in models:
cmd = f"python3 -m fastchat.serve.huggingface_api --model-path {model_path}"
ret = run_cmd(cmd)
if ret != 0:
return
print("")
if __name__ == "__main__":
test_single_gpu()
test_multi_gpu()
test_8bit()
test_hf_api()
|