Spaces:

OpenGVLab
/

InternGPT

Running

InternGPT / third-party /llama_download.sh

laizeqiang

update

1d63199 about 1 year ago

No virus

1.33 kB

	# Copyright (c) Meta Platforms, Inc. and affiliates.
	# This software may be used and distributed according to the terms of the GNU General Public License version 3.

	PRESIGNED_URL="" # replace with presigned url from email
	MODEL_SIZE="7B" # edit this list with the model sizes you wish to download
	TARGET_FOLDER="model_zoo/llama" # where all files should end up

	declare -A N_SHARD_DICT

	N_SHARD_DICT["7B"]="0"
	N_SHARD_DICT["13B"]="1"
	N_SHARD_DICT["30B"]="3"
	N_SHARD_DICT["65B"]="7"

	echo "Downloading tokenizer"
	wget ${PRESIGNED_URL/'*'/"tokenizer.model"} -O ${TARGET_FOLDER}"/tokenizer.model"
	wget ${PRESIGNED_URL/'*'/"tokenizer_checklist.chk"} -O ${TARGET_FOLDER}"/tokenizer_checklist.chk"

	(cd ${TARGET_FOLDER} && md5sum -c tokenizer_checklist.chk)

	for i in ${MODEL_SIZE//,/ }
	do
	echo "Downloading ${i}"
	mkdir -p ${TARGET_FOLDER}"/${i}"
	for s in $(seq -f "0%g" 0 ${N_SHARD_DICT[$i]})
	do
	wget ${PRESIGNED_URL/'*'/"${i}/consolidated.${s}.pth"} -O ${TARGET_FOLDER}"/${i}/consolidated.${s}.pth"
	done
	wget ${PRESIGNED_URL/'*'/"${i}/params.json"} -O ${TARGET_FOLDER}"/${i}/params.json"
	wget ${PRESIGNED_URL/'*'/"${i}/checklist.chk"} -O ${TARGET_FOLDER}"/${i}/checklist.chk"
	echo "Checking checksums"
	(cd ${TARGET_FOLDER}"/${i}" && md5sum -c checklist.chk)
	done