Update gen.py
Browse files
gen.py
CHANGED
@@ -20,20 +20,26 @@ def download_pretrained_models(
|
|
20 |
ckpt_path: str,
|
21 |
tokenizer_path: str
|
22 |
):
|
|
|
23 |
os.makedirs(llama_weight_path)
|
24 |
os.makedirs(tokenizer_weight_path)
|
25 |
|
|
|
26 |
storage_client = storage.Client.create_anonymous_client()
|
27 |
bucket = storage_client.bucket(bucket_name)
|
28 |
|
|
|
29 |
blobs = bucket.list_blobs(prefix=f"{ckpt_path}/")
|
30 |
for blob in blobs:
|
31 |
filename = blob.name.split("/")[1]
|
|
|
32 |
blob.download_to_filename(f"{llama_weight_path}/{filename}")
|
33 |
|
|
|
34 |
blobs = bucket.list_blobs(prefix=f"{tokenizer_path}/")
|
35 |
for blob in blobs:
|
36 |
filename = blob.name.split("/")[1]
|
|
|
37 |
blob.download_to_filename(f"{tokenizer_weight_path}/{filename}")
|
38 |
|
39 |
def get_pretrained_models(
|
|
|
20 |
ckpt_path: str,
|
21 |
tokenizer_path: str
|
22 |
):
|
23 |
+
print("creating local directories...")
|
24 |
os.makedirs(llama_weight_path)
|
25 |
os.makedirs(tokenizer_weight_path)
|
26 |
|
27 |
+
print("initialize GCS client...")
|
28 |
storage_client = storage.Client.create_anonymous_client()
|
29 |
bucket = storage_client.bucket(bucket_name)
|
30 |
|
31 |
+
print(f"download {ckpt_path} model weights...")
|
32 |
blobs = bucket.list_blobs(prefix=f"{ckpt_path}/")
|
33 |
for blob in blobs:
|
34 |
filename = blob.name.split("/")[1]
|
35 |
+
print(f"-{filename}")
|
36 |
blob.download_to_filename(f"{llama_weight_path}/{filename}")
|
37 |
|
38 |
+
print(f"download {tokenizer_path} tokenizer weights...")
|
39 |
blobs = bucket.list_blobs(prefix=f"{tokenizer_path}/")
|
40 |
for blob in blobs:
|
41 |
filename = blob.name.split("/")[1]
|
42 |
+
print(f"-{filename}")
|
43 |
blob.download_to_filename(f"{tokenizer_weight_path}/{filename}")
|
44 |
|
45 |
def get_pretrained_models(
|