chansung commited on
Commit
b50dce8
1 Parent(s): fde2ede

Update gen.py

Browse files
Files changed (1) hide show
  1. gen.py +6 -0
gen.py CHANGED
@@ -20,20 +20,26 @@ def download_pretrained_models(
20
  ckpt_path: str,
21
  tokenizer_path: str
22
  ):
 
23
  os.makedirs(llama_weight_path)
24
  os.makedirs(tokenizer_weight_path)
25
 
 
26
  storage_client = storage.Client.create_anonymous_client()
27
  bucket = storage_client.bucket(bucket_name)
28
 
 
29
  blobs = bucket.list_blobs(prefix=f"{ckpt_path}/")
30
  for blob in blobs:
31
  filename = blob.name.split("/")[1]
 
32
  blob.download_to_filename(f"{llama_weight_path}/{filename}")
33
 
 
34
  blobs = bucket.list_blobs(prefix=f"{tokenizer_path}/")
35
  for blob in blobs:
36
  filename = blob.name.split("/")[1]
 
37
  blob.download_to_filename(f"{tokenizer_weight_path}/{filename}")
38
 
39
  def get_pretrained_models(
 
20
  ckpt_path: str,
21
  tokenizer_path: str
22
  ):
23
+ print("creating local directories...")
24
  os.makedirs(llama_weight_path)
25
  os.makedirs(tokenizer_weight_path)
26
 
27
+ print("initialize GCS client...")
28
  storage_client = storage.Client.create_anonymous_client()
29
  bucket = storage_client.bucket(bucket_name)
30
 
31
+ print(f"download {ckpt_path} model weights...")
32
  blobs = bucket.list_blobs(prefix=f"{ckpt_path}/")
33
  for blob in blobs:
34
  filename = blob.name.split("/")[1]
35
+ print(f"-{filename}")
36
  blob.download_to_filename(f"{llama_weight_path}/{filename}")
37
 
38
+ print(f"download {tokenizer_path} tokenizer weights...")
39
  blobs = bucket.list_blobs(prefix=f"{tokenizer_path}/")
40
  for blob in blobs:
41
  filename = blob.name.split("/")[1]
42
+ print(f"-{filename}")
43
  blob.download_to_filename(f"{tokenizer_weight_path}/{filename}")
44
 
45
  def get_pretrained_models(