Commit
•
d06ad3b
1
Parent(s):
6ab5895
GitHub API rate limit fix (#1894)
Browse files- utils/google_utils.py +47 -45
utils/google_utils.py
CHANGED
@@ -12,71 +12,73 @@ import torch
|
|
12 |
|
13 |
def gsutil_getsize(url=''):
|
14 |
# gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du
|
15 |
-
s = subprocess.check_output('gsutil du
|
16 |
return eval(s.split(' ')[0]) if len(s) else 0 # bytes
|
17 |
|
18 |
|
19 |
-
def attempt_download(
|
20 |
-
# Attempt
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
def gdrive_download(id='16TiPfZj7htmTyhntwcZyEEAejOUxuT6m',
|
51 |
# Downloads a file from Google Drive. from yolov5.utils.google_utils import *; gdrive_download()
|
52 |
t = time.time()
|
53 |
-
|
54 |
-
|
55 |
-
|
|
|
|
|
56 |
|
57 |
# Attempt file download
|
58 |
out = "NUL" if platform.system() == "Windows" else "/dev/null"
|
59 |
-
os.system('curl -c ./cookie -s -L "drive.google.com/uc?export=download&id
|
60 |
if os.path.exists('cookie'): # large file
|
61 |
-
s = 'curl -Lb ./cookie "drive.google.com/uc?export=download&confirm
|
62 |
else: # small file
|
63 |
-
s = 'curl -s -L -o
|
64 |
r = os.system(s) # execute, capture return
|
65 |
-
|
66 |
|
67 |
# Error check
|
68 |
if r != 0:
|
69 |
-
|
70 |
print('Download error ') # raise Exception('Download error')
|
71 |
return r
|
72 |
|
73 |
# Unzip if archive
|
74 |
-
if
|
75 |
print('unzipping... ', end='')
|
76 |
-
os.system('unzip -q
|
77 |
-
|
78 |
|
79 |
-
print('Done (
|
80 |
return r
|
81 |
|
82 |
|
|
|
12 |
|
13 |
def gsutil_getsize(url=''):
|
14 |
# gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du
|
15 |
+
s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8')
|
16 |
return eval(s.split(' ')[0]) if len(s) else 0 # bytes
|
17 |
|
18 |
|
19 |
+
def attempt_download(file):
|
20 |
+
# Attempt file download if does not exist
|
21 |
+
file = Path(str(file).strip().replace("'", '').lower())
|
22 |
+
|
23 |
+
if not file.exists():
|
24 |
+
response = requests.get('https://api.github.com/repos/ultralytics/yolov5/releases/latest').json() # github api
|
25 |
+
assets = [x['name'] for x in response['assets']] # release assets, i.e. ['yolov5s.pt', 'yolov5m.pt', ...]
|
26 |
+
name = file.name
|
27 |
+
|
28 |
+
if name in assets:
|
29 |
+
msg = f'{file} missing, try downloading from https://github.com/ultralytics/yolov5/releases/'
|
30 |
+
redundant = False # second download option
|
31 |
+
try: # GitHub
|
32 |
+
tag = response['tag_name'] # i.e. 'v1.0'
|
33 |
+
url = f'https://github.com/ultralytics/yolov5/releases/download/{tag}/{name}'
|
34 |
+
print(f'Downloading {url} to {file}...')
|
35 |
+
torch.hub.download_url_to_file(url, file)
|
36 |
+
assert file.exists() and file.stat().st_size > 1E6 # check
|
37 |
+
except Exception as e: # GCP
|
38 |
+
print(f'Download error: {e}')
|
39 |
+
assert redundant, 'No secondary mirror'
|
40 |
+
url = f'https://storage.googleapis.com/ultralytics/yolov5/ckpt/{name}'
|
41 |
+
print(f'Downloading {url} to {file}...')
|
42 |
+
os.system(f'curl -L {url} -o {file}') # torch.hub.download_url_to_file(url, weights)
|
43 |
+
finally:
|
44 |
+
if not file.exists() or file.stat().st_size < 1E6: # check
|
45 |
+
file.unlink(missing_ok=True) # remove partial downloads
|
46 |
+
print(f'ERROR: Download failure: {msg}')
|
47 |
+
return
|
48 |
+
|
49 |
+
|
50 |
+
def gdrive_download(id='16TiPfZj7htmTyhntwcZyEEAejOUxuT6m', file='tmp.zip'):
|
51 |
# Downloads a file from Google Drive. from yolov5.utils.google_utils import *; gdrive_download()
|
52 |
t = time.time()
|
53 |
+
file = Path(file)
|
54 |
+
cookie = Path('cookie') # gdrive cookie
|
55 |
+
print(f'Downloading https://drive.google.com/uc?export=download&id={id} as {file}... ', end='')
|
56 |
+
file.unlink(missing_ok=True) # remove existing file
|
57 |
+
cookie.unlink(missing_ok=True) # remove existing cookie
|
58 |
|
59 |
# Attempt file download
|
60 |
out = "NUL" if platform.system() == "Windows" else "/dev/null"
|
61 |
+
os.system(f'curl -c ./cookie -s -L "drive.google.com/uc?export=download&id={id}" > {out}')
|
62 |
if os.path.exists('cookie'): # large file
|
63 |
+
s = f'curl -Lb ./cookie "drive.google.com/uc?export=download&confirm={get_token()}&id={id}" -o {file}'
|
64 |
else: # small file
|
65 |
+
s = f'curl -s -L -o {file} "drive.google.com/uc?export=download&id={id}"'
|
66 |
r = os.system(s) # execute, capture return
|
67 |
+
cookie.unlink(missing_ok=True) # remove existing cookie
|
68 |
|
69 |
# Error check
|
70 |
if r != 0:
|
71 |
+
file.unlink(missing_ok=True) # remove partial
|
72 |
print('Download error ') # raise Exception('Download error')
|
73 |
return r
|
74 |
|
75 |
# Unzip if archive
|
76 |
+
if file.suffix == '.zip':
|
77 |
print('unzipping... ', end='')
|
78 |
+
os.system(f'unzip -q {file}') # unzip
|
79 |
+
file.unlink() # remove zip to free space
|
80 |
|
81 |
+
print(f'Done ({time.time() - t:.1f}s)')
|
82 |
return r
|
83 |
|
84 |
|