yonikremer
commited on
Commit
·
d3e85c8
1
Parent(s):
0a466f9
downloading only the pytorch model and important files, not the other versions of the model
Browse files- download_repo.py +18 -3
download_repo.py
CHANGED
@@ -15,8 +15,11 @@ def change_default_timeout(new_timeout: int) -> None:
|
|
15 |
urllib3.util.timeout.DEFAULT_TIMEOUT = new_timeout
|
16 |
|
17 |
|
18 |
-
def
|
19 |
-
"""
|
|
|
|
|
|
|
20 |
number_of_seconds_in_a_day: int = 86_400
|
21 |
change_default_timeout(number_of_seconds_in_a_day)
|
22 |
curr_folder: str = os.path.dirname(__file__)
|
@@ -26,5 +29,17 @@ def download_repository(name: str) -> None:
|
|
26 |
etag_timeout=number_of_seconds_in_a_day,
|
27 |
resume_download=True,
|
28 |
repo_type="model",
|
29 |
-
library_name="pt"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
)
|
|
|
|
|
|
|
|
|
|
15 |
urllib3.util.timeout.DEFAULT_TIMEOUT = new_timeout
|
16 |
|
17 |
|
18 |
+
def download_pytorch_model(name: str) -> None:
|
19 |
+
"""
|
20 |
+
Downloads a pytorch model and all the small files from the model's repository.
|
21 |
+
Other model formats (tensorflow, tflite, safetensors, msgpack and ot) are not downloaded.
|
22 |
+
"""
|
23 |
number_of_seconds_in_a_day: int = 86_400
|
24 |
change_default_timeout(number_of_seconds_in_a_day)
|
25 |
curr_folder: str = os.path.dirname(__file__)
|
|
|
29 |
etag_timeout=number_of_seconds_in_a_day,
|
30 |
resume_download=True,
|
31 |
repo_type="model",
|
32 |
+
library_name="pt",
|
33 |
+
# h5, tflite, safetensors, msgpack and ot models files are not needed
|
34 |
+
ignore_patterns=[
|
35 |
+
"*.h5",
|
36 |
+
"*.tflite",
|
37 |
+
"*.safetensors",
|
38 |
+
"*.msgpack",
|
39 |
+
"*.ot",
|
40 |
+
],
|
41 |
)
|
42 |
+
|
43 |
+
|
44 |
+
if __name__ == "__main__":
|
45 |
+
download_pytorch_model("facebook/opt-125m")
|