update downloading code 2
Browse files
files_cells/notebooks/en/downloading_en.ipynb
CHANGED
@@ -29,7 +29,6 @@
|
|
29 |
"import subprocess\n",
|
30 |
"from datetime import timedelta\n",
|
31 |
"from subprocess import getoutput\n",
|
32 |
-
"from urllib.parse import unquote\n",
|
33 |
"from IPython.utils import capture\n",
|
34 |
"from IPython.display import clear_output\n",
|
35 |
"\n",
|
@@ -339,7 +338,7 @@
|
|
339 |
"}\n",
|
340 |
"\n",
|
341 |
"extension_repo = []\n",
|
342 |
-
"directories =
|
343 |
"!mkdir -p {\" \".join(directories)}\n",
|
344 |
"\n",
|
345 |
"hf_token = optional_huggingface_token if optional_huggingface_token else \"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\"\n",
|
@@ -348,7 +347,7 @@
|
|
348 |
"''' Get Image Preview | CivitAi '''\n",
|
349 |
"\n",
|
350 |
"def get_data_from_api(model_id, api_key): # get model data\n",
|
351 |
-
" base_url = f\"https://civitai.com/api/v1/model-versions/{model_id}
|
352 |
" headers = {\"Content-Type\": \"application/json\"}\n",
|
353 |
" try:\n",
|
354 |
" response = requests.get(base_url, headers=headers)\n",
|
@@ -358,10 +357,18 @@
|
|
358 |
" print(f\"An error occurred: {e}\")\n",
|
359 |
" return None\n",
|
360 |
"\n",
|
361 |
-
"def extract_model_info(data):\n",
|
362 |
-
"
|
363 |
-
"
|
364 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
365 |
"\n",
|
366 |
"def gen_preview_filename(model_name, image_url):\n",
|
367 |
" name = model_name.split('.')\n",
|
@@ -386,34 +393,37 @@
|
|
386 |
" extension_repo.append((url, file_name))\n",
|
387 |
"\n",
|
388 |
"def manual_download(url, dst_dir, file_name):\n",
|
389 |
-
" header_option = f\"--header={user_header}\"\n",
|
390 |
-
" basename = url.split(\"/\")[-1] if file_name is None else file_name\n",
|
391 |
" aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'\n",
|
|
|
|
|
392 |
"\n",
|
393 |
" # === CivitAi API ===\n",
|
|
|
394 |
" civitai_token = \"62c0c5956b2f9defbd844d754000180b\"\n",
|
|
|
|
|
|
|
395 |
" if 'civitai' in url:\n",
|
396 |
-
" clean_url = url # virgin url without token\n",
|
397 |
-
" image_file_name = None # for PreviewDl\n",
|
398 |
-
" model_id = url.split('/')[-1]\n",
|
399 |
" url = f\"{url}{'&' if '?' in url else '?'}token={civitai_token}\" # some authors are pussies and they need API token...\n",
|
|
|
|
|
400 |
"\n",
|
401 |
" data = get_data_from_api(model_id, civitai_token)\n",
|
402 |
" if data:\n",
|
403 |
-
" model_name, image_url = extract_model_info(data)\n",
|
404 |
" if model_name and image_url:\n",
|
405 |
" image_file_name = gen_preview_filename(model_name if not file_name else file_name, image_url)\n",
|
406 |
-
" if
|
407 |
" with capture.capture_output() as cap: # clear shit\n",
|
408 |
" !aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}\n",
|
409 |
" del cap\n",
|
410 |
-
"
|
411 |
"\n",
|
412 |
" \"\"\" information output \"\"\"\n",
|
413 |
-
" print(f\"\\n\\033[32m{'---'*45}\\n\\033[33mURL: \\033[34m{clean_url if 'civitai' in url else url}\\n\\033[33mSAVE DIR: \\033[34m{dst_dir}\\n\\033[33mFILE NAME: \\033[34m{file_name if not 'huggingface' in url else basename}\\033[0m\") # wrold's best print info\n",
|
414 |
" # print(url, dst_dir, file_name if not 'huggingface' in url else basename) # for my tests\n",
|
415 |
-
" print(f\"\\033[32m[
|
416 |
-
" print(\"\\033[
|
|
|
417 |
" # ===================\n",
|
418 |
"\n",
|
419 |
" # -- GDrive --\n",
|
@@ -553,8 +563,8 @@
|
|
553 |
" pass\n",
|
554 |
"\n",
|
555 |
"# url prefixing\n",
|
556 |
-
"urls =
|
557 |
-
"prefixed_urls =
|
558 |
"url += \", \".join(prefixed_urls) + \", \" + file_urls\n",
|
559 |
"\n",
|
560 |
"if detailed_download == \"on\":\n",
|
|
|
29 |
"import subprocess\n",
|
30 |
"from datetime import timedelta\n",
|
31 |
"from subprocess import getoutput\n",
|
|
|
32 |
"from IPython.utils import capture\n",
|
33 |
"from IPython.display import clear_output\n",
|
34 |
"\n",
|
|
|
338 |
"}\n",
|
339 |
"\n",
|
340 |
"extension_repo = []\n",
|
341 |
+
"directories = (value for key, value in prefixes.items()) # for unpucking zip files\n",
|
342 |
"!mkdir -p {\" \".join(directories)}\n",
|
343 |
"\n",
|
344 |
"hf_token = optional_huggingface_token if optional_huggingface_token else \"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\"\n",
|
|
|
347 |
"''' Get Image Preview | CivitAi '''\n",
|
348 |
"\n",
|
349 |
"def get_data_from_api(model_id, api_key): # get model data\n",
|
350 |
+
" base_url = f\"https://civitai.com/api/v1/model-versions/{model_id}\" # model_id - contains not only ID but also params\n",
|
351 |
" headers = {\"Content-Type\": \"application/json\"}\n",
|
352 |
" try:\n",
|
353 |
" response = requests.get(base_url, headers=headers)\n",
|
|
|
357 |
" print(f\"An error occurred: {e}\")\n",
|
358 |
" return None\n",
|
359 |
"\n",
|
360 |
+
"def extract_model_info(data, url):\n",
|
361 |
+
" # Get 'Type, Name and imgURL' current model\n",
|
362 |
+
" if 'type=' in url:\n",
|
363 |
+
" model_type = data['files'][1]['type']\n",
|
364 |
+
" model_name = data['files'][1]['name']\n",
|
365 |
+
" image_url = data['images'][0]['url']\n",
|
366 |
+
" else:\n",
|
367 |
+
" model_type = data['model']['type']\n",
|
368 |
+
" model_name = data['files'][0]['name']\n",
|
369 |
+
" image_url = data['images'][0]['url']\n",
|
370 |
+
"\n",
|
371 |
+
" return model_type, model_name, image_url\n",
|
372 |
"\n",
|
373 |
"def gen_preview_filename(model_name, image_url):\n",
|
374 |
" name = model_name.split('.')\n",
|
|
|
393 |
" extension_repo.append((url, file_name))\n",
|
394 |
"\n",
|
395 |
"def manual_download(url, dst_dir, file_name):\n",
|
|
|
|
|
396 |
" aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'\n",
|
397 |
+
" basename = url.split(\"/\")[-1] if file_name is None else file_name\n",
|
398 |
+
" header_option = f\"--header={user_header}\"\n",
|
399 |
"\n",
|
400 |
" # === CivitAi API ===\n",
|
401 |
+
" supported_types = ('Checkpoint', 'TextualInversion', 'LORA') # for dl preview image\n",
|
402 |
" civitai_token = \"62c0c5956b2f9defbd844d754000180b\"\n",
|
403 |
+
" image_file_name = None\n",
|
404 |
+
" model_type = None\n",
|
405 |
+
"\n",
|
406 |
" if 'civitai' in url:\n",
|
|
|
|
|
|
|
407 |
" url = f\"{url}{'&' if '?' in url else '?'}token={civitai_token}\" # some authors are pussies and they need API token...\n",
|
408 |
+
" model_id = url.split('/')[-1]\n",
|
409 |
+
" clean_url = url.split('?')[-1]\n",
|
410 |
"\n",
|
411 |
" data = get_data_from_api(model_id, civitai_token)\n",
|
412 |
" if data:\n",
|
413 |
+
" model_type, model_name, image_url = extract_model_info(data, url)\n",
|
414 |
" if model_name and image_url:\n",
|
415 |
" image_file_name = gen_preview_filename(model_name if not file_name else file_name, image_url)\n",
|
416 |
+
" if model_type in supported_types:\n",
|
417 |
" with capture.capture_output() as cap: # clear shit\n",
|
418 |
" !aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}\n",
|
419 |
" del cap\n",
|
420 |
+
" file_name = file_name or model_name # assigns the original file name if not specified initially\n",
|
421 |
"\n",
|
422 |
" \"\"\" information output \"\"\"\n",
|
|
|
423 |
" # print(url, dst_dir, file_name if not 'huggingface' in url else basename) # for my tests\n",
|
424 |
+
" print(f\"\\n\\033[32m{'---'*45}\\n\\033[33mURL: \\033[34m{clean_url if 'civitai' in url else url}\\n\\033[33mSAVE DIR: \\033[34m{dst_dir}\\n\\033[33mFILE NAME: \\033[34m{file_name if not 'huggingface' in url else basename}\\033[0m\") # wrold's best print info\n",
|
425 |
+
" print(f\"\\033[32m[Preview DL]\\033[0m: {image_file_name}\\n\") if 'civitai' in url and model_type in supported_types else None # PreviewDl: print file name\n",
|
426 |
+
" print(\"\\033[31m[Data Info]\\033[0m: Failed to retrieve data from the API.\\n\") if 'civitai' in url and not data else None # DATA: ERROR\n",
|
427 |
" # ===================\n",
|
428 |
"\n",
|
429 |
" # -- GDrive --\n",
|
|
|
563 |
" pass\n",
|
564 |
"\n",
|
565 |
"# url prefixing\n",
|
566 |
+
"urls = (Model_url, Vae_url, LoRA_url, Embedding_url, Extensions_url)\n",
|
567 |
+
"prefixed_urls = (f\"{prefix}:{url}\" for prefix, url in zip(prefixes.keys(), urls) if url for url in url.replace(',', '').split())\n",
|
568 |
"url += \", \".join(prefixed_urls) + \", \" + file_urls\n",
|
569 |
"\n",
|
570 |
"if detailed_download == \"on\":\n",
|
files_cells/notebooks/ru/downloading_ru.ipynb
CHANGED
@@ -29,7 +29,6 @@
|
|
29 |
"import subprocess\n",
|
30 |
"from datetime import timedelta\n",
|
31 |
"from subprocess import getoutput\n",
|
32 |
-
"from urllib.parse import unquote\n",
|
33 |
"from IPython.utils import capture\n",
|
34 |
"from IPython.display import clear_output\n",
|
35 |
"\n",
|
@@ -339,7 +338,7 @@
|
|
339 |
"}\n",
|
340 |
"\n",
|
341 |
"extension_repo = []\n",
|
342 |
-
"directories =
|
343 |
"!mkdir -p {\" \".join(directories)}\n",
|
344 |
"\n",
|
345 |
"hf_token = optional_huggingface_token if optional_huggingface_token else \"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\"\n",
|
@@ -348,7 +347,7 @@
|
|
348 |
"''' Get Image Preview | CivitAi '''\n",
|
349 |
"\n",
|
350 |
"def get_data_from_api(model_id, api_key): # get model data\n",
|
351 |
-
" base_url = f\"https://civitai.com/api/v1/model-versions/{model_id}
|
352 |
" headers = {\"Content-Type\": \"application/json\"}\n",
|
353 |
" try:\n",
|
354 |
" response = requests.get(base_url, headers=headers)\n",
|
@@ -358,10 +357,18 @@
|
|
358 |
" print(f\"An error occurred: {e}\")\n",
|
359 |
" return None\n",
|
360 |
"\n",
|
361 |
-
"def extract_model_info(data):\n",
|
362 |
-
"
|
363 |
-
"
|
364 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
365 |
"\n",
|
366 |
"def gen_preview_filename(model_name, image_url):\n",
|
367 |
" name = model_name.split('.')\n",
|
@@ -386,34 +393,37 @@
|
|
386 |
" extension_repo.append((url, file_name))\n",
|
387 |
"\n",
|
388 |
"def manual_download(url, dst_dir, file_name):\n",
|
389 |
-
" header_option = f\"--header={user_header}\"\n",
|
390 |
-
" basename = url.split(\"/\")[-1] if file_name is None else file_name\n",
|
391 |
" aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'\n",
|
|
|
|
|
392 |
"\n",
|
393 |
" # === CivitAi API ===\n",
|
|
|
394 |
" civitai_token = \"62c0c5956b2f9defbd844d754000180b\"\n",
|
|
|
|
|
|
|
395 |
" if 'civitai' in url:\n",
|
396 |
-
" clean_url = url # virgin url without token\n",
|
397 |
-
" image_file_name = None # for PreviewDl\n",
|
398 |
-
" model_id = url.split('/')[-1]\n",
|
399 |
" url = f\"{url}{'&' if '?' in url else '?'}token={civitai_token}\" # some authors are pussies and they need API token...\n",
|
|
|
|
|
400 |
"\n",
|
401 |
" data = get_data_from_api(model_id, civitai_token)\n",
|
402 |
" if data:\n",
|
403 |
-
" model_name, image_url = extract_model_info(data)\n",
|
404 |
" if model_name and image_url:\n",
|
405 |
" image_file_name = gen_preview_filename(model_name if not file_name else file_name, image_url)\n",
|
406 |
-
" if
|
407 |
" with capture.capture_output() as cap: # clear shit\n",
|
408 |
" !aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}\n",
|
409 |
" del cap\n",
|
410 |
-
"
|
411 |
"\n",
|
412 |
" \"\"\" information output \"\"\"\n",
|
413 |
-
" print(f\"\\n\\033[32m{'---'*45}\\n\\033[33mURL: \\033[34m{clean_url if 'civitai' in url else url}\\n\\033[33mSAVE DIR: \\033[34m{dst_dir}\\n\\033[33mFILE NAME: \\033[34m{file_name if not 'huggingface' in url else basename}\\033[0m\") # wrold's best print info\n",
|
414 |
" # print(url, dst_dir, file_name if not 'huggingface' in url else basename) # for my tests\n",
|
415 |
-
" print(f\"\\033[32m[
|
416 |
-
" print(\"\\033[
|
|
|
417 |
" # ===================\n",
|
418 |
"\n",
|
419 |
" # -- GDrive --\n",
|
@@ -553,8 +563,8 @@
|
|
553 |
" pass\n",
|
554 |
"\n",
|
555 |
"# url prefixing\n",
|
556 |
-
"urls =
|
557 |
-
"prefixed_urls =
|
558 |
"url += \", \".join(prefixed_urls) + \", \" + file_urls\n",
|
559 |
"\n",
|
560 |
"if detailed_download == \"on\":\n",
|
|
|
29 |
"import subprocess\n",
|
30 |
"from datetime import timedelta\n",
|
31 |
"from subprocess import getoutput\n",
|
|
|
32 |
"from IPython.utils import capture\n",
|
33 |
"from IPython.display import clear_output\n",
|
34 |
"\n",
|
|
|
338 |
"}\n",
|
339 |
"\n",
|
340 |
"extension_repo = []\n",
|
341 |
+
"directories = (value for key, value in prefixes.items()) # for unpucking zip files\n",
|
342 |
"!mkdir -p {\" \".join(directories)}\n",
|
343 |
"\n",
|
344 |
"hf_token = optional_huggingface_token if optional_huggingface_token else \"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\"\n",
|
|
|
347 |
"''' Get Image Preview | CivitAi '''\n",
|
348 |
"\n",
|
349 |
"def get_data_from_api(model_id, api_key): # get model data\n",
|
350 |
+
" base_url = f\"https://civitai.com/api/v1/model-versions/{model_id}\" # model_id - contains not only ID but also params\n",
|
351 |
" headers = {\"Content-Type\": \"application/json\"}\n",
|
352 |
" try:\n",
|
353 |
" response = requests.get(base_url, headers=headers)\n",
|
|
|
357 |
" print(f\"An error occurred: {e}\")\n",
|
358 |
" return None\n",
|
359 |
"\n",
|
360 |
+
"def extract_model_info(data, url):\n",
|
361 |
+
" # Get 'Type, Name and imgURL' current model\n",
|
362 |
+
" if 'type=' in url:\n",
|
363 |
+
" model_type = data['files'][1]['type']\n",
|
364 |
+
" model_name = data['files'][1]['name']\n",
|
365 |
+
" image_url = data['images'][0]['url']\n",
|
366 |
+
" else:\n",
|
367 |
+
" model_type = data['model']['type']\n",
|
368 |
+
" model_name = data['files'][0]['name']\n",
|
369 |
+
" image_url = data['images'][0]['url']\n",
|
370 |
+
"\n",
|
371 |
+
" return model_type, model_name, image_url\n",
|
372 |
"\n",
|
373 |
"def gen_preview_filename(model_name, image_url):\n",
|
374 |
" name = model_name.split('.')\n",
|
|
|
393 |
" extension_repo.append((url, file_name))\n",
|
394 |
"\n",
|
395 |
"def manual_download(url, dst_dir, file_name):\n",
|
|
|
|
|
396 |
" aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'\n",
|
397 |
+
" basename = url.split(\"/\")[-1] if file_name is None else file_name\n",
|
398 |
+
" header_option = f\"--header={user_header}\"\n",
|
399 |
"\n",
|
400 |
" # === CivitAi API ===\n",
|
401 |
+
" supported_types = ('Checkpoint', 'TextualInversion', 'LORA') # for dl preview image\n",
|
402 |
" civitai_token = \"62c0c5956b2f9defbd844d754000180b\"\n",
|
403 |
+
" image_file_name = None\n",
|
404 |
+
" model_type = None\n",
|
405 |
+
"\n",
|
406 |
" if 'civitai' in url:\n",
|
|
|
|
|
|
|
407 |
" url = f\"{url}{'&' if '?' in url else '?'}token={civitai_token}\" # some authors are pussies and they need API token...\n",
|
408 |
+
" model_id = url.split('/')[-1]\n",
|
409 |
+
" clean_url = url.split('?')[-1]\n",
|
410 |
"\n",
|
411 |
" data = get_data_from_api(model_id, civitai_token)\n",
|
412 |
" if data:\n",
|
413 |
+
" model_type, model_name, image_url = extract_model_info(data, url)\n",
|
414 |
" if model_name and image_url:\n",
|
415 |
" image_file_name = gen_preview_filename(model_name if not file_name else file_name, image_url)\n",
|
416 |
+
" if model_type in supported_types:\n",
|
417 |
" with capture.capture_output() as cap: # clear shit\n",
|
418 |
" !aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}\n",
|
419 |
" del cap\n",
|
420 |
+
" file_name = file_name or model_name # assigns the original file name if not specified initially\n",
|
421 |
"\n",
|
422 |
" \"\"\" information output \"\"\"\n",
|
|
|
423 |
" # print(url, dst_dir, file_name if not 'huggingface' in url else basename) # for my tests\n",
|
424 |
+
" print(f\"\\n\\033[32m{'---'*45}\\n\\033[33mURL: \\033[34m{clean_url if 'civitai' in url else url}\\n\\033[33mSAVE DIR: \\033[34m{dst_dir}\\n\\033[33mFILE NAME: \\033[34m{file_name if not 'huggingface' in url else basename}\\033[0m\") # wrold's best print info\n",
|
425 |
+
" print(f\"\\033[32m[Preview DL]\\033[0m: {image_file_name}\\n\") if 'civitai' in url and model_type in supported_types else None # PreviewDl: print file name\n",
|
426 |
+
" print(\"\\033[31m[Data Info]\\033[0m: Failed to retrieve data from the API.\\n\") if 'civitai' in url and not data else None # DATA: ERROR\n",
|
427 |
" # ===================\n",
|
428 |
"\n",
|
429 |
" # -- GDrive --\n",
|
|
|
563 |
" pass\n",
|
564 |
"\n",
|
565 |
"# url prefixing\n",
|
566 |
+
"urls = (Model_url, Vae_url, LoRA_url, Embedding_url, Extensions_url)\n",
|
567 |
+
"prefixed_urls = (f\"{prefix}:{url}\" for prefix, url in zip(prefixes.keys(), urls) if url for url in url.replace(',', '').split())\n",
|
568 |
"url += \", \".join(prefixed_urls) + \", \" + file_urls\n",
|
569 |
"\n",
|
570 |
"if detailed_download == \"on\":\n",
|
files_cells/python/en/downloading_en.py
CHANGED
@@ -10,7 +10,6 @@ import requests
|
|
10 |
import subprocess
|
11 |
from datetime import timedelta
|
12 |
from subprocess import getoutput
|
13 |
-
from urllib.parse import unquote
|
14 |
from IPython.utils import capture
|
15 |
from IPython.display import clear_output
|
16 |
|
@@ -320,7 +319,7 @@ prefixes = {
|
|
320 |
}
|
321 |
|
322 |
extension_repo = []
|
323 |
-
directories =
|
324 |
get_ipython().system('mkdir -p {" ".join(directories)}')
|
325 |
|
326 |
hf_token = optional_huggingface_token if optional_huggingface_token else "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"
|
@@ -329,7 +328,7 @@ user_header = f"\"Authorization: Bearer {hf_token}\""
|
|
329 |
''' Get Image Preview | CivitAi '''
|
330 |
|
331 |
def get_data_from_api(model_id, api_key): # get model data
|
332 |
-
base_url = f"https://civitai.com/api/v1/model-versions/{model_id}
|
333 |
headers = {"Content-Type": "application/json"}
|
334 |
try:
|
335 |
response = requests.get(base_url, headers=headers)
|
@@ -339,10 +338,18 @@ def get_data_from_api(model_id, api_key): # get model data
|
|
339 |
print(f"An error occurred: {e}")
|
340 |
return None
|
341 |
|
342 |
-
def extract_model_info(data):
|
343 |
-
|
344 |
-
|
345 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
346 |
|
347 |
def gen_preview_filename(model_name, image_url):
|
348 |
name = model_name.split('.')
|
@@ -367,34 +374,37 @@ def handle_manual(url):
|
|
367 |
extension_repo.append((url, file_name))
|
368 |
|
369 |
def manual_download(url, dst_dir, file_name):
|
370 |
-
header_option = f"--header={user_header}"
|
371 |
-
basename = url.split("/")[-1] if file_name is None else file_name
|
372 |
aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'
|
|
|
|
|
373 |
|
374 |
# === CivitAi API ===
|
|
|
375 |
civitai_token = "62c0c5956b2f9defbd844d754000180b"
|
|
|
|
|
|
|
376 |
if 'civitai' in url:
|
377 |
-
clean_url = url # virgin url without token
|
378 |
-
image_file_name = None # for PreviewDl
|
379 |
-
model_id = url.split('/')[-1]
|
380 |
url = f"{url}{'&' if '?' in url else '?'}token={civitai_token}" # some authors are pussies and they need API token...
|
|
|
|
|
381 |
|
382 |
data = get_data_from_api(model_id, civitai_token)
|
383 |
if data:
|
384 |
-
model_name, image_url = extract_model_info(data)
|
385 |
if model_name and image_url:
|
386 |
image_file_name = gen_preview_filename(model_name if not file_name else file_name, image_url)
|
387 |
-
if
|
388 |
with capture.capture_output() as cap: # clear shit
|
389 |
get_ipython().system('aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}')
|
390 |
del cap
|
391 |
-
|
392 |
|
393 |
""" information output """
|
394 |
-
print(f"\n\033[32m{'---'*45}\n\033[33mURL: \033[34m{clean_url if 'civitai' in url else url}\n\033[33mSAVE DIR: \033[34m{dst_dir}\n\033[33mFILE NAME: \033[34m{file_name if not 'huggingface' in url else basename}\033[0m") # wrold's best print info
|
395 |
# print(url, dst_dir, file_name if not 'huggingface' in url else basename) # for my tests
|
396 |
-
print(f"\033[32m[
|
397 |
-
print("\033[
|
|
|
398 |
# ===================
|
399 |
|
400 |
# -- GDrive --
|
@@ -534,8 +544,8 @@ if custom_file_urls:
|
|
534 |
pass
|
535 |
|
536 |
# url prefixing
|
537 |
-
urls =
|
538 |
-
prefixed_urls =
|
539 |
url += ", ".join(prefixed_urls) + ", " + file_urls
|
540 |
|
541 |
if detailed_download == "on":
|
|
|
10 |
import subprocess
|
11 |
from datetime import timedelta
|
12 |
from subprocess import getoutput
|
|
|
13 |
from IPython.utils import capture
|
14 |
from IPython.display import clear_output
|
15 |
|
|
|
319 |
}
|
320 |
|
321 |
extension_repo = []
|
322 |
+
directories = (value for key, value in prefixes.items()) # for unpucking zip files
|
323 |
get_ipython().system('mkdir -p {" ".join(directories)}')
|
324 |
|
325 |
hf_token = optional_huggingface_token if optional_huggingface_token else "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"
|
|
|
328 |
''' Get Image Preview | CivitAi '''
|
329 |
|
330 |
def get_data_from_api(model_id, api_key): # get model data
|
331 |
+
base_url = f"https://civitai.com/api/v1/model-versions/{model_id}" # model_id - contains not only ID but also params
|
332 |
headers = {"Content-Type": "application/json"}
|
333 |
try:
|
334 |
response = requests.get(base_url, headers=headers)
|
|
|
338 |
print(f"An error occurred: {e}")
|
339 |
return None
|
340 |
|
341 |
+
def extract_model_info(data, url):
|
342 |
+
# Get 'Type, Name and imgURL' current model
|
343 |
+
if 'type=' in url:
|
344 |
+
model_type = data['files'][1]['type']
|
345 |
+
model_name = data['files'][1]['name']
|
346 |
+
image_url = data['images'][0]['url']
|
347 |
+
else:
|
348 |
+
model_type = data['model']['type']
|
349 |
+
model_name = data['files'][0]['name']
|
350 |
+
image_url = data['images'][0]['url']
|
351 |
+
|
352 |
+
return model_type, model_name, image_url
|
353 |
|
354 |
def gen_preview_filename(model_name, image_url):
|
355 |
name = model_name.split('.')
|
|
|
374 |
extension_repo.append((url, file_name))
|
375 |
|
376 |
def manual_download(url, dst_dir, file_name):
|
|
|
|
|
377 |
aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'
|
378 |
+
basename = url.split("/")[-1] if file_name is None else file_name
|
379 |
+
header_option = f"--header={user_header}"
|
380 |
|
381 |
# === CivitAi API ===
|
382 |
+
supported_types = ('Checkpoint', 'TextualInversion', 'LORA') # for dl preview image
|
383 |
civitai_token = "62c0c5956b2f9defbd844d754000180b"
|
384 |
+
image_file_name = None
|
385 |
+
model_type = None
|
386 |
+
|
387 |
if 'civitai' in url:
|
|
|
|
|
|
|
388 |
url = f"{url}{'&' if '?' in url else '?'}token={civitai_token}" # some authors are pussies and they need API token...
|
389 |
+
model_id = url.split('/')[-1]
|
390 |
+
clean_url = url.split('?')[-1]
|
391 |
|
392 |
data = get_data_from_api(model_id, civitai_token)
|
393 |
if data:
|
394 |
+
model_type, model_name, image_url = extract_model_info(data, url)
|
395 |
if model_name and image_url:
|
396 |
image_file_name = gen_preview_filename(model_name if not file_name else file_name, image_url)
|
397 |
+
if model_type in supported_types:
|
398 |
with capture.capture_output() as cap: # clear shit
|
399 |
get_ipython().system('aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}')
|
400 |
del cap
|
401 |
+
file_name = file_name or model_name # assigns the original file name if not specified initially
|
402 |
|
403 |
""" information output """
|
|
|
404 |
# print(url, dst_dir, file_name if not 'huggingface' in url else basename) # for my tests
|
405 |
+
print(f"\n\033[32m{'---'*45}\n\033[33mURL: \033[34m{clean_url if 'civitai' in url else url}\n\033[33mSAVE DIR: \033[34m{dst_dir}\n\033[33mFILE NAME: \033[34m{file_name if not 'huggingface' in url else basename}\033[0m") # wrold's best print info
|
406 |
+
print(f"\033[32m[Preview DL]\033[0m: {image_file_name}\n") if 'civitai' in url and model_type in supported_types else None # PreviewDl: print file name
|
407 |
+
print("\033[31m[Data Info]\033[0m: Failed to retrieve data from the API.\n") if 'civitai' in url and not data else None # DATA: ERROR
|
408 |
# ===================
|
409 |
|
410 |
# -- GDrive --
|
|
|
544 |
pass
|
545 |
|
546 |
# url prefixing
|
547 |
+
urls = (Model_url, Vae_url, LoRA_url, Embedding_url, Extensions_url)
|
548 |
+
prefixed_urls = (f"{prefix}:{url}" for prefix, url in zip(prefixes.keys(), urls) if url for url in url.replace(',', '').split())
|
549 |
url += ", ".join(prefixed_urls) + ", " + file_urls
|
550 |
|
551 |
if detailed_download == "on":
|
files_cells/python/ru/downloading_ru.py
CHANGED
@@ -10,7 +10,6 @@ import requests
|
|
10 |
import subprocess
|
11 |
from datetime import timedelta
|
12 |
from subprocess import getoutput
|
13 |
-
from urllib.parse import unquote
|
14 |
from IPython.utils import capture
|
15 |
from IPython.display import clear_output
|
16 |
|
@@ -320,7 +319,7 @@ prefixes = {
|
|
320 |
}
|
321 |
|
322 |
extension_repo = []
|
323 |
-
directories =
|
324 |
get_ipython().system('mkdir -p {" ".join(directories)}')
|
325 |
|
326 |
hf_token = optional_huggingface_token if optional_huggingface_token else "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"
|
@@ -329,7 +328,7 @@ user_header = f"\"Authorization: Bearer {hf_token}\""
|
|
329 |
''' Get Image Preview | CivitAi '''
|
330 |
|
331 |
def get_data_from_api(model_id, api_key): # get model data
|
332 |
-
base_url = f"https://civitai.com/api/v1/model-versions/{model_id}
|
333 |
headers = {"Content-Type": "application/json"}
|
334 |
try:
|
335 |
response = requests.get(base_url, headers=headers)
|
@@ -339,10 +338,18 @@ def get_data_from_api(model_id, api_key): # get model data
|
|
339 |
print(f"An error occurred: {e}")
|
340 |
return None
|
341 |
|
342 |
-
def extract_model_info(data):
|
343 |
-
|
344 |
-
|
345 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
346 |
|
347 |
def gen_preview_filename(model_name, image_url):
|
348 |
name = model_name.split('.')
|
@@ -367,34 +374,37 @@ def handle_manual(url):
|
|
367 |
extension_repo.append((url, file_name))
|
368 |
|
369 |
def manual_download(url, dst_dir, file_name):
|
370 |
-
header_option = f"--header={user_header}"
|
371 |
-
basename = url.split("/")[-1] if file_name is None else file_name
|
372 |
aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'
|
|
|
|
|
373 |
|
374 |
# === CivitAi API ===
|
|
|
375 |
civitai_token = "62c0c5956b2f9defbd844d754000180b"
|
|
|
|
|
|
|
376 |
if 'civitai' in url:
|
377 |
-
clean_url = url # virgin url without token
|
378 |
-
image_file_name = None # for PreviewDl
|
379 |
-
model_id = url.split('/')[-1]
|
380 |
url = f"{url}{'&' if '?' in url else '?'}token={civitai_token}" # some authors are pussies and they need API token...
|
|
|
|
|
381 |
|
382 |
data = get_data_from_api(model_id, civitai_token)
|
383 |
if data:
|
384 |
-
model_name, image_url = extract_model_info(data)
|
385 |
if model_name and image_url:
|
386 |
image_file_name = gen_preview_filename(model_name if not file_name else file_name, image_url)
|
387 |
-
if
|
388 |
with capture.capture_output() as cap: # clear shit
|
389 |
get_ipython().system('aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}')
|
390 |
del cap
|
391 |
-
|
392 |
|
393 |
""" information output """
|
394 |
-
print(f"\n\033[32m{'---'*45}\n\033[33mURL: \033[34m{clean_url if 'civitai' in url else url}\n\033[33mSAVE DIR: \033[34m{dst_dir}\n\033[33mFILE NAME: \033[34m{file_name if not 'huggingface' in url else basename}\033[0m") # wrold's best print info
|
395 |
# print(url, dst_dir, file_name if not 'huggingface' in url else basename) # for my tests
|
396 |
-
print(f"\033[32m[
|
397 |
-
print("\033[
|
|
|
398 |
# ===================
|
399 |
|
400 |
# -- GDrive --
|
@@ -534,8 +544,8 @@ if custom_file_urls:
|
|
534 |
pass
|
535 |
|
536 |
# url prefixing
|
537 |
-
urls =
|
538 |
-
prefixed_urls =
|
539 |
url += ", ".join(prefixed_urls) + ", " + file_urls
|
540 |
|
541 |
if detailed_download == "on":
|
|
|
10 |
import subprocess
|
11 |
from datetime import timedelta
|
12 |
from subprocess import getoutput
|
|
|
13 |
from IPython.utils import capture
|
14 |
from IPython.display import clear_output
|
15 |
|
|
|
319 |
}
|
320 |
|
321 |
extension_repo = []
|
322 |
+
directories = (value for key, value in prefixes.items()) # for unpucking zip files
|
323 |
get_ipython().system('mkdir -p {" ".join(directories)}')
|
324 |
|
325 |
hf_token = optional_huggingface_token if optional_huggingface_token else "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"
|
|
|
328 |
''' Get Image Preview | CivitAi '''
|
329 |
|
330 |
def get_data_from_api(model_id, api_key): # get model data
|
331 |
+
base_url = f"https://civitai.com/api/v1/model-versions/{model_id}" # model_id - contains not only ID but also params
|
332 |
headers = {"Content-Type": "application/json"}
|
333 |
try:
|
334 |
response = requests.get(base_url, headers=headers)
|
|
|
338 |
print(f"An error occurred: {e}")
|
339 |
return None
|
340 |
|
341 |
+
def extract_model_info(data, url):
|
342 |
+
# Get 'Type, Name and imgURL' current model
|
343 |
+
if 'type=' in url:
|
344 |
+
model_type = data['files'][1]['type']
|
345 |
+
model_name = data['files'][1]['name']
|
346 |
+
image_url = data['images'][0]['url']
|
347 |
+
else:
|
348 |
+
model_type = data['model']['type']
|
349 |
+
model_name = data['files'][0]['name']
|
350 |
+
image_url = data['images'][0]['url']
|
351 |
+
|
352 |
+
return model_type, model_name, image_url
|
353 |
|
354 |
def gen_preview_filename(model_name, image_url):
|
355 |
name = model_name.split('.')
|
|
|
374 |
extension_repo.append((url, file_name))
|
375 |
|
376 |
def manual_download(url, dst_dir, file_name):
|
|
|
|
|
377 |
aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'
|
378 |
+
basename = url.split("/")[-1] if file_name is None else file_name
|
379 |
+
header_option = f"--header={user_header}"
|
380 |
|
381 |
# === CivitAi API ===
|
382 |
+
supported_types = ('Checkpoint', 'TextualInversion', 'LORA') # for dl preview image
|
383 |
civitai_token = "62c0c5956b2f9defbd844d754000180b"
|
384 |
+
image_file_name = None
|
385 |
+
model_type = None
|
386 |
+
|
387 |
if 'civitai' in url:
|
|
|
|
|
|
|
388 |
url = f"{url}{'&' if '?' in url else '?'}token={civitai_token}" # some authors are pussies and they need API token...
|
389 |
+
model_id = url.split('/')[-1]
|
390 |
+
clean_url = url.split('?')[-1]
|
391 |
|
392 |
data = get_data_from_api(model_id, civitai_token)
|
393 |
if data:
|
394 |
+
model_type, model_name, image_url = extract_model_info(data, url)
|
395 |
if model_name and image_url:
|
396 |
image_file_name = gen_preview_filename(model_name if not file_name else file_name, image_url)
|
397 |
+
if model_type in supported_types:
|
398 |
with capture.capture_output() as cap: # clear shit
|
399 |
get_ipython().system('aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}')
|
400 |
del cap
|
401 |
+
file_name = file_name or model_name # assigns the original file name if not specified initially
|
402 |
|
403 |
""" information output """
|
|
|
404 |
# print(url, dst_dir, file_name if not 'huggingface' in url else basename) # for my tests
|
405 |
+
print(f"\n\033[32m{'---'*45}\n\033[33mURL: \033[34m{clean_url if 'civitai' in url else url}\n\033[33mSAVE DIR: \033[34m{dst_dir}\n\033[33mFILE NAME: \033[34m{file_name if not 'huggingface' in url else basename}\033[0m") # wrold's best print info
|
406 |
+
print(f"\033[32m[Preview DL]\033[0m: {image_file_name}\n") if 'civitai' in url and model_type in supported_types else None # PreviewDl: print file name
|
407 |
+
print("\033[31m[Data Info]\033[0m: Failed to retrieve data from the API.\n") if 'civitai' in url and not data else None # DATA: ERROR
|
408 |
# ===================
|
409 |
|
410 |
# -- GDrive --
|
|
|
544 |
pass
|
545 |
|
546 |
# url prefixing
|
547 |
+
urls = (Model_url, Vae_url, LoRA_url, Embedding_url, Extensions_url)
|
548 |
+
prefixed_urls = (f"{prefix}:{url}" for prefix, url in zip(prefixes.keys(), urls) if url for url in url.replace(',', '').split())
|
549 |
url += ", ".join(prefixed_urls) + ", " + file_urls
|
550 |
|
551 |
if detailed_download == "on":
|