Upload 9 files

README.md

@@ -7,6 +7,7 @@ sdk: gradio
 sdk_version: 4.44.0
 app_file: app.py
 pinned: false
+license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -15,9 +15,11 @@ with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", fill_width=True, css=css, delete_ca
         is_upload = gr.Checkbox(label="Upload safetensors to HF Repo", info="Fast download, but files will be public.", value=False)
         with gr.Accordion("Advanced", open=False):
             dtype = gr.Radio(label="Output data type", choices=["fp16", "fp32", "bf16", "default"], value="fp16")
-            hf_token = gr.Textbox(label="Your HF write token", placeholder="hf_...", value="", max_lines=1)
             with gr.Row():
-                newrepo_id = gr.Textbox(label="Upload repo ID", placeholder="author/model", value="", max_lines=1)
+                hf_token = gr.Textbox(label="Your HF write token", placeholder="hf_...", value="", max_lines=1)
+                gr.Markdown("Your token is available at [hf.co/settings/tokens](https://huggingface.co/settings/tokens).")
+            with gr.Row():
+                newrepo_id = gr.Textbox(label="Upload repo ID", placeholder="yourid/newrepo", value="", max_lines=1)
                 newrepo_type = gr.Radio(label="Upload repo type", choices=["model", "dataset"], value="model")
                 is_private = gr.Checkbox(label="Create / Use private repo", value=True)
         uploaded_urls = gr.CheckboxGroup(visible=False, choices=[], value=None) # hidden

convert_repo_to_safetensors_gr.py

@@ -15,23 +15,8 @@ import os
 from pathlib import Path
 import shutil
 import gc
+from utils import get_token, set_token, is_repo_exists
 
-def get_token():
-    try:
-        token = HfFolder.get_token()
-    except Exception:
-        token = ""
-    return token
-
-def is_repo_exists(repo_id: str, repo_type: str="model"):
-    hf_token = get_token()
-    api = HfApi(token=hf_token)
-    try:
-        if api.repo_exists(repo_id=repo_id, repo_type=repo_type, token=hf_token): return True
-        else: return False
-    except Exception as e:
-        print(f"Error: Failed to connect {repo_id} ({repo_type}). {e}")
-        return True # for safe
 
 # =================#
 # UNet Conversion #
@@ -292,7 +277,7 @@ def convert_openai_text_enc_state_dict(text_enc_dict):
     return text_enc_dict
 
 
-def convert_diffusers_to_safetensors(model_path, checkpoint_path, dtype="fp16"):
+def convert_diffusers_to_safetensors(model_path, checkpoint_path, dtype="fp16", progress=gr.Progress(track_tqdm=True)):
     # Path for safetensors
     unet_path = osp.join(model_path, "unet", "diffusion_pytorch_model.safetensors")
     vae_path = osp.join(model_path, "vae", "diffusion_pytorch_model.safetensors")
@@ -355,10 +340,12 @@ def convert_diffusers_to_safetensors(model_path, checkpoint_path, dtype="fp16"):
     save_file(state_dict, checkpoint_path)
 
 
-def download_repo(repo_id, dir_path):
+# https://huggingface.co/docs/huggingface_hub/v0.25.1/en/package_reference/file_download#huggingface_hub.snapshot_download
+def download_repo(repo_id, dir_path, progress=gr.Progress(track_tqdm=True)):
     hf_token = get_token()
     try:
-        snapshot_download(repo_id=repo_id, local_dir=dir_path, token=hf_token)
+        snapshot_download(repo_id=repo_id, local_dir=dir_path, token=hf_token, allow_patterns=["*.safetensors", "*.bin"],
+                          ignore_patterns=["*.fp16.*", "/*.safetensors", "/*.bin"], force_download=True)
     except Exception as e:
         print(f"Error: Failed to download {repo_id}. {e}")
         gr.Warning(f"Error: Failed to download {repo_id}. {e}")
@@ -370,11 +357,11 @@ def upload_safetensors_to_repo(filename, repo_id, repo_type, is_private, progres
     hf_token = get_token()
     api = HfApi(token=hf_token)
     try:
-        if not is_repo_exists(repo_id, repo_type): api.create_repo(repo_id, repo_type=repo_type, token=hf_token, private=is_private)
+        if not is_repo_exists(repo_id, repo_type): api.create_repo(repo_id=repo_id, repo_type=repo_type, token=hf_token, private=is_private)
         progress(0, desc="Start uploading...")
-        api.upload_file(path_or_fileobj=filename, path_in_repo=output_filename, repo_type=repo_type, token=hf_token, repo_id=repo_id)
+        api.upload_file(path_or_fileobj=filename, path_in_repo=output_filename, repo_type=repo_type, revision="main", token=hf_token, repo_id=repo_id)
         progress(1, desc="Uploaded.")
-        url = hf_hub_url(repo_id=repo_id, repo_type=repo_type, filename=output_filename, token=hf_token)
+        url = hf_hub_url(repo_id=repo_id, repo_type=repo_type, filename=output_filename)
     except Exception as e:
         print(f"Error: Failed to upload to {repo_id}. {e}")
         gr.Warning(f"Error: Failed to upload to {repo_id}. {e}")
@@ -396,9 +383,9 @@ def convert_repo_to_safetensors(repo_id, dtype="fp16", progress=gr.Progress(trac
 
 def convert_repo_to_safetensors_multi(repo_id, hf_token, files, urls, dtype="fp16", is_upload=False,
                                       newrepo_id="", repo_type="model", is_private=True, progress=gr.Progress(track_tqdm=True)):
-    if is_upload and newrepo_id and not hf_token: raise gr.Error("Invalid token.")
-    if hf_token: HfFolder.save_token(hf_token)
-    else: HfFolder.save_token(os.environ.get("HF_TOKEN"))
+    if hf_token: set_token(hf_token)
+    else: set_token(os.environ.get("HF_TOKEN"))
+    if is_upload and newrepo_id and not hf_token: raise gr.Error("HF write token is required for this process.")
     if not newrepo_id: newrepo_id = os.environ.get("HF_OUTPUT_REPO")
     file = convert_repo_to_safetensors(repo_id, dtype)
     if not urls: urls = []

local/convert_repo_to_safetensors.py

@@ -269,7 +269,7 @@ def convert_openai_text_enc_state_dict(text_enc_dict):
     return text_enc_dict
 
 
-def convert_diffusers_to_safetensors(model_path, checkpoint_path, half = True):
+def convert_diffusers_to_safetensors(model_path, checkpoint_path, dtype="fp16"):
     # Path for safetensors
     unet_path = osp.join(model_path, "unet", "diffusion_pytorch_model.safetensors")
     vae_path = osp.join(model_path, "vae", "diffusion_pytorch_model.safetensors")
@@ -325,8 +325,9 @@ def convert_diffusers_to_safetensors(model_path, checkpoint_path, half = True):
     # Put together new checkpoint
     state_dict = {**unet_state_dict, **vae_state_dict, **text_enc_dict, **text_enc_2_dict}
 
-    if half:
-        state_dict = {k: v.half() for k, v in state_dict.items()}
+    if dtype == "fp16": state_dict = {k: v.half() for k, v in state_dict.items()}
+    elif dtype == "fp32": state_dict = {k: v.to(torch.float32) for k, v in state_dict.items()}
+    elif dtype == "bf16": state_dict = {k: v.to(torch.bfloat16) for k, v in state_dict.items()}
 
     save_file(state_dict, checkpoint_path)
 
@@ -336,15 +337,15 @@ def download_repo(repo_id, dir_path):
     try:
         snapshot_download(repo_id=repo_id, local_dir=dir_path)
     except Exception as e:
-        print(f"Error: Failed to download {repo_id}. ")
+        print(f"Error: Failed to download {repo_id}. {e}")
         return
 
 
-def convert_repo_to_safetensors(repo_id, half=True):
+def convert_repo_to_safetensors(repo_id, dtype="fp16"):
     download_dir = f"{repo_id.split('/')[0]}_{repo_id.split('/')[-1]}"
     output_filename = f"{repo_id.split('/')[0]}_{repo_id.split('/')[-1]}.safetensors"
     download_repo(repo_id, download_dir)
-    convert_diffusers_to_safetensors(download_dir, output_filename, half)
+    convert_diffusers_to_safetensors(download_dir, output_filename, dtype)
     return output_filename
 
 
@@ -352,12 +353,12 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
     parser.add_argument("--repo_id", default=None, type=str, required=True, help="HF Repo ID of the model to convert.")
-    parser.add_argument("--half", default=True, help="Save weights in half precision.")
+    parser.add_argument("--dtype", default="fp16", type=str, choices=["fp16", "fp32", "bf16", "default"], help='Output data type. (Default: "fp16")')
 
     args = parser.parse_args()
     assert args.repo_id is not None, "Must provide a Repo ID!"
 
-    convert_repo_to_safetensors(args.repo_id, args.half)
+    convert_repo_to_safetensors(args.repo_id, args.dtype)
 
 
 # Usage: python convert_repo_to_safetensors.py --repo_id GraydientPlatformAPI/goodfit-pony41-xl

packages.txt

@@ -0,0 +1 @@
+git-lfs aria2

utils.py

@@ -0,0 +1,132 @@
+import gradio as gr
+from huggingface_hub import HfApi, HfFolder, hf_hub_download
+import os
+from pathlib import Path
+import shutil
+import gc
+import re
+import urllib.parse
+
+
+def get_token():
+    try:
+        token = HfFolder.get_token()
+    except Exception:
+        token = ""
+    return token
+
+
+def set_token(token):
+    try:
+        HfFolder.save_token(token)
+    except Exception:
+        print(f"Error: Failed to save token.")
+
+
+def is_repo_exists(repo_id: str, repo_type: str="model"):
+    hf_token = get_token()
+    api = HfApi(token=hf_token)
+    try:
+        if api.repo_exists(repo_id=repo_id, repo_type=repo_type, token=hf_token): return True
+        else: return False
+    except Exception as e:
+        print(f"Error: Failed to connect {repo_id} ({repo_type}). {e}")
+        return True # for safe
+
+
+def list_sub(a, b):
+    return [e for e in a if e not in b]
+
+
+def is_repo_name(s):
+    return re.fullmatch(r'^[^/,\s\"\']+/[^/,\s\"\']+$', s)
+
+
+def split_hf_url(url: str):
+    try:
+        s = list(re.findall(r'^(?:https?://huggingface.co/)(?:(datasets)/)?(.+?/.+?)/\w+?/.+?/(?:(.+)/)?(.+?.safetensors)(?:\?download=true)?$', url)[0])
+        if len(s) < 4: return "", "", "", ""
+        repo_id = s[1]
+        repo_type = "dataset" if s[0] == "datasets" else "model"
+        subfolder = urllib.parse.unquote(s[2]) if s[2] else None
+        filename = urllib.parse.unquote(s[3])
+        return repo_id, filename, subfolder, repo_type
+    except Exception as e:
+        print(e)
+
+
+def download_hf_file(directory, url, progress=gr.Progress(track_tqdm=True)):
+    hf_token = get_token()
+    repo_id, filename, subfolder, repo_type = split_hf_url(url)
+    try:
+        if subfolder is not None: hf_hub_download(repo_id=repo_id, filename=filename, subfolder=subfolder, repo_type=repo_type, local_dir=directory, token=hf_token)
+        else: hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type, local_dir=directory, token=hf_token)
+    except Exception as e:
+        print(f"Failed to download: {e}")
+
+
+def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(track_tqdm=True)): # requires aria2
+    hf_token = get_token()
+    url = url.strip()
+    if "drive.google.com" in url:
+        original_dir = os.getcwd()
+        os.chdir(directory)
+        os.system(f"gdown --fuzzy {url}")
+        os.chdir(original_dir)
+    elif "huggingface.co" in url:
+        url = url.replace("?download=true", "")
+        if "/blob/" in url:
+            url = url.replace("/blob/", "/resolve/")
+        #user_header = f'"Authorization: Bearer {hf_token}"'
+        if hf_token:
+            download_hf_file(directory, url)
+            #os.system(f"aria2c --console-log-level=error --summary-interval=10 --header={user_header} -c -x 16 -k 1M -s 16 {url} -d {directory}  -o {url.split('/')[-1]}")
+        else:
+            os.system(f"aria2c --optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 {url} -d {directory}  -o {url.split('/')[-1]}")
+    elif "civitai.com" in url:
+        if "?" in url:
+            url = url.split("?")[0]
+        if civitai_api_key:
+            url = url + f"?token={civitai_api_key}"
+            os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
+        else:
+            print("You need an API key to download Civitai models.")
+    else:
+        os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
+
+
+def get_local_model_list(dir_path):
+    model_list = []
+    valid_extensions = ('.safetensors')
+    for file in Path(dir_path).glob("**/*.*"):
+        if file.is_file() and file.suffix in valid_extensions:
+            file_path = str(file)
+            model_list.append(file_path)
+    return model_list
+
+
+def get_download_file(temp_dir, url, civitai_key, progress=gr.Progress(track_tqdm=True)):
+    if not "http" in url and is_repo_name(url) and not Path(url).exists():
+        print(f"Use HF Repo: {url}")
+        new_file = url
+    elif not "http" in url and Path(url).exists():
+        print(f"Use local file: {url}")
+        new_file = url
+    elif Path(f"{temp_dir}/{url.split('/')[-1]}").exists():
+        print(f"File to download alreday exists: {url}")
+        new_file = f"{temp_dir}/{url.split('/')[-1]}"
+    else:
+        print(f"Start downloading: {url}")
+        before = get_local_model_list(temp_dir)
+        try:
+            download_thing(temp_dir, url.strip(), civitai_key)
+        except Exception:
+            print(f"Download failed: {url}")
+            return ""
+        after = get_local_model_list(temp_dir)
+        new_file = list_sub(after, before)[0] if list_sub(after, before) else ""
+    if not new_file:
+        print(f"Download failed: {url}")
+        return ""
+    print(f"Download completed: {url}")
+    return new_file