Upload folder using huggingface_hub

README.md

@@ -1,12 +1,6 @@
 ---
-title: Model Memory And Popularity
-emoji: 🏆
-colorFrom: green
-colorTo: blue
+title: Model_Memory_and_Popularity
+app_file: app.py
 sdk: gradio
 sdk_version: 4.16.0
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,86 @@
+import gradio as gr
+import pandas as pd
+from hub_utils import check_for_discussion, report_results
+from model_utils import calculate_memory, get_model
+from huggingface_hub.utils import HfHubHTTPError
+from hub_model_stats_utils import get_model_type_downloads
+
+# We need to store them as globals because gradio doesn't have a way for us to pass them into the button
+MODEL = None
+TASK_INP = None
+
+def get_mem_results(model_name: str, library: str, options: list, access_token: str):
+    global MODEL
+    MODEL = get_model(model_name, library, access_token)
+    try:
+        has_discussion = check_for_discussion(model_name)
+    except HfHubHTTPError:
+        has_discussion = True
+    title = f"## Memory usage for '{model_name}'"
+    data = calculate_memory(MODEL, options)
+    return [title, gr.update(visible=True, value=pd.DataFrame(data)), gr.update(visible=not has_discussion)]
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("""<h1>Positron Model Universe Explorer</h1>""")
+    with gr.Tab("Model Size"):
+        gr.Markdown(
+            """<h1>Model Memory Calculator</h1>
+
+            This tool will help you calculate how much vRAM is needed to train and perform big model inference
+            on a model hosted on the 🤗 Hugging Face Hub. The minimum recommended vRAM needed for a model
+            s denoted as the size of the "largest layer", and training of a model is roughly 4x its size (for Adam)."""
+        )
+
+        out_text = gr.Markdown()
+        mem_out = gr.DataFrame(
+            headers=["dtype", "Largest Layer", "Total Size", "Training using Adam"],
+            interactive=False,
+            visible=True,
+        )
+        with gr.Row():
+            model_name_inp = gr.Textbox(label="Model Name or URL", value="TheBloke/Nous-Hermes-13B-GPTQ")
+        with gr.Row():
+            library = gr.Radio(["auto", "transformers", "timm"], label="Library", value="auto")
+            options = gr.CheckboxGroup(
+                ["float32", "float16/bfloat16", "int8", "int4"],
+                value="float32",
+                label="Model Precision",
+            )
+            access_token = gr.Textbox(label="API Token", placeholder="Optional (for gated models)")
+        with gr.Row():
+            mem_btn = gr.Button("Calculate Memory Usage")
+            post_to_hub = gr.Button(
+                value="Report results in this model repo's discussions!\n(Will open in a new tab)", visible=False
+            )
+
+        mem_btn.click(
+            get_mem_results,
+            inputs=[model_name_inp, library, options, access_token],
+            outputs=[out_text, mem_out, post_to_hub],
+        )
+
+    with gr.Tab("Model Type"):
+        gr.Markdown(
+            """<h1>Models by Model Task</h1>"""
+        )
+
+        with gr.Row():
+            task_inp = gr.Dropdown(choices = ["text-generation", "question-answering", "text-classification", "unconditional-image-generation"], 
+                value="text-generation", interactive=True, filterable=True, label="Model Task", show_label=True
+            )
+            stat_inp = gr.Radio(choices = ["downloads", "likes"], value = "downloads", label = "Sorting By", show_label=True, interactive = True)
+        
+        with gr.Row():
+            stats_btn = gr.Button("Pull Stats from HF API")
+
+        stats_btn.click(
+            get_model_type_downloads,
+            inputs=[task_inp],
+            outputs=[stats_out]
+        )
+
+        stats_out = gr.Dataframe(row_count=10)
+        # gr.Dataframe(get_model_type_likes(), row_count=10)
+
+demo.launch(share=True)

config_json_utils.py

@@ -0,0 +1,62 @@
+from pathlib import Path
+import json
+from collections import Counter
+import operator
+
+import numpy as np
+import matplotlib.pyplot as plt
+from string import ascii_lowercase as alc
+from string import ascii_uppercase as auc
+ 
+# assign directory
+directory = 'HFFiles'
+arch_list = []
+invalid_counter = 0
+
+# iterate over files in directory
+files = Path(directory).glob('*.json')
+for file in files:
+    f = open(file)
+    try:
+        data = json.load(f)
+    except:
+        # print(str(file) + " does not look like valid JSON.")
+        invalid_counter += 1
+        continue
+    try:
+        for i in data['architectures']:
+            # print(i)
+            arch_list.append(i[0])
+    except:
+        # print(str(file) + " does not contain architectures key.")
+        invalid_counter += 1
+
+    f.close()
+
+my_dict = Counter(arch_list)
+# print(my_dict)
+print("\n")
+print(str(invalid_counter) + " JSON files are invalid.")
+
+for i in alc:
+    del my_dict[i]
+for i in auc:
+    del my_dict[i]
+
+causal_dict = {k:v for (k,v) in my_dict.items() if "Causal" in k}
+
+sorted_causal_dict = dict(sorted(causal_dict.items(),key=operator.itemgetter(1),reverse=True))
+sorted_causal_dict = {key:val for key, val in sorted_causal_dict.items() if val != 1}
+
+print(sorted_causal_dict)
+
+labels, values = zip(*sorted_causal_dict.items())
+
+indexes = np.arange(len(labels))
+width = 1
+
+plt.figure(figsize=(16,9))
+plt.bar(indexes, values, width)
+plt.xticks(indexes + width * 0.5, labels)
+plt.xticks(rotation=90)
+plt.show()

hub_model_stats_utils.py

@@ -0,0 +1,28 @@
+import pandas as pd
+from huggingface_hub import HfApi, ModelFilter
+
+def call_hub_api(model_task: str):
+    api = HfApi()
+    models = api.list_models(
+        filter=ModelFilter(
+            task=model_task
+        )
+    )
+    return list(models)
+
+
+def get_model_type_downloads(model_task: str):
+    models = call_hub_api(model_task)
+
+    dict_list = []
+    for cur_model in models:
+        this_dict = {"id":cur_model.id, "downloads":cur_model.downloads, "likes":cur_model.likes, "author": cur_model.author, "created": cur_model.created_at, "modified": cur_model.last_modified}
+        dict_list.append(this_dict)
+
+    df = pd.DataFrame(dict_list)
+    return df.sort_values("downloads", ascending=False).head(50)
+
+    # df_likes_desc = df.sort_values("likes", ascending=False)
+    # df_likes_desc.head(10)
+
+    # df_likes_desc['id'].str.split('/', n=1, expand=True)

hub_utils.py

@@ -0,0 +1,44 @@
+# Utilities related to searching and posting on the Hub
+import os
+import webbrowser
+
+import pandas as pd
+from huggingface_hub import HfApi
+from model_utils import calculate_memory, extract_from_url, get_model
+
+
+def check_for_discussion(model_name: str):
+    "Checks if an automated discussion has been opened on the model by `model-sizer-bot`"
+    api = HfApi(token=os.environ.get("HUGGINGFACE_API_LOGIN", None))
+    model_name = extract_from_url(model_name)
+    discussions = list(api.get_repo_discussions(model_name))
+    return any(
+        discussion.title == "[AUTOMATED] Model Memory Requirements" and discussion.author == "model-sizer-bot"
+        for discussion in discussions
+    )
+
+
+def report_results(model_name, library, access_token):
+    "Reports the results of a memory calculation to the model's discussion page, and opens a new tab to it afterwards"
+    model = get_model(model_name, library, access_token)
+    data = calculate_memory(model, ["float32", "float16/bfloat16", "int8", "int4"])
+    df = pd.DataFrame(data).to_markdown(index=False)
+
+    post = f"""# Model Memory Requirements\n
+
+    You will need about {data[1]} VRAM to load this model for inference, and {data[3]} VRAM to train it using Adam.
+        
+    These calculations were measured from the [Model Memory Utility Space](https://hf.co/spaces/hf-accelerate/model-memory-utility) on the Hub.
+        
+    The minimum recommended vRAM needed for this model assumes using [Accelerate or `device_map="auto"`](https://huggingface.co/docs/accelerate/usage_guides/big_modeling) and is denoted by the size of the "largest layer". 
+    When performing inference, expect to add up to an additional 20% to this, as found by [EleutherAI](https://blog.eleuther.ai/transformer-math/). More tests will be performed in the future to get a more accurate benchmark for each model.
+
+    When training with `Adam`, you can expect roughly 4x the reported results to be used. (1x for the model, 1x for the gradients, and 2x for the optimizer).
+
+    ## Results:
+
+    {df}
+    """
+    api = HfApi(token=os.environ.get("HUGGINGFACE_API_LOGIN", None))
+    discussion = api.create_discussion(model_name, "[AUTOMATED] Model Memory Requirements", description=post)
+    webbrowser.open_new_tab(discussion.url)

model_utils.py

@@ -0,0 +1,101 @@
+# Utilities related to loading in and working with models/specific models
+from urllib.parse import urlparse
+
+import gradio as gr
+import torch
+from accelerate.commands.estimate import check_has_model, create_empty_model
+from accelerate.utils import calculate_maximum_sizes, convert_bytes
+from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError
+
+
+DTYPE_MODIFIER = {"float32": 1, "float16/bfloat16": 2, "int8": 4, "int4": 8}
+
+
+def extract_from_url(name: str):
+    "Checks if `name` is a URL, and if so converts it to a model name"
+    is_url = False
+    try:
+        result = urlparse(name)
+        is_url = all([result.scheme, result.netloc])
+    except Exception:
+        is_url = False
+    # Pass through if not a URL
+    if not is_url:
+        return name
+    else:
+        path = result.path
+        return path[1:]
+
+
+def translate_llama2(text):
+    "Translates llama-2 to its hf counterpart"
+    if not text.endswith("-hf"):
+        return text + "-hf"
+    return text
+
+
+def get_model(model_name: str, library: str, access_token: str):
+    "Finds and grabs model from the Hub, and initializes on `meta`"
+    if "meta-llama" in model_name:
+        model_name = translate_llama2(model_name)
+    if library == "auto":
+        library = None
+    model_name = extract_from_url(model_name)
+    try:
+        model = create_empty_model(model_name, library_name=library, trust_remote_code=True, access_token=access_token)
+    except GatedRepoError:
+        raise gr.Error(
+            f"Model `{model_name}` is a gated model, please ensure to pass in your access token and try again if you have access. You can find your access token here : https://huggingface.co/settings/tokens. "
+        )
+    except RepositoryNotFoundError:
+        raise gr.Error(f"Model `{model_name}` was not found on the Hub, please try another model name.")
+    except ValueError:
+        raise gr.Error(
+            f"Model `{model_name}` does not have any library metadata on the Hub, please manually select a library_name to use (such as `transformers`)"
+        )
+    except (RuntimeError, OSError) as e:
+        library = check_has_model(e)
+        if library != "unknown":
+            raise gr.Error(
+                f"Tried to load `{model_name}` with `{library}` but a possible model to load was not found inside the repo."
+            )
+        raise gr.Error(
+            f"Model `{model_name}` had an error, please open a discussion on the model's page with the error message and name: `{e}`"
+        )
+    except ImportError:
+        # hacky way to check if it works with `trust_remote_code=False`
+        model = create_empty_model(
+            model_name, library_name=library, trust_remote_code=False, access_token=access_token
+        )
+    except Exception as e:
+        raise gr.Error(
+            f"Model `{model_name}` had an error, please open a discussion on the model's page with the error message and name: `{e}`"
+        )
+    return model
+
+
+def calculate_memory(model: torch.nn.Module, options: list):
+    "Calculates the memory usage for a model init on `meta` device"
+    total_size, largest_layer = calculate_maximum_sizes(model)
+
+    data = []
+    for dtype in options:
+        dtype_total_size = total_size
+        dtype_largest_layer = largest_layer[0]
+
+        modifier = DTYPE_MODIFIER[dtype]
+        dtype_total_size /= modifier
+        dtype_largest_layer /= modifier
+
+        dtype_training_size = convert_bytes(dtype_total_size * 4)
+        dtype_total_size = convert_bytes(dtype_total_size)
+        dtype_largest_layer = convert_bytes(dtype_largest_layer)
+        data.append(
+            {
+                "dtype": dtype,
+                "Largest Layer or Residual Group": dtype_largest_layer,
+                "Total Size": dtype_total_size,
+                "Training using Adam": dtype_training_size,
+            }
+        )
+    return data