cleaning up a bit

app.py

@@ -68,67 +68,34 @@ def process_dataset():
     )
 
 
-def format_dataframe_size_column(_df, column_name):
-    """
-    Format the size to petabytes and return the formatted size.
-    """
-    _df[column_name] = _df[column_name] / 1e15
-    _df[column_name] = _df[column_name].map("{:.2f}".format)
-    return _df
-
-
-def cumulative_growth_plot_analysis(df, df_compressed, repo_sizes):
-    """
-    Calculates the cumulative growth of models, spaces, and datasets over time and generates a plot and dataframe from the analysis.
-
-    Args:
-        df (DataFrame): The input dataframe containing the data.
-        df_compressed (DataFrame): The input dataframe containing the compressed data.
-
-    Returns:
-        tuple: A tuple containing two elements:
-            - fig (Figure): The Plotly figure showing the cumulative growth of models, spaces, and datasets over time.
-            - last_10_months (DataFrame): The last 10 months of data showing the month-to-month growth in petabytes.
-
-    Raises:
-        None
-    """
-    # Convert year and month into a datetime column
-    df["date"] = pd.to_datetime(df[["year", "month"]].assign(day=1))
-    df_compressed["date"] = pd.to_datetime(
-        df_compressed[["year", "month"]].assign(day=1)
-    )
-
+def cumulative_growth_df(_df):
     # Sort by date to ensure correct cumulative sum
-    df = df.sort_values(by="date")
-    df_compressed = df_compressed.sort_values(by="date")
-
-    # Pivot the dataframe to get the totalsize for each type
-    pivot_df = df.pivot_table(
-        index="date", columns="type", values="totalsize", aggfunc="sum"
-    ).fillna(0)
-    pivot_df_compressed = df_compressed.pivot_table(
+    _df = _df.sort_values(by="date")
+    # Pivot the dataframe to get the totalsize
+    pivot_df = _df.pivot_table(
         index="date", columns="type", values="totalsize", aggfunc="sum"
     ).fillna(0)
-
-    # Calculate cumulative sum for each type
+    # Calculate cumulative sum
     cumulative_df = pivot_df.cumsum()
-    cumulative_df_compressed = pivot_df_compressed.cumsum()
+    return cumulative_df
+
 
-    last_10_months = cumulative_df.tail(10).copy()
+def compare_last_10_months(_cumulative_df, _cumulative_df_compressed):
+    last_10_months = _cumulative_df.tail(10).copy()
     last_10_months["total"] = last_10_months.sum(axis=1)
     last_10_months["total_change"] = last_10_months["total"].diff()
     last_10_months["compressed_change"] = (
-        cumulative_df_compressed.tail(10).sum(axis=1).diff()
+        _cumulative_df_compressed.tail(10).sum(axis=1).diff()
     )
     last_10_months["savings"] = (
         last_10_months["total_change"] - last_10_months["compressed_change"]
     )
-    last_10_months = format_dataframe_size_column(last_10_months, "savings")
-    last_10_months = format_dataframe_size_column(last_10_months, "compressed_change")
-    last_10_months = format_dataframe_size_column(last_10_months, "total_change")
 
-    last_10_months["date"] = cumulative_df.tail(10).index
+    last_10_months = format_dataframe_size_column(
+        last_10_months, ["total_change", "compressed_change", "savings"]
+    )
+
+    last_10_months["date"] = _cumulative_df.tail(10).index
     # drop the dataset, model, and space
     last_10_months = last_10_months.drop(columns=["model", "space", "dataset"])
     # pretiffy the date column to not have 00:00:00
@@ -148,17 +115,10 @@ def cumulative_growth_plot_analysis(df, df_compressed, repo_sizes):
             "savings": "Dedupe Savings (PBs)",
         }
     )
+    return last_10_months
 
-    # Create a Plotly figure
-    fig = go.Figure()
-
-    # Define a color map for each type
-    color_map = {
-        "model": px.colors.qualitative.Alphabet[3],
-        "space": px.colors.qualitative.Alphabet[2],
-        "dataset": px.colors.qualitative.Alphabet[9],
-    }
 
+def tabular_analysis(repo_sizes, cumulative_df, cumulative_df_compressed):
     # create a new column in the repository sizes dataframe for "compressed size" and set it to empty atif rist
     repo_sizes["Compressed Size (PBs)"] = ""
     repo_sizes["Dedupe Savings (PBs)"] = ""
@@ -175,6 +135,39 @@ def cumulative_growth_plot_analysis(df, df_compressed, repo_sizes):
             repo_sizes["Repository Type"] == column.capitalize(), "Dedupe Savings (PBs)"
         ] = repo_size_diff
 
+    # add a row that sums the total size and compressed size
+    repo_sizes.loc["Total"] = repo_sizes.sum()
+    repo_sizes.loc["Total", "Repository Type"] = "Total"
+    return repo_sizes
+
+
+def cumulative_growth_plot_analysis(cumulative_df, cumulative_df_compressed):
+    """
+    Calculates the cumulative growth of models, spaces, and datasets over time and generates a plot and dataframe from the analysis.
+
+    Args:
+        df (DataFrame): The input dataframe containing the data.
+        df_compressed (DataFrame): The input dataframe containing the compressed data.
+
+    Returns:
+        tuple: A tuple containing two elements:
+            - fig (Figure): The Plotly figure showing the cumulative growth of models, spaces, and datasets over time.
+            - last_10_months (DataFrame): The last 10 months of data showing the month-to-month growth in petabytes.
+
+    Raises:
+        None
+    """
+
+    # Create a Plotly figure
+    fig = go.Figure()
+
+    # Define a color map for each type
+    color_map = {
+        "model": px.colors.qualitative.Alphabet[3],
+        "space": px.colors.qualitative.Alphabet[2],
+        "dataset": px.colors.qualitative.Alphabet[9],
+    }
+
     # Add a scatter trace for each type
     for column in cumulative_df.columns:
         fig.add_trace(
@@ -207,7 +200,7 @@ def cumulative_growth_plot_analysis(df, df_compressed, repo_sizes):
         legend_title="Type",
         yaxis=dict(tickformat=".2f"),  # Format y-axis labels to 2 decimal places
     )
-    return fig, last_10_months
+    return fig
 
 
 def plot_total_sum(by_type_arr):
@@ -244,7 +237,7 @@ def filter_by_extension_month(_df, _extension):
 
     Parameters:
     _df (DataFrame): The input DataFrame containing the data.
-    extension (str): The extension to filter the DataFrame by. If set to "All", no filtering is applied.
+    extension (str): The extension to filter the DataFrame by. If None, no filtering is applied.
 
     Returns:
     fig (Figure): The Plotly figure object representing the line plot.
@@ -273,7 +266,7 @@ def filter_by_extension_month(_df, _extension):
             fig.add_trace(
                 go.Scatter(
                     x=pivot_df.index,
-                    y=pivot_df[column] / 1e12,  # Convert to petabytes
+                    y=pivot_df[column] / 1e12,  # Convert to TBs
                     mode="lines",
                     name=column,
                     line=dict(color=px.colors.qualitative.Alphabet[i]),
@@ -292,12 +285,70 @@ def filter_by_extension_month(_df, _extension):
     return fig
 
 
+def area_plot_by_extension_month(_df):
+    _df["total_size"] = _df["total_size"] / 1e15
+    _df["date"] = pd.to_datetime(_df[["year", "month"]].assign(day=1))
+    # make a plotly area chart with data and extension
+    fig = px.area(_df, x="date", y="total_size", color="extension")
+    # Update layout
+    fig.update_layout(
+        title="File Extension Cumulative Growth (in PBs) Over Time",
+        xaxis_title="Date",
+        yaxis_title="Size (PBs)",
+        legend_title="Type",
+        # format y-axis to be PBs (currently bytes) with two decimal places
+        yaxis=dict(tickformat=".2f"),
+    )
+
+    return fig
+
+
+## Utility functions
+def div_px(height):
+    """
+    Returns a string representing a div element with the specified height in pixels.
+    """
+    return f"<div style='height: {height}px;'></div>"
+
+
+def format_dataframe_size_column(_df, column_names):
+    """
+    Format the size to petabytes and return the formatted size.
+    """
+    for column_name in column_names:
+        _df[column_name] = _df[column_name] / 1e15
+        _df[column_name] = _df[column_name].map("{:.2f}".format)
+    return _df
+
+
+def month_year_to_date(_df):
+    """
+    Converts the 'year' and 'month' columns in the given DataFrame to a single 'date' column.
+    """
+    _df["date"] = pd.to_datetime(_df[["year", "month"]].assign(day=1))
+    return _df
+
+
 # Create a gradio blocks interface and launch a demo
 with gr.Blocks() as demo:
     df, file_df, by_repo_type, by_extension, by_extension_month = process_dataset()
 
+    # Convert year and month into a datetime column
+    df = month_year_to_date(df)
+    df_compressed = month_year_to_date(file_df)
+
+    # Calculate the cumulative growth of models, spaces, and datasets over time
+    cumulative_df = cumulative_growth_df(df)
+    cumulative_df_compressed = cumulative_growth_df(df_compressed)
+
+    last_10_months = compare_last_10_months(cumulative_df, cumulative_df_compressed)
+
+    by_repo_type = tabular_analysis(
+        by_repo_type, cumulative_df, cumulative_df_compressed
+    )
+
     # get the figure for the cumulative growth plot and the last 10 months dataframe
-    fig, last_10_months = cumulative_growth_plot_analysis(df, file_df, by_repo_type)
+    fig = cumulative_growth_plot_analysis(cumulative_df, cumulative_df_compressed)
 
     # Add top level heading and introduction text
     gr.Markdown("# Git LFS Usage Across the Hub")
@@ -308,48 +359,53 @@ with gr.Blocks() as demo:
     gr.Markdown(
         "Now, you might ask yourself, 'Why are you doing this?' Well, the [Xet Team](https://huggingface.co/xet-team) is a [new addition to Hugging Face](https://huggingface.co/blog/xethub-joins-hf), bringing a new way to store massive datasets and models to enable ML teams to operate like software teams: Quickly and without friction. Because this story all starts with storage, that's where we've begun with our own deep dives into what the Hub holds. As part of this, we've included a look at what happens with just one simple deduplication strategy - deduplicating at the file level. Read on to see more!"
     )
-    gr.HTML("<div style='height: 20px;'></div>")
+    gr.HTML(div_px(25))
     # Cumulative growth analysis
     gr.Markdown("## Repository Growth")
     with gr.Row():
         gr.Plot(fig)
+
+    gr.HTML(div_px(5))
+    # @TODO Talk to Allison about variant="panel"
     with gr.Row():
         with gr.Column(scale=1):
             gr.Markdown(
-                "This table shows the total number of files and cumulative size of those files across all repositories on the Hub. These numbers might be hard to grok, so let's try to put them in context. The last [Common Crawl](https://commoncrawl.org/) download was [451 TBs](https://github.com/commoncrawl/cc-crawl-statistics/blob/master/stats/crawler/CC-MAIN-2024-38.json#L31). The Spaces repositories alone outpaces that. Meanwhile, between Datasets and Model repos, the Hub stores **64 Common Crawls** 🤯."
+                "This table shows the total number of files, cumulative size of those files across all repositories on the Hub, and the potential file-level dedupe savings. To put this in context, the last [Common Crawl](https://commoncrawl.org/) download was [451 TBs](https://github.com/commoncrawl/cc-crawl-statistics/blob/master/stats/crawler/CC-MAIN-2024-38.json#L31). The Spaces repositories alone outpaces that! Meanwhile, between Datasets and Model repos, the Hub stores **64 Common Crawls** 🤯. Current estimates put total deduplication savings at approximately 3.24 PBs (7.2 Common Crawls)!"
             )
         with gr.Column(scale=3):
             # Convert the total size to petabytes and format to two decimal places
             by_repo_type = format_dataframe_size_column(
-                by_repo_type, "Total Size (PBs)"
-            )
-            by_repo_type = format_dataframe_size_column(
-                by_repo_type, "Compressed Size (PBs)"
-            )
-            by_repo_type = format_dataframe_size_column(
-                by_repo_type, "Dedupe Savings (PBs)"
+                by_repo_type,
+                ["Total Size (PBs)", "Compressed Size (PBs)", "Dedupe Savings (PBs)"],
             )
             gr.Dataframe(by_repo_type)
+
+    gr.HTML(div_px(5))
     with gr.Row():
         with gr.Column(scale=1):
             gr.Markdown(
                 "The cumulative growth of models, spaces, and datasets over time can be seen in the adjacent chart. Beside that is a view of the total change, from the previous month to the current one, of LFS files stored on the hub over 2024. We're averaging nearly **2.3 PBs uploaded to LFS per month!**"
             )
+
+            gr.Markdown(
+                "By the same token, the monthly file deduplication savings are nearly 225TBs. Borrowing from the [Common Crawl](https://commoncrawl.org/) analogy, that's about half a crawl saved each month!"
+            )
         with gr.Column(scale=3):
             gr.Dataframe(last_10_months, height=250)
 
-    gr.HTML("<div style='height: 20px;'></div>")
+    gr.HTML(div_px(25))
     # File Extension analysis
     gr.Markdown("## File Extensions on the Hub")
     gr.Markdown(
-        "Breaking this down by file extension, some interesting trends emerge. [Safetensors](https://huggingface.co/docs/safetensors/en/index) are quickly becoming the defacto standard on the hub, accounting for over 7PBs (25%) of LFS storage. The top 20 file extensions seen here and in the table below account for 82% of all LFS storage on the hub."
+        "Breaking this down by file extension, some interesting trends emerge. The following sections filter the analysis to the top 20 file extensions stored (in bytes) using LFS (which accounts for 82% of storage consumption)."
+    )
+    gr.Markdown(
+        "As is evident in the chart below, [Safetensors](https://huggingface.co/docs/safetensors/en/index) is quickly becoming the defacto standard on the Hub for storing tensor files, accounting for over 7PBs (25%) of LFS storage. If you want to know why you'd want to check out YAF (yet another format), this explanation from the [Safetensors docs](https://github.com/huggingface/safetensors?tab=readme-ov-file#yet-another-format-) is a good place to start. Speaking of YAF, [GGUF (GPT-Generated Unified Format)](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md) is also on the rise, accounting for 3.2 PBs (11%) of LFS storage. GGUF, like Safetensors, is a format for storing tensor files, with a different set of optimizations. The Hub has a few [built-in tools](https://huggingface.co/docs/hub/en/gguf) for working with GGUF."
     )
     # Get the top 10 file extnesions by size
     by_extension_size = by_extension.sort_values(by="size", ascending=False).head(22)
-    # get the top 10 file extensions by count
-    # by_extension_count = by_extension.sort_values(by="count", ascending=False).head(20)
 
-    # make a pie chart of the by_extension_size dataframe
+    # make a bar chart of the by_extension_size dataframe
     gr.Plot(plot_total_sum(by_extension_size[["extension", "size"]].values))
     # drop the unnamed: 0 column
     by_extension_size = by_extension_size.drop(columns=["Unnamed: 0"])
@@ -364,7 +420,7 @@ with gr.Blocks() as demo:
         "Average File Size (MBs)"
     ].map("{:.2f}".format)
     # format the size column
-    by_extension_size = format_dataframe_size_column(by_extension_size, "size")
+    by_extension_size = format_dataframe_size_column(by_extension_size, ["size"])
     # Rename the other columns
     by_extension_size = by_extension_size.rename(
         columns={
@@ -373,24 +429,30 @@ with gr.Blocks() as demo:
             "size": "Total Size (PBs)",
         }
     )
+
+    gr.HTML(div_px(5))
+    gr.Markdown(
+        "Below, we have a more detailed tabular view of the same top 20 file extensions by total size, number of files, and average file size."
+    )
     gr.Dataframe(by_extension_size)
 
-    gr.HTML("<div style='height: 20px;'></div>")
-    gr.Markdown("## File Extension Growth Over Time")
-    by_extension_month["date"] = pd.to_datetime(
-        by_extension_month[["year", "month"]].assign(day=1)
+    gr.HTML(div_px(5))
+    gr.Markdown("### File Extension Monthly Additions (in PBs)")
+    gr.Markdown(
+        "What if we want to see trends over time? The following area chart shows the number of bytes added to LFS storage each month, faceted by the most popular file extensions."
     )
-    # make a plotly area chart with data and extension
-    figure = px.area(by_extension_month, x="date", y="total_size", color="extension")
-    gr.Plot(figure)
+    gr.Plot(area_plot_by_extension_month(by_extension_month))
+
+    gr.HTML(div_px(5))
     gr.Markdown(
-        "Want to dig a little deeper? Select a file extension to see how many bytes of that type were uploaded to the Hub each month."
+        "To dig a little deeper, the following dropdown allows you to filter the area chart by file extension."
     )
 
     # build a dropdown using the unique values in the extension column
     extension = gr.Dropdown(
         choices=by_extension["extension"].unique().tolist(),
         multiselect=True,
+        label="File Extension",
     )
     _by_extension_month = gr.State(by_extension_month)
     gr.Plot(filter_by_extension_month, inputs=[_by_extension_month, extension])