Update app.py

app.py

@@ -1,15 +1,20 @@
 import gradio as gr
 import pandas as pd
-from huggingface_hub import HfApi
+import requests
 
-# Initialize Hugging Face API
-api = HfApi()
-
-# Constants
-GGUF_TAG = "gguf"
 CHUNK_SIZE = 1000
+DATA_URL = "https://erkhov.com/huggingspace_data"
+TIME_URL = "https://erkhov.com/huggingspace_time"
+
+def fetch_time():
+    response = requests.get(TIME_URL)
+    return response.text.strip()  # assume the endpoint returns a raw timestamp or numeric value as string
+
+def fetch_data():
+    response = requests.get(DATA_URL)
+    data = response.json()
+    return data
 
-# Clickable links function
 def clickable(x, which_one):
     if x in ["Not Found", "Unknown"]:
         return "Not Found"
@@ -18,55 +23,40 @@ def clickable(x, which_one):
     else:
         return f'<a target="_blank" href="https://huggingface.co/{which_one}/{x}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{x}</a>'
 
-# Fetch models and return a DataFrame with clickable links
-def fetch_models():
-    models = api.list_models(filter=GGUF_TAG, full=True)
-    data = []
-    for model in models:
-        model_id = model.id if model.id else "Not Found"
-        author = model.author if model.author else "Unknown"
-        data.append({
-            "Model ID": model_id,
-            "Author Name": author,
-            "Downloads (30d)": model.downloads or 0,
-            "Likes": model.likes or 0,
-            "Created At": model.created_at.isoformat() if model.created_at else "N/A",
-            "Last Modified": model.last_modified.isoformat() if model.last_modified else "N/A",
-        })
-    df = pd.DataFrame(data)
-    # Apply clickable links to models and authors
-    df["Model ID"] = df["Model ID"].apply(lambda x: clickable(x, "models"))
-    df["Author Name"] = df["Author Name"].apply(lambda x: clickable(x, "models"))
-    return df
-
-# Prepare authors DataFrame
-def prepare_authors_df(models_df):
-    authors_df = models_df.copy()
-    # Extract the author name from the href in the clickable link
-    authors_df["Clean Author Name"] = authors_df["Author Name"].str.extract(r'href="https://huggingface\.co/(.*?)"')
-
-    grouped = authors_df.groupby("Clean Author Name").agg(
-        Models_Count=("Model ID", "count"),
-        Total_Downloads=("Downloads (30d)", "sum"),
-        Total_Likes=("Likes", "sum")
-    ).reset_index()
-
-    grouped.rename(columns={"Clean Author Name": "Author Name"}, inplace=True)
-    grouped["Author Name"] = grouped["Author Name"].apply(lambda x: clickable(x, "models"))
-    return grouped.sort_values(by="Models_Count", ascending=False)
-
-all_models_df = fetch_models().sort_values(by="Downloads (30d)", ascending=False)
-authors_df = prepare_authors_df(all_models_df)
-
-# Calculate totals
-total_models_count = len(all_models_df)
-total_downloads = all_models_df["Downloads (30d)"].sum()
-total_likes = all_models_df["Likes"].sum()
-
-def apply_model_filters(search_query, min_downloads, min_likes):
-    df = all_models_df.copy()
-
-    # Extract visible text for filtering purposes:
+def create_dataframes(data):
+    models_data = data["models"]
+    authors_data = data["authors"]
+    
+    # Create DataFrame for models
+    models_df = pd.DataFrame(models_data)
+    models_df.rename(columns={
+        "id": "Model ID",
+        "author": "Author Name",
+        "downloads": "Downloads (30d)",
+        "likes": "Likes",
+        "created_at": "Created At",
+        "last_modified": "Last Modified"
+    }, inplace=True)
+    models_df["Model ID"] = models_df["Model ID"].apply(lambda x: clickable(x, "models"))
+    models_df["Author Name"] = models_df["Author Name"].apply(lambda x: clickable(x, "models"))
+    models_df = models_df.sort_values(by="Downloads (30d)", ascending=False)
+
+    # Create DataFrame for authors
+    authors_df = pd.DataFrame(authors_data)
+    authors_df.rename(columns={
+        "author": "Author Name",
+        "models_count": "Models_Count",
+        "downloads": "Total_Downloads",
+        "likes": "Total_Likes"
+    }, inplace=True)
+    authors_df["Author Name"] = authors_df["Author Name"].apply(lambda x: clickable(x, "models"))
+    authors_df = authors_df.sort_values(by="Models_Count", ascending=False)
+
+    return models_df, authors_df
+
+def apply_model_filters(models_df, search_query, min_downloads, min_likes):
+    df = models_df.copy()
+    # Extract visible text for filtering
     visible_model_id = df["Model ID"].str.extract(r'>(.*?)<')[0]
     visible_author_name = df["Author Name"].str.extract(r'>(.*?)<')[0]
 
@@ -86,8 +76,8 @@ def apply_model_filters(search_query, min_downloads, min_likes):
 
     return df
 
-def filter_models(search_query, min_downloads, min_likes):
-    filtered = apply_model_filters(search_query, min_downloads, min_likes)
+def filter_models(models_df, search_query, min_downloads, min_likes):
+    filtered = apply_model_filters(models_df, search_query, min_downloads, min_likes)
     return filtered.iloc[:CHUNK_SIZE], CHUNK_SIZE, filtered
 
 def update_model_table(start_idx, filtered_df):
@@ -95,7 +85,7 @@ def update_model_table(start_idx, filtered_df):
     combined_df = filtered_df.iloc[:new_end].copy()
     return combined_df, new_end
 
-def apply_author_filters(search_query, min_author_downloads, min_author_likes):
+def apply_author_filters(authors_df, search_query, min_author_downloads, min_author_likes):
     df = authors_df.copy()
 
     # Extract visible text for author filtering:
@@ -116,22 +106,62 @@ def apply_author_filters(search_query, min_author_downloads, min_author_likes):
 
     return df
 
+def filter_authors(authors_df, author_search_query, min_author_downloads, min_author_likes):
+    filtered_authors = apply_author_filters(authors_df, author_search_query, min_author_downloads, min_author_likes)
+    return filtered_authors
+
+def refresh_data(last_time, models_df_state, authors_df_state, stats_markdown, model_table, author_table):
+    # Check if time changed
+    current_time = fetch_time()
+    if current_time != last_time and current_time != 0:
+        # Time changed, re-fetch data
+        data = fetch_data()
+        models_df, authors_df = create_dataframes(data)
+        
+        total_models_count = data["total_models"]
+        total_downloads = data["total_downloads"]
+        total_likes = models_df["Likes"].sum() if "Likes" in models_df.columns else 0
+        
+        # Update stats markdown
+        new_stats_markdown = f"""
+        # GGUF Models and Authors Leaderboard
+        **Total Models:** {total_models_count} | **Total Downloads (30d):** {total_downloads} | **Total Likes:** {total_likes}  
+        **Last Updated:** {current_time}
+        """
+        
+        # Update states
+        return current_time, models_df, authors_df, gr.update(value=new_stats_markdown), gr.update(value=models_df.iloc[:CHUNK_SIZE]), gr.update(value=authors_df)
+    else:
+        # No change
+        return last_time, models_df_state, authors_df_state, stats_markdown, model_table, author_table
+
+# Initial fetch
+initial_time = fetch_time()
+data = fetch_data()
+all_models_df, authors_df = create_dataframes(data)
+
+total_models_count = data["total_models"]
+total_downloads = data["total_downloads"]
+total_likes = all_models_df["Likes"].sum() if "Likes" in all_models_df.columns else 0
+
+initial_stats_markdown = f"""
+# GGUF Models and Authors Leaderboard
+**Total Models:** {total_models_count} | **Total Downloads (30d):** {total_downloads} | **Total Likes:** {total_likes}  
+**Last Updated:** {initial_time}
+"""
 
 with gr.Blocks() as demo:
     gr.Markdown(f"""
     # 🚀GGUF Tracker🚀
     Welcome to 🚀**GGUF Tracker**🚀, a live-updating leaderboard for all things GGUF on 🚀Hugging Face.  
-    Stats refresh every hour, giving you the latest numbers.
+    Stats refresh every hour on the backend, but this interface checks every 5 minutes for updates.
 
     By the way, I’m 🚀Richard Erkhov, and you can check out more of what I’m working on at my [🌟**github**](https://github.com/RichardErkhov), 
     [🌟**huggingface**](https://huggingface.co/RichardErkhov) or [🌟**erkhov.com**](https://erkhov.com). Go take a look—I think you’ll like what you find.
     """)
 
-    gr.Markdown(f"""
-    # GGUF Models and Authors Leaderboard
-    **Total Models:** {total_models_count} | **Total Downloads (30d):** {total_downloads} | **Total Likes:** {total_likes}
-    """)
-
+    stats_markdown = gr.Markdown(initial_stats_markdown)
+    
     with gr.Tabs():
         with gr.TabItem("Models"):
             with gr.Row():
@@ -149,13 +179,13 @@ with gr.Blocks() as demo:
             )
             load_more_button = gr.Button("Load More Models")
             
-            # States
+            # States for models
             start_idx = gr.State(value=CHUNK_SIZE)
-            filtered_df_state = gr.State(value=all_models_df)  # holds the currently filtered df
+            filtered_df_state = gr.State(value=all_models_df)
 
             filter_button.click(
                 fn=filter_models,
-                inputs=[search_query, min_downloads, min_likes],
+                inputs=[filtered_df_state, search_query, min_downloads, min_likes],
                 outputs=[model_table, start_idx, filtered_df_state]
             )
             load_more_button.click(fn=update_model_table, inputs=[start_idx, filtered_df_state], outputs=[model_table, start_idx])
@@ -175,14 +205,32 @@ with gr.Blocks() as demo:
                 datatype=["markdown", "number", "number", "number"]
             )
 
-            def filter_authors(author_search_query, min_author_downloads, min_author_likes):
-                filtered_authors = apply_author_filters(author_search_query, min_author_downloads, min_author_likes)
-                return filtered_authors
-
             author_filter_button.click(
                 fn=filter_authors,
-                inputs=[author_search_query, min_author_downloads, min_author_likes],
+                inputs=[authors_df, author_search_query, min_author_downloads, min_author_likes],
                 outputs=author_table
             )
 
+    # States for refresh
+    last_hf_time_state = gr.State(value=initial_time)
+    models_df_state = gr.State(value=all_models_df)
+    authors_df_state = gr.State(value=authors_df)
+
+    # Timer to check every 5 minutes = 300 seconds
+    timer = gr.Timer(interval=300, fn=refresh_data, inputs=[
+        last_hf_time_state,
+        models_df_state,
+        authors_df_state,
+        stats_markdown,
+        model_table,
+        author_table
+    ], outputs=[
+        last_hf_time_state,
+        models_df_state,
+        authors_df_state,
+        stats_markdown,
+        model_table,
+        author_table
+    ])
+
     demo.launch()