add cache

app.py

@@ -1,4 +1,3 @@
-import altair as alt
 import pandas as pd
 import streamlit as st
 import wandb
@@ -7,9 +6,12 @@ from dashboard_utils.bubbles import get_new_bubble_data
 from dashboard_utils.main_metrics import get_main_metrics
 from streamlit_observable import observable
 
+# Only need to set these here as we are add controls outside of Hydralit, to customise a run Hydralit!
+st.set_page_config(page_title="Dashboard", layout="centered")
+
 wandb.login(anonymous="must")
 
-st.title("Training transformers together dashboard")
+st.markdown("<h1 style='text-align: center;'>Dashboard</h1>", unsafe_allow_html=True)
 st.caption("Training Loss")
 
 steps, dates, losses, alive_peers = get_main_metrics()
@@ -58,10 +60,9 @@ st.vega_lite_chart(
 
 st.header("Collaborative training participants")
 serialized_data, profiles = get_new_bubble_data()
-with st.spinner("Wait for it..."):
-    observers = observable(
-        "Participants",
-        notebook="d/9ae236a507f54046",  # "@huggingface/participants-bubbles-chart",
-        targets=["c_noaws"],
-        redefine={"serializedData": serialized_data, "profileSimple": profiles},
-    )
+observable(
+    "Participants",
+    notebook="d/9ae236a507f54046",  # "@huggingface/participants-bubbles-chart",
+    targets=["c_noaws"],
+    redefine={"serializedData": serialized_data, "profileSimple": profiles},
+)

dashboard_utils/bubbles.py

@@ -1,8 +1,8 @@
 import datetime
-import json
 from concurrent.futures import as_completed
 from urllib import parse
 
+import streamlit as st
 import wandb
 from requests_futures.sessions import FuturesSession
 
@@ -10,12 +10,13 @@ from dashboard_utils.time_tracker import _log, simple_time_tracker
 
 URL_QUICKSEARCH = "https://huggingface.co/api/quicksearch?"
 WANDB_REPO = "learning-at-home/Worker_logs"
+CACHE_TTL = 100
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_new_bubble_data():
-    # serialized_data_points, latest_timestamp = get_serialized_data_points()
-    serialized_data_points, latest_timestamp = None, None
+    serialized_data_points, latest_timestamp = get_serialized_data_points()
     serialized_data = get_serialized_data(serialized_data_points, latest_timestamp)
 
     usernames = []
@@ -27,6 +28,7 @@ def get_new_bubble_data():
     return serialized_data, profiles
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_profiles(usernames):
     profiles = []
@@ -58,6 +60,7 @@ def get_profiles(usernames):
     return profiles
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_serialized_data_points():
 
@@ -105,38 +108,33 @@ def get_serialized_data_points():
     return serialized_data_points, latest_timestamp
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_serialized_data(serialized_data_points, latest_timestamp):
-    # serialized_data_points_v2 = []
-    # max_velocity = 1
-    # for run_name, serialized_data_point in serialized_data_points.items():
-    #     activeRuns = []
-    #     loss = 0
-    #     runtime = 0
-    #     batches = 0
-    #     velocity = 0
-    #     for run in serialized_data_point["Runs"]:
-    #         if run["date"] == latest_timestamp:
-    #             run["date"] = run["date"].isoformat()
-    #             activeRuns.append(run)
-    #             loss += run["loss"]
-    #             velocity += run["velocity"]
-    #         loss = loss / len(activeRuns) if activeRuns else 0
-    #         runtime += run["runtime"]
-    #         batches += run["batches"]
-    #     new_item = {
-    #         "date": latest_timestamp.isoformat(),
-    #         "profileId": run_name,
-    #         "batches": batches,
-    #         "runtime": runtime,
-    #         "activeRuns": activeRuns,
-    #     }
-    #     serialized_data_points_v2.append(new_item)
-    # serialized_data = {"points": [serialized_data_points_v2], "maxVelocity": max_velocity}
-    with open(
-        "/mnt/storage/Documents/hugging_face/colaborative_hub_training/demo_neurips/training-transformers-together-dashboard/data/"
-        "serializaledata_V2.json",
-        "r",
-    ) as f:
-        serialized_data = json.load(f)
+    serialized_data_points_v2 = []
+    max_velocity = 1
+    for run_name, serialized_data_point in serialized_data_points.items():
+        activeRuns = []
+        loss = 0
+        runtime = 0
+        batches = 0
+        velocity = 0
+        for run in serialized_data_point["Runs"]:
+            if run["date"] == latest_timestamp:
+                run["date"] = run["date"].isoformat()
+                activeRuns.append(run)
+                loss += run["loss"]
+                velocity += run["velocity"]
+            loss = loss / len(activeRuns) if activeRuns else 0
+            runtime += run["runtime"]
+            batches += run["batches"]
+        new_item = {
+            "date": latest_timestamp.isoformat(),
+            "profileId": run_name,
+            "batches": batches,
+            "runtime": runtime,
+            "activeRuns": activeRuns,
+        }
+        serialized_data_points_v2.append(new_item)
+    serialized_data = {"points": [serialized_data_points_v2], "maxVelocity": max_velocity}
     return serialized_data

dashboard_utils/main_metrics.py

@@ -1,12 +1,15 @@
 import datetime
 
+import streamlit as st
 import wandb
 
 from dashboard_utils.time_tracker import _log, simple_time_tracker
 
 WANDB_REPO = "learning-at-home/Main_metrics"
+CACHE_TTL = 100
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_main_metrics():
     api = wandb.Api()