Merge pull request #1 from SaulLu/add-cache

app.py

@@ -1,4 +1,3 @@
-import altair as alt
 import pandas as pd
 import streamlit as st
 import wandb
@@ -7,9 +6,12 @@ from dashboard_utils.bubbles import get_new_bubble_data
 from dashboard_utils.main_metrics import get_main_metrics
 from streamlit_observable import observable
 
+# Only need to set these here as we are add controls outside of Hydralit, to customise a run Hydralit!
+st.set_page_config(page_title="Dashboard", layout="centered")
+
 wandb.login(anonymous="must")
 
-st.title("Training transformers together dashboard")
+st.markdown("<h1 style='text-align: center;'>Dashboard</h1>", unsafe_allow_html=True)
 st.caption("Training Loss")
 
 steps, dates, losses, alive_peers = get_main_metrics()
@@ -58,10 +60,9 @@ st.vega_lite_chart(
 
 st.header("Collaborative training participants")
 serialized_data, profiles = get_new_bubble_data()
-with st.spinner("Wait for it..."):
-    observers = observable(
-        "Participants",
-        notebook="d/9ae236a507f54046",  # "@huggingface/participants-bubbles-chart",
-        targets=["c_noaws"],
-        redefine={"serializedData": serialized_data, "profileSimple": profiles},
-    )
+observable(
+    "Participants",
+    notebook="d/9ae236a507f54046",  # "@huggingface/participants-bubbles-chart",
+    targets=["c_noaws"],
+    redefine={"serializedData": serialized_data, "profileSimple": profiles},
+)

dashboard_utils/bubbles.py

@@ -2,6 +2,7 @@ import datetime
 from concurrent.futures import as_completed
 from urllib import parse
 
+import streamlit as st
 import wandb
 from requests_futures.sessions import FuturesSession
 
@@ -9,23 +10,31 @@ from dashboard_utils.time_tracker import _log, simple_time_tracker
 
 URL_QUICKSEARCH = "https://huggingface.co/api/quicksearch?"
 WANDB_REPO = "learning-at-home/Worker_logs"
+CACHE_TTL = 100
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_new_bubble_data():
     serialized_data_points, latest_timestamp = get_serialized_data_points()
     serialized_data = get_serialized_data(serialized_data_points, latest_timestamp)
-    profiles = get_profiles(serialized_data_points)
+
+    usernames = []
+    for item in serialized_data["points"][0]:
+        usernames.append(item["profileId"])
+
+    profiles = get_profiles(usernames)
 
     return serialized_data, profiles
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
-def get_profiles(serialized_data_points):
+def get_profiles(usernames):
     profiles = []
     with FuturesSession() as session:
         futures = []
-        for username in serialized_data_points.keys():
+        for username in usernames:
             future = session.get(URL_QUICKSEARCH + parse.urlencode({"type": "user", "q": username}))
             future.username = username
             futures.append(future)
@@ -51,6 +60,7 @@ def get_profiles(serialized_data_points):
     return profiles
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_serialized_data_points():
 
@@ -98,6 +108,7 @@ def get_serialized_data_points():
     return serialized_data_points, latest_timestamp
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_serialized_data(serialized_data_points, latest_timestamp):
     serialized_data_points_v2 = []

dashboard_utils/main_metrics.py

@@ -1,12 +1,15 @@
 import datetime
 
+import streamlit as st
 import wandb
 
 from dashboard_utils.time_tracker import _log, simple_time_tracker
 
 WANDB_REPO = "learning-at-home/Main_metrics"
+CACHE_TTL = 100
 
 
+@st.cache(ttl=CACHE_TTL)
 @simple_time_tracker(_log)
 def get_main_metrics():
     api = wandb.Api()

perso/change_data.py

@@ -0,0 +1,19 @@
+import json
+import random
+
+with open(
+    "/mnt/storage/Documents/hugging_face/colaborative_hub_training/demo_neurips/training-transformers-together-dashboard/data/"
+    "serializaledata.json",
+    "r",
+) as f:
+    serialized_data = json.load(f)
+
+serialized_data_v2 = serialized_data
+serialized_data_v2["points"] = [[item for item in serialized_data["points"][-1] if random.random() > 0.8]]
+
+with open(
+    "/mnt/storage/Documents/hugging_face/colaborative_hub_training/demo_neurips/training-transformers-together-dashboard/data/"
+    "serializaledata_V2.json",
+    "w",
+) as f:
+    f.write(json.dumps(serialized_data_v2))

perso/get_usernames.py

@@ -0,0 +1,14 @@
+import json
+
+with open(
+    "/mnt/storage/Documents/hugging_face/colaborative_hub_training/demo_neurips/training-transformers-together-dashboard/data/"
+    "serializaledata_V2.json",
+    "r",
+) as f:
+    serialized_data = json.load(f)
+
+usernames = []
+for item in serialized_data["points"][0]:
+    usernames.append(item["profileId"])
+
+print(usernames)