adding option to show all submissions

app.py

@@ -107,14 +107,23 @@ def make_heatmap(results, label="generated", symbol="👤"):
 
     return chart
 
+@st.cache_data
+def load_roc_file(task, submission_ids):
+    rocs = pd.read_csv(f"{results_path}/{task}_rocs.csv")
+    # if best_only:
+    rocs = rocs[rocs["submission_id"].isin(submission_ids)]
+    return rocs
+@st.cache_data
+def get_unique_teams(teams):
+    return teams.unique().tolist()
 
 @st.cache_data
-def make_roc_curves(task, submission_cols, best_only=True):
+def filter_teams(temp, selected_team):
+    return temp.query(f"team=='{selected_team}'")
 
-    rocs = pd.read_csv(f"{results_path}/{task}_rocs.csv")
+def make_roc_curves(task, submission_ids):
 
-    if best_only:
-        rocs = rocs[rocs["submission_id"].isin(submission_cols)]
+    rocs = load_roc_file(task, submission_ids)
 
     roc_chart = alt.Chart(rocs).mark_line().encode(x="fpr", y="tpr", color="team:N", detail="submission_id:N")
 
@@ -380,6 +389,7 @@ def make_roc(results):
             size=alt.Size(
                 "total_time:Q", title="🕒 Inference Time", scale=alt.Scale(rangeMin=100)
             ),  # Size by quantitative field
+            detail=["submission_id","auc","balanced_accuracy"]
         )
         .properties(width=400, height=400, title="Detection vs False Alarm vs Inference Time")
     )
@@ -400,7 +410,7 @@ def make_acc(results):
         alt.Chart(results)
         .mark_circle(size=200)
         .encode(
-            x=alt.X("total_time:Q", title="🕒 Inference Time", scale=alt.Scale(domain=[0.0, 10000])),
+            x=alt.X("total_time:Q", title="🕒 Inference Time", scale=alt.Scale(type = "log")),
             y=alt.Y(
                 "balanced_accuracy:Q",
                 title="Balanced Accuracy",
@@ -411,7 +421,7 @@ def make_acc(results):
         .properties(width=400, height=400, title="Inference Time vs Balanced Accuracy")
     )
     diag_line = (
-        alt.Chart(pd.DataFrame(dict(t=[0, results["total_time"].max()], y=[0.5, 0.5])))
+        alt.Chart(pd.DataFrame(dict(t=[100, results["total_time"].max()], y=[0.5, 0.5])))
         .mark_line(color="lightgray", strokeDash=[8, 4])
         .encode(x="t", y="y")
     )
@@ -431,15 +441,31 @@ def get_heatmaps(temp):
 
 
 def make_plots_for_task(task, split, best_only):
+    
+                
     results = load_results(task, best_only=best_only)
     temp = results[f"{split}_score"].reset_index()
+    teams = get_unique_teams(temp["team"])
+
 
     t1, t2 = st.tabs(["Tables", "Charts"])
     with t1:
         show_leaderboard(results, task)
 
     with t2:
-
+        # st.write(temp)
+        if split == "private":
+            best_only = st.toggle("Best Only", value=True, key = f"best only {task}")
+            if not best_only:
+                results = load_results(task, best_only=best_only)
+                temp = results[f"{split}_score"].reset_index()
+            selected_team = st.pills("Team",["ALL"] + teams, key = f"teams {task}", default="ALL")
+            if not selected_team:
+                selected_team = "ALL"
+            if selected_team != "ALL":
+                temp = filter_teams(temp, selected_team)
+
+        # with st.spinner("making plots...", show_time=True):
         roc_scatter = make_roc(temp)
         acc_vs_time = make_acc(temp)
 
@@ -447,12 +473,15 @@ def make_plots_for_task(task, split, best_only):
             full_curves = st.toggle("Full curve", value=True, key=f"all curves {task}")
 
             if full_curves:
-                roc_scatter = make_roc_curves(task, temp["submission_id"].values.tolist(), best_only) + roc_scatter
+                roc_scatter = make_roc_curves(task, temp["submission_id"].values.tolist()) + roc_scatter
 
             st.altair_chart(roc_scatter | acc_vs_time, use_container_width=False)
         else:
             st.altair_chart(roc_scatter | acc_vs_time, use_container_width=False)
 
+        st.info(f"loading {temp["submission_id"].nunique()} submissioms")
+
+
 
 updated = get_updated_time()
 st.markdown(updated)

test.sh

@@ -1 +1,2 @@
-HF_TOKEN=$(cat ~/.cache/huggingface/token) streamlit run app.py
+# HF_TOKEN=$(cat ~/.cache/huggingface/token) streamlit run app.py
+HF_TOKEN=test streamlit run app.py

utils.py

@@ -137,8 +137,12 @@ def extract_roc(results: Dict[str, Any]) -> Dict[str, Any]:
 if __name__ == "__main__":
 
     ## Download data
-    spaces: List[str] = ["safe-challenge/video-challenge-pilot-config", "safe-challenge/video-challenge-task-1-config"]
-    download_competition_data(competition_names=spaces)
+    # spaces: List[str] = ["safe-challenge/video-challenge-pilot-config", "safe-challenge/video-challenge-task-1-config"]
+    spaces: List[str] = ["safe-challenge/video-challenge-task-1-config"]
+
+    # download_competition_data(competition_names=spaces)
+
+
 
     ## Loop
     for space in spaces:
@@ -155,7 +159,26 @@ if __name__ == "__main__":
 
         ## Loop and save by team
         public, private, rocs = [], [], []
-        for team_id, submission_set in submissions.items():
+        # for team_id, submission_set in submissions.items():
+        for team_id, submission_set_ids in submission_summaries.query("status_reason=='SUCCESS'").groupby("team_id")["submission_id"]:
+            ### lets check if we have the solution csvs
+            submission_set = submissions[team_id]
+            submission_set_ids_from_csvs = set(submission_set.keys())
+            submission_set_ids = set(submission_set_ids)
+            union = submission_set_ids | submission_set_ids_from_csvs
+
+            if not (submission_set_ids.issubset(submission_set_ids_from_csvs)):
+                # intersction = set(submission_set_ids) & submission_set_ids_from_csvs
+                missing = union - submission_set_ids_from_csvs
+                print(f"not all submission csv files found for {team_id}, missing {len(missing)}")
+
+            if submission_set_ids != submission_set_ids_from_csvs:
+                extra = union - submission_set_ids
+                print(f"extra {len(extra)} submissions in csvs than in summary file for team {team_id}")
+                print(f"dropping {extra}")
+                for submission_id in extra:
+                    submission_set.pop(submission_id)
+            
             results = compute_metric_per_team(solution_df=solutions_df, team_submissions=submission_set)
             public_results = {
                 key: prep_public(value["public_score"]) for key, value in results.items() if key in team_submissions
@@ -289,9 +312,9 @@ if __name__ == "__main__":
             private.append(private_df)
 
         ## Save as csvs
-        public = pd.concat(public, axis=0).sort_values(by="balanced_accuracy", ascending=False)
-        private = pd.concat(private, axis=0).sort_values(by="balanced_accuracy", ascending=False)
-        rocs = pd.concat(rocs, axis=0).explode(["tpr", "fpr", "threshold"], ignore_index=True)
+        public = pd.concat(public, axis=0,ignore_index=True).sort_values(by="balanced_accuracy", ascending=False)
+        private = pd.concat(private, axis=0,ignore_index=True).sort_values(by="balanced_accuracy", ascending=False)
+        rocs = pd.concat(rocs, axis=0,ignore_index=True).explode(["tpr", "fpr", "threshold"], ignore_index=True)
         public.to_csv(
             Path("competition_cache") / "cached_results" / f"{str(local_dir).split('/')[-1]}_public_score.csv",
             index=False,