nits

src/logic/data_fetching.py

@@ -1,4 +1,4 @@
-from functools import partial
+from functools import lru_cache, partial
 import os
 import json
 import re
@@ -17,19 +17,17 @@ def find_folders(base_folder: str, path: str) -> List[str]:
     base_folder_df = get_datafolder(base_folder)
     if not base_folder_df.exists(path):
         return []
-    return sorted(
-        [
+    return [
             folder
             for folder,info in base_folder_df.find(path, maxdepth=1, withdirs=True, detail=True).items()
             if info["type"] == "directory" and not (folder.rstrip("/") == path.rstrip("/"))
         ]
-    )
 
-def fetch_datasets(base_folder: str):
-    datasets = sorted(find_folders(base_folder, ""))
+def fetch_datasets(base_folder: str, progress=gr.Progress()):
+    datasets = sorted(progress.tqdm(find_folders(base_folder, "")))
     if len(datasets) == 0:
         raise ValueError("No datasets found")
-    return datasets
+    return datasets, None
 
 def fetch_groups(base_folder: str, datasets: List[str], old_groups: str, type: str = "intersection"):
     if not datasets:

src/view/help_tab.py

@@ -7,10 +7,10 @@ def create_help_tab():
 
 # Dataset Metrics Explorer
 ## Features:
-- View metrics for various datasets you computed using datatrove
-- Search for metrics across datasets
+- Inspect datasets throught various metrics computed using datatrove
+- Search for datasets containing certain metrics
 
-## View metrics Usage:
+## Metrics View Usage:
 1) Specify Metrics location (Stats block `output_folder`) and click "Fetch Datasets"
 2) Select datasets you are interested in using the dropdown or regex filter
 3) Specify Grouping (histogram/summary/fqdn/suffix) and Metric name
@@ -27,7 +27,7 @@ def create_help_tab():
 - **summary**: Shows the average value of given metric for every dataset
     * show_stds: Show the standard deviation from mean for every datasets
 
-## Reverse search Usage:
+## Reverse Metrics Search Usage:
 To search for datasets containing a grouping and certain metric, use the Reverse search section.
 Specify the search parameters and click "Search". This will show you found datasets in the "Found datasets" textbox. You can modify the selection after search by removing unwanted lines and clicking "Add to selection".
 

src/view/metric_view_tab.py

@@ -6,12 +6,12 @@ from functools import partial
 import re
 import json
 
-from src.logic.data_fetching import fetch_datasets, fetch_graph_data, fetch_groups, fetch_metrics, update_datasets_with_regex
+from src.logic.data_fetching import fetch_datasets, fetch_graph_data, fetch_groups, fetch_metrics, update_datasets_with_regex, update_datasets_with_regex
 from src.logic.data_processing import export_data
 from src.logic.graph_settings import update_graph_options
 from src.logic.plotting import plot_data
 
-def create_metric_view_tab(METRICS_LOCATION_DEFAULT: str, available_datasets: gr.State, selected_datasets: gr.State):
+def create_metric_view_tab(METRICS_LOCATION_DEFAULT: str, available_datasets: gr.State):
     metric_data = gr.State([])
 
     with gr.Row():
@@ -120,53 +120,40 @@ def create_metric_view_tab(METRICS_LOCATION_DEFAULT: str, available_datasets: gr
             
 
 
-    def update_selected_datasets_dropdown(available_datasets, selected_datasets):
-        return gr.Dropdown(choices=available_datasets, value=sorted(selected_datasets))
+    def update_selected_datasets_dropdown(available_datasets, selected_datasets_dropdown):
+        selected_datasets = selected_datasets_dropdown or []
+        selected_datasets = set(selected_datasets) & set(available_datasets)
+        return gr.Dropdown(choices=available_datasets, value=sorted(list(selected_datasets)))
 
 
     datasets_fetch.click(
         fn=fetch_datasets,
         inputs=[base_folder],
-        outputs=[available_datasets],
+        outputs=[available_datasets, selected_datasets_dropdown],
     )
 
     available_datasets.change(
         fn=update_selected_datasets_dropdown,
-        inputs=[available_datasets, selected_datasets],
+        inputs=[available_datasets, selected_datasets_dropdown],
         outputs=selected_datasets_dropdown,
     )
 
     regex_button.click(
         fn=update_datasets_with_regex,
-        inputs=[regex_select, selected_datasets, available_datasets],
-        outputs=selected_datasets,
+        inputs=[regex_select, selected_datasets_dropdown, available_datasets],
+        outputs=selected_datasets_dropdown,
     )
     
-    def update_selected_datasets(selected_datasets_dropdown):
-        return selected_datasets_dropdown
     
     selected_datasets_dropdown.change(
-        fn=update_selected_datasets,
-        inputs=[selected_datasets_dropdown],
-        outputs=selected_datasets,
-    )
-    
-    selected_datasets.change(
-        fn=update_selected_datasets_dropdown,
-        inputs=[available_datasets, selected_datasets],
-        outputs=selected_datasets_dropdown,
-    )
-
-
-    selected_datasets.change(
         fn=fetch_groups,
-        inputs=[base_folder, selected_datasets, grouping_dropdown],
+        inputs=[base_folder, selected_datasets_dropdown, grouping_dropdown],
         outputs=grouping_dropdown,
     )
 
     grouping_dropdown.change(
         fn=fetch_metrics,
-        inputs=[base_folder, selected_datasets, grouping_dropdown, metric_name_dropdown],
+        inputs=[base_folder, selected_datasets_dropdown, grouping_dropdown, metric_name_dropdown],
         outputs=metric_name_dropdown,
     )
 
@@ -174,7 +161,7 @@ def create_metric_view_tab(METRICS_LOCATION_DEFAULT: str, available_datasets: gr
         fn=fetch_graph_data,
         inputs=[
             base_folder,
-            selected_datasets,
+            selected_datasets_dropdown,
             metric_name_dropdown,
             grouping_dropdown,
         ],
@@ -219,4 +206,4 @@ def create_metric_view_tab(METRICS_LOCATION_DEFAULT: str, available_datasets: gr
         outputs=[export_data_json],
     )
     
-    return base_folder
+    return base_folder, selected_datasets_dropdown

src/view/reverse_search_tab.py

@@ -3,7 +3,7 @@ import gradio as gr
 
 from src.logic.data_fetching import fetch_groups, fetch_metrics, reverse_search, reverse_search_add
 
-def create_reverse_search_tab(base_folder: gr.Textbox, datasets_available: gr.State, datasets_selected: gr.State):
+def create_reverse_search_tab(base_folder: gr.Textbox, datasets_available: gr.State, datasets_selected: gr.Dropdown):
     reverse_search_headline = gr.Markdown(value="# Reverse Metrics Search")
 
     with gr.Row():

src/view/view.py

@@ -11,16 +11,15 @@ METRICS_LOCATION_DEFAULT = os.getenv("METRICS_LOCATION_DEFAULT", "hf://datasets/
 
 def create_interface():
     with gr.Blocks() as demo:
-        metrics_headline = gr.Markdown(value="# Metrics Exploration")
+        metrics_headline = gr.Markdown(value="# Datasets Metrics Explorer")
         available_datasets = gr.State([])
-        selected_datasets = gr.State([])
 
         with gr.Tabs():
             with gr.Tab("Help"):
                 create_help_tab()
             
             with gr.TabItem("Metric View"):
-                base_folder = create_metric_view_tab(METRICS_LOCATION_DEFAULT, available_datasets, selected_datasets)
+                base_folder, selected_datasets = create_metric_view_tab(METRICS_LOCATION_DEFAULT, available_datasets)
 
             with gr.TabItem("Reverse Metrics Search"):
                 create_reverse_search_tab(base_folder, available_datasets, selected_datasets)