add leaderboard ui and refactor code

app.py

@@ -5,11 +5,13 @@
 import gradio as gr
 
 from app_text_classification import get_demo as get_demo_text_classification
-
+from app_leaderboard import get_demo as get_demo_leaderboard
 
 with gr.Blocks(theme=gr.themes.Soft(primary_hue="green")) as demo:
     with gr.Tab("Text Classification"):
         get_demo_text_classification()
-    with gr.Tab("Leaderboard - Text Classification"):
-        pass
-demo.launch()
+    with gr.Tab("Leaderboard"):
+        get_demo_leaderboard()
+
+demo.queue(max_size=100)
+demo.launch(share=False)

app_leaderboard.py

@@ -0,0 +1,95 @@
+import gradio as gr
+import datasets
+import logging
+from fetch_utils import check_dataset_and_get_config, check_dataset_and_get_split
+
+def get_records_from_dataset_repo(dataset_id):
+    dataset_config = check_dataset_and_get_config(dataset_id)
+    logging.info(f"Dataset {dataset_id} has configs {dataset_config}")
+    dataset_split = check_dataset_and_get_split(dataset_id, dataset_config[0])
+    logging.info(f"Dataset {dataset_id} has splits {dataset_split}")
+    try:
+        ds = datasets.load_dataset(dataset_id, dataset_config[0])[dataset_split[0]]
+        df = ds.to_pandas()
+        return df
+    except Exception as e:
+        logging.warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
+        return None
+
+def get_model_ids(ds):
+    logging.info(f"Dataset {ds} column names: {ds['model_id']}")
+    models = ds['model_id'].tolist()
+    # return unique elements in the list model_ids
+    model_ids = list(set(models))
+    return model_ids
+
+def get_dataset_ids(ds):
+    logging.info(f"Dataset {ds} column names: {ds['dataset_id']}")
+    datasets = ds['dataset_id'].tolist()
+    dataset_ids = list(set(datasets))
+    return dataset_ids
+
+def get_types(ds):
+    # set all types for each column
+    types = [str(t) for t in ds.dtypes.to_list()]
+    types = [t.replace('object', 'markdown') for t in types]
+    types = [t.replace('float64', 'number') for t in types]
+    types = [t.replace('int64', 'number') for t in types]
+    return types
+
+def get_display_df(df):
+    # style all elements in the model_id column
+    display_df = df.copy()
+    if display_df['model_id'].any():
+        display_df['model_id'] = display_df['model_id'].apply(lambda x: f'<p href="https://huggingface.co/{x}" style="color:blue">🔗{x}</p>')
+    # style all elements in the dataset_id column
+    if display_df['dataset_id'].any():
+        display_df['dataset_id'] = display_df['dataset_id'].apply(lambda x: f'<p href="https://huggingface.co/datasets/{x}" style="color:blue">🔗{x}</p>')
+    # style all elements in the report_link column
+    if display_df['report_link'].any():
+        display_df['report_link'] = display_df['report_link'].apply(lambda x: f'<p href="{x}" style="color:blue">🔗{x}</p>')
+    return display_df
+
+def get_demo():
+    records = get_records_from_dataset_repo('ZeroCommand/test-giskard-report')
+
+    model_ids = get_model_ids(records)
+    dataset_ids = get_dataset_ids(records)
+
+    column_names = records.columns.tolist()
+    default_columns = ['model_id', 'dataset_id', 'total_issue', 'report_link']
+    # set the default columns to show
+    default_df = records[default_columns]
+    types = get_types(default_df)
+    display_df = get_display_df(default_df)
+
+    with gr.Row():
+        task_select = gr.Dropdown(label='Task', choices=['text_classification', 'tabular'], value='text_classification', interactive=True)
+        model_select = gr.Dropdown(label='Model id', choices=model_ids, interactive=True)
+        dataset_select = gr.Dropdown(label='Dataset id', choices=dataset_ids, interactive=True)
+    
+    with gr.Row():
+        columns_select = gr.CheckboxGroup(label='Show columns', choices=column_names, value=default_columns, interactive=True)
+
+    with gr.Row():
+        leaderboard_df = gr.DataFrame(display_df, datatype=types, interactive=False)
+    
+    @gr.on(triggers=[model_select.change, dataset_select.change, columns_select.change, task_select.change],
+           inputs=[model_select, dataset_select, columns_select, task_select],
+           outputs=[leaderboard_df])
+    def filter_table(model_id, dataset_id, columns, task):
+        # filter the table based on task
+        df = records[(records['hf_pipeline_type'] == task)]
+        # filter the table based on the model_id and dataset_id
+        if model_id:
+            df = records[(records['model_id'] == model_id)]
+        if dataset_id:
+            df = records[(records['dataset_id'] == dataset_id)]
+
+        # filter the table based on the columns
+        df = df[columns]
+        types = get_types(df)
+        display_df = get_display_df(df)
+        return (
+            gr.update(value=display_df, datatype=types, interactive=False)
+        )

app_legacy.py

@@ -11,7 +11,7 @@ import json
 from transformers.pipelines import TextClassificationPipeline
 
 from text_classification import check_column_mapping_keys_validity, text_classification_fix_column_mapping
-from utils import read_scanners, write_scanners, read_inference_type, write_inference_type, convert_column_mapping_to_json
+from io_utils import read_scanners, write_scanners, read_inference_type, write_inference_type, convert_column_mapping_to_json
 from wordings import CONFIRM_MAPPING_DETAILS_MD, CONFIRM_MAPPING_DETAILS_FAIL_MD
 
 HF_REPO_ID = 'HF_REPO_ID'

app_text_classification.py

@@ -9,9 +9,9 @@ import json
 
 from transformers.pipelines import TextClassificationPipeline
 
-from text_classification import get_labels_and_features_from_dataset, check_model, get_example_prediction, check_column_mapping_keys_validity, text_classification_fix_column_mapping
-from utils import read_scanners, write_scanners, read_inference_type, read_column_mapping, write_column_mapping, write_inference_type, convert_column_mapping_to_json
-from wordings import CONFIRM_MAPPING_DETAILS_MD, CONFIRM_MAPPING_DETAILS_FAIL_MD, CONFIRM_MAPPING_DETAILS_FAIL_RAW
+from text_classification import get_labels_and_features_from_dataset, check_model, get_example_prediction
+from io_utils import read_scanners, write_scanners, read_inference_type, read_column_mapping, write_column_mapping, write_inference_type
+from wordings import CONFIRM_MAPPING_DETAILS_MD, CONFIRM_MAPPING_DETAILS_FAIL_RAW
 
 HF_REPO_ID = 'HF_REPO_ID'
 HF_SPACE_ID = 'SPACE_ID'
@@ -222,7 +222,11 @@ def get_demo():
         check_dataset_and_get_split, 
         inputs=[dataset_id_input, dataset_config_input], 
         outputs=[dataset_split_input])
-    
+
+    run_inference.change(
+        write_inference_type,
+        inputs=[run_inference]
+    )
     gr.on(
         triggers=[
             run_btn.click,

fetch_utils.py

@@ -0,0 +1,23 @@
+import huggingface_hub
+import datasets
+
+def check_dataset_and_get_config(dataset_id):
+    try:
+        configs = datasets.get_dataset_config_names(dataset_id)
+        return configs
+    except Exception:
+        # Dataset may not exist
+        return None
+
+def check_dataset_and_get_split(dataset_id, dataset_config):
+    try:
+        ds = datasets.load_dataset(dataset_id, dataset_config)
+    except Exception:
+        # Dataset may not exist
+        return None
+    try:
+        splits = list(ds.keys())
+        return splits
+    except Exception:
+        # Dataset has no splits
+        return None

utils.py → io_utils.py

@@ -17,13 +17,13 @@ def read_scanners(path):
 
 # convert a list of scanners to yaml file
 def write_scanners(scanners):
-    with open(YAML_PATH, "r") as f:
+    print(scanners)
+    with open(YAML_PATH, "r+") as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
-
-    config["detectors"] = scanners
-    with open(YAML_PATH, "w") as f:
-        # save scanners to detectors in yaml
-        yaml.dump(config, f, Dumper=Dumper)
+        if config:
+            config["detectors"] = scanners
+            # save scanners to detectors in yaml
+            yaml.dump(config, f, Dumper=Dumper)
 
 # read model_type from yaml file
 def read_inference_type(path):
@@ -35,15 +35,14 @@ def read_inference_type(path):
 
 # write model_type to yaml file
 def write_inference_type(use_inference):
-    with open(YAML_PATH, "r") as f:
+    with open(YAML_PATH, "r+") as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
     if use_inference:
         config["inference_type"] = 'hf_inference_api'
     else:
         config["inference_type"] = 'hf_pipeline'
-    with open(YAML_PATH, "w") as f:
-        # save inference_type to inference_type in yaml
-        yaml.dump(config, f, Dumper=Dumper)
+    # save inference_type to inference_type in yaml
+    yaml.dump(config, f, Dumper=Dumper)
 
 # read column mapping from yaml file
 def read_column_mapping(path):