restructure and improve user interface with dropdown (#14)

- change structure and improve ui (1b0a56bc85f9a75dfb26b429a583738482a69162)
- clean up and change run btn| (4434857b3c7422d8e0b9532200df04e683af5fd5)


Co-authored-by: zcy <ZeroCommand@users.noreply.huggingface.co>

app.py

@@ -1,374 +1,15 @@
-import gradio as gr
-import datasets
-import huggingface_hub
-import os
-import time
-import subprocess
-import logging
-
-import json
-
-from transformers.pipelines import TextClassificationPipeline
-
-from text_classification import check_column_mapping_keys_validity, text_classification_fix_column_mapping
-from utils import read_scanners, write_scanners, read_inference_type, write_inference_type, convert_column_mapping_to_json
-
-HF_REPO_ID = 'HF_REPO_ID'
-HF_SPACE_ID = 'SPACE_ID'
-HF_WRITE_TOKEN = 'HF_WRITE_TOKEN'
-
-theme = gr.themes.Soft(
-    primary_hue="green",
-)
-
-def check_model(model_id):
-    try:
-        task = huggingface_hub.model_info(model_id).pipeline_tag
-    except Exception:
-        return None, None
-
-    try:
-        from transformers import pipeline
-        ppl = pipeline(task=task, model=model_id)
-
-        return model_id, ppl
-    except Exception as e:
-        return model_id, e
-
-
-def check_dataset(dataset_id, dataset_config="default", dataset_split="test"):
-    try:
-        configs = datasets.get_dataset_config_names(dataset_id)
-    except Exception:
-        # Dataset may not exist
-        return None, dataset_config, dataset_split
-
-    if dataset_config not in configs:
-        # Need to choose dataset subset (config)
-        return dataset_id, configs, dataset_split
-
-    ds = datasets.load_dataset(dataset_id, dataset_config)
-
-    if isinstance(ds, datasets.DatasetDict):
-        # Need to choose dataset split
-        if dataset_split not in ds.keys():
-            return dataset_id, None, list(ds.keys())
-    elif not isinstance(ds, datasets.Dataset):
-        # Unknown type
-        return dataset_id, None, None
-    return dataset_id, dataset_config, dataset_split
-
-def try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_mapping='{}'):
-    # Validate model
-    if m_id is None:
-        gr.Warning('Model is not accessible. Please set your HF_TOKEN if it is a private model.')
-        return (
-            gr.update(interactive=False),   # Submit button
-            gr.update(visible=True),       # Loading row
-            gr.update(visible=False),        # Preview row
-            gr.update(visible=False),       # Model prediction input
-            gr.update(visible=False),       # Model prediction preview
-            gr.update(visible=False),       # Label mapping preview
-            gr.update(visible=False),       # feature mapping preview
-        )
-    if isinstance(ppl, Exception):
-        gr.Warning(f'Failed to load model": {ppl}')
-        return (
-            gr.update(interactive=False),   # Submit button
-            gr.update(visible=True),       # Loading row
-            gr.update(visible=False),        # Preview row
-            gr.update(visible=False),       # Model prediction input
-            gr.update(visible=False),       # Model prediction preview
-            gr.update(visible=False),       # Label mapping preview
-            gr.update(visible=False),       # feature mapping preview
-        )
-
-    # Validate dataset
-    d_id, config, split = check_dataset(dataset_id=dataset_id, dataset_config=dataset_config, dataset_split=dataset_split)
-
-    dataset_ok = False
-    if d_id is None:
-        gr.Warning(f'Dataset "{dataset_id}" is not accessible. Please set your HF_TOKEN if it is a private dataset.')
-    elif isinstance(config, list):
-        gr.Warning(f'Dataset "{dataset_id}" does not have "{dataset_config}" config. Please choose a valid config.')
-        config = gr.update(choices=config, value=config[0])
-    elif isinstance(split, list):
-        gr.Warning(f'Dataset "{dataset_id}" does not have "{dataset_split}" split. Please choose a valid split.')
-        split = gr.update(choices=split, value=split[0])
-    else:
-        dataset_ok = True
-
-    if not dataset_ok:
-        return (
-            gr.update(interactive=False),   # Submit button
-            gr.update(visible=True),       # Loading row
-            gr.update(visible=False),        # Preview row
-            gr.update(visible=False),       # Model prediction input
-            gr.update(visible=False),       # Model prediction preview
-            gr.update(visible=False),       # Label mapping preview
-            gr.update(visible=False),       # feature mapping preview
-        )
-
-    # TODO: Validate column mapping by running once
-    prediction_result = None
-    id2label_df = None
-    if isinstance(ppl, TextClassificationPipeline):
-        try:
-            print('validating phase, ', column_mapping)
-            column_mapping = json.loads(column_mapping)
-        except Exception:
-            column_mapping = {}
-
-        column_mapping, prediction_input, prediction_result, id2label_df, feature_df = \
-            text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split)
-
-        column_mapping = json.dumps(column_mapping, indent=2)
 
-    if prediction_result is None and id2label_df is not None:
-        gr.Warning('The model failed to predict with the first row in the dataset. Please provide column mappings in "Advance" settings.')
-        return (
-            gr.update(interactive=False),   # Submit button
-            gr.update(visible=False),       # Loading row
-            gr.update(visible=True),        # Preview row
-            gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
-            gr.update(visible=False),   # Model prediction preview
-            gr.update(value=id2label_df, visible=True, interactive=True),   # Label mapping preview
-            gr.update(value=feature_df, visible=True, interactive=True),   # feature mapping preview
-        )
-    elif id2label_df is None:
-        gr.Warning('The prediction result does not conform the labels in the dataset. Please provide label mappings in "Advance" settings.')
-        return (
-            gr.update(interactive=False),   # Submit button
-            gr.update(visible=False),       # Loading row
-            gr.update(visible=True),        # Preview row
-            gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
-            gr.update(value=prediction_result, visible=True),   # Model prediction preview
-            gr.update(visible=True, interactive=True),   # Label mapping preview
-            gr.update(visible=True, interactive=True),   # feature mapping preview
-        )
+# Start apps
+# from pathlib import Path
 
-    gr.Info("Model and dataset validations passed. Your can submit the evaluation task.")
-
-    return (
-        gr.update(interactive=True),    # Submit button
-        gr.update(visible=False),       # Loading row
-        gr.update(visible=True),        # Preview row
-        gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
-        gr.update(value=prediction_result, visible=True),   # Model prediction preview
-        gr.update(value=id2label_df, visible=True, interactive=True), # Label mapping preview
-        gr.update(value=feature_df, visible=True, interactive=True),   # feature mapping preview
-    )
-
-
-def try_submit(m_id, d_id, config, split, id2label_mapping_dataframe, feature_mapping_dataframe, local):
-    label_mapping = {}
-    for i, label in id2label_mapping_dataframe["Model Prediction Labels"].items():
-        label_mapping.update({str(i): label})
-    
-    feature_mapping = {}
-    for i, feature in feature_mapping_dataframe["Dataset Features"].items():
-        feature_mapping.update({feature_mapping_dataframe["Model Input Features"][i]: feature})
-
-    # TODO: Set column mapping for some dataset such as `amazon_polarity`
-
-    if local:
-        command = [
-            "python",
-            "cli.py",
-            "--loader", "huggingface",
-            "--model", m_id,
-            "--dataset", d_id,
-            "--dataset_config", config,
-            "--dataset_split", split,
-            "--hf_token", os.environ.get(HF_WRITE_TOKEN),
-            "--discussion_repo", os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
-            "--output_format", "markdown",
-            "--output_portal", "huggingface",
-            "--feature_mapping", json.dumps(feature_mapping),
-            "--label_mapping", json.dumps(label_mapping),
-            "--scan_config", "../config.yaml",
-        ]
-
-        eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
-        start = time.time()
-        logging.info(f"Start local evaluation on {eval_str}")
-
-        evaluator = subprocess.Popen(
-            command,
-            cwd=os.path.join(os.path.dirname(os.path.realpath(__file__)), "cicd"),
-            stderr=subprocess.STDOUT,
-        )
-        result = evaluator.wait()
-
-        logging.info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")
+import gradio as gr
 
-        gr.Info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")
-    else:
-        gr.Info("TODO: Submit task to an endpoint")
-    
-    return gr.update(interactive=True)  # Submit button
+from app_text_classification import get_demo as get_demo_text_classification
 
 
-with gr.Blocks(theme=theme) as iface:
+with gr.Blocks(theme=gr.themes.Soft(primary_hue="green")) as demo:
     with gr.Tab("Text Classification"):
-        def check_dataset_and_get_config(dataset_id):
-            try:
-                configs = datasets.get_dataset_config_names(dataset_id)
-                return gr.Dropdown(configs, value=configs[0], visible=True)
-            except Exception:
-                # Dataset may not exist
-                pass
-
-        def check_dataset_and_get_split(dataset_config, dataset_id):
-            try:
-                splits = list(datasets.load_dataset(dataset_id, dataset_config).keys())
-                return gr.Dropdown(splits, value=splits[0], visible=True)
-            except Exception as e:
-                # Dataset may not exist
-                gr.Warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
-                pass
-        
-        def gate_validate_btn(model_id, dataset_id, dataset_config, dataset_split, id2label_mapping_dataframe=None, feature_mapping_dataframe=None):
-            column_mapping = '{}'
-            _, ppl = check_model(model_id=model_id)
-
-            if id2label_mapping_dataframe is not None:
-                labels = convert_column_mapping_to_json(id2label_mapping_dataframe.value, label="data")
-                features = convert_column_mapping_to_json(feature_mapping_dataframe.value, label="text")
-                column_mapping = json.dumps({**labels, **features}, indent=2)
-
-            if check_column_mapping_keys_validity(column_mapping, ppl) is False:
-                gr.Warning('Label mapping table has invalid contents. Please check again.')
-                return (gr.update(interactive=False), 
-                        gr.update(),
-                        gr.update(),
-                        gr.update(),
-                        gr.update(),
-                        gr.update(),
-                        gr.update())
-            else:
-                if model_id and dataset_id and dataset_config and dataset_split:
-                    return try_validate(model_id, ppl, dataset_id, dataset_config, dataset_split, column_mapping)
-                else:
-                    return (gr.update(interactive=False), 
-                            gr.update(visible=True),
-                            gr.update(visible=False),
-                            gr.update(visible=False),
-                            gr.update(visible=False),
-                            gr.update(visible=False),
-                            gr.update(visible=False))
-        with gr.Row():
-            gr.Markdown('''
-                <h1 style="text-align: center;">
-                Giskard Evaluator
-                </h1>
-                Welcome to Giskard Evaluator Space! Get your report immediately by simply input your model id and dataset id below. Follow our leads and improve your model in no time.
-                ''')
-        with gr.Row():
-            run_local = gr.Checkbox(value=True, label="Run in this Space")
-            use_inference = read_inference_type('./config.yaml') == 'hf_inference_api'
-            run_inference = gr.Checkbox(value=use_inference, label="Run with Inference API")
-        
-        with gr.Row() as advanced_row:
-            selected = read_scanners('./config.yaml')
-            scan_config = selected + ['data_leakage']
-            scanners = gr.CheckboxGroup(choices=scan_config, value=selected, label='Scan Settings', visible=True)
-
-        with gr.Row():
-            model_id_input = gr.Textbox(
-                label="Hugging Face model id",
-                placeholder="cardiffnlp/twitter-roberta-base-sentiment-latest",
-            )
-
-            dataset_id_input = gr.Textbox(
-                label="Hugging Face Dataset id",
-                placeholder="tweet_eval",
-            )
-        with gr.Row():
-            dataset_config_input = gr.Dropdown(['default'], value='default', label='Dataset Config', visible=False)
-            dataset_split_input = gr.Dropdown(['default'], value='default', label='Dataset Split', visible=False)
-
-            dataset_id_input.blur(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
-            dataset_id_input.submit(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
-
-            dataset_config_input.change(
-                check_dataset_and_get_split, 
-                inputs=[dataset_config_input, dataset_id_input], 
-                outputs=[dataset_split_input])
-        
-        with gr.Row(visible=True) as loading_row:
-            gr.Markdown('''
-                        <p style="text-align: center;">
-                        🚀🐢Please validate your model and dataset first...
-                        </p>
-                        ''')
-            
-        with gr.Row(visible=False) as preview_row:
-            gr.Markdown('''
-                <h1 style="text-align: center;">
-                Confirm Pre-processing Details
-                </h1>
-                Base on your model and dataset, we inferred this label mapping and feature mapping. <b>If the mapping is incorrect, please modify it in the table below.</b>
-                ''')
-        
-        with gr.Row():
-            id2label_mapping_dataframe = gr.DataFrame(label="Preview of label mapping", interactive=True, visible=False)
-            feature_mapping_dataframe = gr.DataFrame(label="Preview of feature mapping", interactive=True, visible=False)
-        with gr.Row():
-            example_input = gr.Markdown('Sample Input: ', visible=False)
-        
-        with gr.Row():
-            example_labels = gr.Label(label='Model Prediction Sample', visible=False)
-        
-        run_btn = gr.Button(
-            "Get Evaluation Result",
-            variant="primary",
-            interactive=False,
-            size="lg",
-        )
-
-        model_id_input.blur(gate_validate_btn, 
-                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
-        dataset_id_input.blur(gate_validate_btn, 
-                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
-                                outputs=[run_btn, loading_row, preview_row, example_input,  example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
-        dataset_config_input.change(gate_validate_btn,
-                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
-        dataset_split_input.change(gate_validate_btn,
-                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
-        id2label_mapping_dataframe.input(gate_validate_btn,
-                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, id2label_mapping_dataframe, feature_mapping_dataframe],
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
-        feature_mapping_dataframe.input(gate_validate_btn,
-                                inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, id2label_mapping_dataframe, feature_mapping_dataframe],
-                                outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
-        scanners.change(write_scanners, inputs=scanners)
-        run_inference.change(
-            write_inference_type,
-            inputs=[run_inference]
-        )
-
-        run_btn.click(
-            try_submit,
-            inputs=[
-                model_id_input,
-                dataset_id_input,
-                dataset_config_input,
-                dataset_split_input,
-                id2label_mapping_dataframe,
-                feature_mapping_dataframe,
-                run_local,
-            ],
-            outputs=[
-                run_btn,
-            ],
-        )
-
-    with gr.Tab("More"):
+        get_demo_text_classification()
+    with gr.Tab("Leaderboard - Text Classification"):
         pass
-
-if __name__ == "__main__":
-    iface.queue(max_size=20).launch()
+demo.launch()

app_legacy.py

@@ -0,0 +1,373 @@
+import gradio as gr
+import datasets
+import huggingface_hub
+import os
+import time
+import subprocess
+import logging
+
+import json
+
+from transformers.pipelines import TextClassificationPipeline
+
+from text_classification import check_column_mapping_keys_validity, text_classification_fix_column_mapping
+from utils import read_scanners, write_scanners, read_inference_type, write_inference_type, convert_column_mapping_to_json
+from wordings import CONFIRM_MAPPING_DETAILS_MD, CONFIRM_MAPPING_DETAILS_FAIL_MD
+
+HF_REPO_ID = 'HF_REPO_ID'
+HF_SPACE_ID = 'SPACE_ID'
+HF_WRITE_TOKEN = 'HF_WRITE_TOKEN'
+
+def check_model(model_id):
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+    except Exception:
+        return None, None
+
+    try:
+        from transformers import pipeline
+        ppl = pipeline(task=task, model=model_id)
+
+        return model_id, ppl
+    except Exception as e:
+        return model_id, e
+
+
+def check_dataset(dataset_id, dataset_config="default", dataset_split="test"):
+    try:
+        configs = datasets.get_dataset_config_names(dataset_id)
+    except Exception:
+        # Dataset may not exist
+        return None, dataset_config, dataset_split
+
+    if dataset_config not in configs:
+        # Need to choose dataset subset (config)
+        return dataset_id, configs, dataset_split
+
+    ds = datasets.load_dataset(dataset_id, dataset_config)
+
+    if isinstance(ds, datasets.DatasetDict):
+        # Need to choose dataset split
+        if dataset_split not in ds.keys():
+            return dataset_id, None, list(ds.keys())
+    elif not isinstance(ds, datasets.Dataset):
+        # Unknown type
+        return dataset_id, None, None
+    return dataset_id, dataset_config, dataset_split
+
+def try_validate(m_id, ppl, dataset_id, dataset_config, dataset_split, column_mapping='{}'):
+    # Validate model
+    if m_id is None:
+        gr.Warning('Model is not accessible. Please set your HF_TOKEN if it is a private model.')
+        return (
+            gr.update(interactive=False),   # Submit button
+            gr.update(visible=True),       # Loading row
+            gr.update(visible=False),        # Preview row
+            gr.update(visible=False),       # Model prediction input
+            gr.update(visible=False),       # Model prediction preview
+            gr.update(visible=False),       # Label mapping preview
+            gr.update(visible=False),       # feature mapping preview
+        )
+    if isinstance(ppl, Exception):
+        gr.Warning(f'Failed to load model": {ppl}')
+        return (
+            gr.update(interactive=False),   # Submit button
+            gr.update(visible=True),       # Loading row
+            gr.update(visible=False),        # Preview row
+            gr.update(visible=False),       # Model prediction input
+            gr.update(visible=False),       # Model prediction preview
+            gr.update(visible=False),       # Label mapping preview
+            gr.update(visible=False),       # feature mapping preview
+        )
+
+    # Validate dataset
+    d_id, config, split = check_dataset(dataset_id=dataset_id, dataset_config=dataset_config, dataset_split=dataset_split)
+
+    dataset_ok = False
+    if d_id is None:
+        gr.Warning(f'Dataset "{dataset_id}" is not accessible. Please set your HF_TOKEN if it is a private dataset.')
+    elif isinstance(config, list):
+        gr.Warning(f'Dataset "{dataset_id}" does not have "{dataset_config}" config. Please choose a valid config.')
+        config = gr.update(choices=config, value=config[0])
+    elif isinstance(split, list):
+        gr.Warning(f'Dataset "{dataset_id}" does not have "{dataset_split}" split. Please choose a valid split.')
+        split = gr.update(choices=split, value=split[0])
+    else:
+        dataset_ok = True
+
+    if not dataset_ok:
+        return (
+            gr.update(interactive=False),   # Submit button
+            gr.update(visible=True),        # Loading row
+            gr.update(visible=False),       # Preview row
+            gr.update(visible=False),       # Model prediction input
+            gr.update(visible=False),       # Model prediction preview
+            gr.update(visible=False),       # Label mapping preview
+            gr.update(visible=False),       # feature mapping preview
+        )
+
+    # TODO: Validate column mapping by running once
+    prediction_result = None
+    id2label_df = None
+    if isinstance(ppl, TextClassificationPipeline):
+        try:
+            column_mapping = json.loads(column_mapping)
+        except Exception:
+            column_mapping = {}
+
+        column_mapping, prediction_input, prediction_result, id2label_df, feature_df = \
+            text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split)
+
+        column_mapping = json.dumps(column_mapping, indent=2)
+
+    if prediction_result is None and id2label_df is not None:
+        gr.Warning('The model failed to predict with the first row in the dataset. Please provide feature mappings in "Advance" settings.')
+        return (
+            gr.update(interactive=False),   # Submit button
+            gr.update(visible=False),       # Loading row
+            gr.update(CONFIRM_MAPPING_DETAILS_MD, visible=True),        # Preview row
+            gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
+            gr.update(visible=False),   # Model prediction preview
+            gr.update(value=id2label_df, visible=True, interactive=True),   # Label mapping preview
+            gr.update(value=feature_df, visible=True, interactive=True),   # feature mapping preview
+        )
+    elif id2label_df is None:
+        gr.Warning('The prediction result does not conform the labels in the dataset. Please provide label mappings in "Advance" settings.')
+        return (
+            gr.update(interactive=False),   # Submit button
+            gr.update(visible=False),       # Loading row
+            gr.update(CONFIRM_MAPPING_DETAILS_MD, visible=True),        # Preview row
+            gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
+            gr.update(value=prediction_result, visible=True),   # Model prediction preview
+            gr.update(visible=True, interactive=True),   # Label mapping preview
+            gr.update(visible=True, interactive=True),   # feature mapping preview
+        )
+
+    gr.Info("Model and dataset validations passed. Your can submit the evaluation task.")
+
+    return (
+        gr.update(interactive=True),    # Submit button
+        gr.update(visible=False),       # Loading row
+        gr.update(CONFIRM_MAPPING_DETAILS_MD, visible=True),        # Preview row
+        gr.update(value=f'**Sample Input**: {prediction_input}', visible=True),       # Model prediction input
+        gr.update(value=prediction_result, visible=True),   # Model prediction preview
+        gr.update(value=id2label_df, visible=True, interactive=True), # Label mapping preview
+        gr.update(value=feature_df, visible=True, interactive=True),   # feature mapping preview
+    )
+
+
+def try_submit(m_id, d_id, config, split, id2label_mapping_dataframe, feature_mapping_dataframe, local):
+    label_mapping = {}
+    for i, label in id2label_mapping_dataframe["Model Prediction Labels"].items():
+        label_mapping.update({str(i): label})
+    
+    feature_mapping = {}
+    for i, feature in feature_mapping_dataframe["Dataset Features"].items():
+        feature_mapping.update({feature_mapping_dataframe["Model Input Features"][i]: feature})
+
+    # TODO: Set column mapping for some dataset such as `amazon_polarity`
+
+    if local:
+        command = [
+            "python",
+            "cli.py",
+            "--loader", "huggingface",
+            "--model", m_id,
+            "--dataset", d_id,
+            "--dataset_config", config,
+            "--dataset_split", split,
+            "--hf_token", os.environ.get(HF_WRITE_TOKEN),
+            "--discussion_repo", os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
+            "--output_format", "markdown",
+            "--output_portal", "huggingface",
+            "--feature_mapping", json.dumps(feature_mapping),
+            "--label_mapping", json.dumps(label_mapping),
+            "--scan_config", "../config.yaml",
+        ]
+
+        eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+        start = time.time()
+        logging.info(f"Start local evaluation on {eval_str}")
+
+        evaluator = subprocess.Popen(
+            command,
+            cwd=os.path.join(os.path.dirname(os.path.realpath(__file__)), "cicd"),
+            stderr=subprocess.STDOUT,
+        )
+        result = evaluator.wait()
+
+        logging.info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")
+
+        gr.Info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")
+    else:
+        gr.Info("TODO: Submit task to an endpoint")
+    
+    return gr.update(interactive=True)  # Submit button
+
+
+def get_demo():
+    # gr.themes.Soft(
+    #     primary_hue="green",
+    # )
+
+    def check_dataset_and_get_config(dataset_id):
+        try:
+            configs = datasets.get_dataset_config_names(dataset_id)
+            return gr.Dropdown(configs, value=configs[0], visible=True)
+        except Exception:
+            # Dataset may not exist
+            pass
+
+    def check_dataset_and_get_split(dataset_config, dataset_id):
+        try:
+            splits = list(datasets.load_dataset(dataset_id, dataset_config).keys())
+            return gr.Dropdown(splits, value=splits[0], visible=True)
+        except Exception as e:
+            # Dataset may not exist
+            gr.Warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
+            pass
+    
+    def clear_column_mapping_tables():
+        return [
+            gr.update(CONFIRM_MAPPING_DETAILS_FAIL_MD, visible=True),
+            gr.update(value=[], visible=False, interactive=True),
+            gr.update(value=[], visible=False, interactive=True),
+        ]
+    
+    def gate_validate_btn(model_id, dataset_id, dataset_config, dataset_split, id2label_mapping_dataframe=None, feature_mapping_dataframe=None):
+        column_mapping = '{}'
+        _, ppl = check_model(model_id=model_id)
+
+        if id2label_mapping_dataframe is not None:
+            labels = convert_column_mapping_to_json(id2label_mapping_dataframe.value, label="data")
+            features = convert_column_mapping_to_json(feature_mapping_dataframe.value, label="text")
+            column_mapping = json.dumps({**labels, **features}, indent=2)
+
+        if check_column_mapping_keys_validity(column_mapping, ppl) is False:
+            gr.Warning('Label mapping table has invalid contents. Please check again.')
+            return (gr.update(interactive=False), 
+                    gr.update(CONFIRM_MAPPING_DETAILS_FAIL_MD, visible=True),
+                    gr.update(),
+                    gr.update(),
+                    gr.update(),
+                    gr.update(),
+                    gr.update())
+        else:
+            if model_id and dataset_id and dataset_config and dataset_split:
+                return try_validate(model_id, ppl, dataset_id, dataset_config, dataset_split, column_mapping)
+            else:
+                return (gr.update(interactive=False), 
+                        gr.update(visible=True),
+                        gr.update(visible=False),
+                        gr.update(visible=False),
+                        gr.update(visible=False),
+                        gr.update(visible=False),
+                        gr.update(visible=False))
+    with gr.Row():
+        gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
+    with gr.Row():
+        run_local = gr.Checkbox(value=True, label="Run in this Space")
+        use_inference = read_inference_type('./config.yaml') == 'hf_inference_api'
+        run_inference = gr.Checkbox(value=use_inference, label="Run with Inference API")
+    
+    with gr.Row():
+        selected = read_scanners('./config.yaml')
+        scan_config = selected + ['data_leakage']
+        scanners = gr.CheckboxGroup(choices=scan_config, value=selected, label='Scan Settings', visible=True)
+
+    with gr.Row():
+        model_id_input = gr.Textbox(
+            label="Hugging Face model id",
+            placeholder="cardiffnlp/twitter-roberta-base-sentiment-latest",
+        )
+
+        dataset_id_input = gr.Textbox(
+            label="Hugging Face Dataset id",
+            placeholder="tweet_eval",
+        )
+    with gr.Row():
+        dataset_config_input = gr.Dropdown(label='Dataset Config', visible=False)
+        dataset_split_input = gr.Dropdown(label='Dataset Split', visible=False)
+    
+    with gr.Row(visible=True) as loading_row:
+        gr.Markdown('''
+                    <p style="text-align: center;">
+                    🚀🐢Please validate your model and dataset first...
+                    </p>
+                    ''')
+        
+    with gr.Row(visible=False) as preview_row:
+        gr.Markdown('''
+            <h1 style="text-align: center;">
+            Confirm Pre-processing Details
+            </h1>
+            Base on your model and dataset, we inferred this label mapping and feature mapping. <b>If the mapping is incorrect, please modify it in the table below.</b>
+            ''')
+    
+    with gr.Row():
+        id2label_mapping_dataframe = gr.DataFrame(label="Preview of label mapping", interactive=True, visible=False)
+        feature_mapping_dataframe = gr.DataFrame(label="Preview of feature mapping", interactive=True, visible=False)
+    with gr.Row():
+        example_input = gr.Markdown('Sample Input: ', visible=False)
+    
+    with gr.Row():
+        example_labels = gr.Label(label='Model Prediction Sample', visible=False)
+    
+    run_btn = gr.Button(
+        "Get Evaluation Result",
+        variant="primary",
+        interactive=False,
+        size="lg",
+    )
+    
+    model_id_input.blur(clear_column_mapping_tables, outputs=[id2label_mapping_dataframe, feature_mapping_dataframe])
+
+
+    dataset_id_input.blur(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
+    dataset_id_input.submit(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
+
+    dataset_config_input.change(
+        check_dataset_and_get_split, 
+        inputs=[dataset_config_input, dataset_id_input], 
+        outputs=[dataset_split_input])
+    
+    dataset_id_input.blur(clear_column_mapping_tables, outputs=[id2label_mapping_dataframe, feature_mapping_dataframe])
+    # model_id_input.blur(gate_validate_btn, 
+    #                         inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
+    #                         outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+    # dataset_id_input.blur(gate_validate_btn, 
+    #                         inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
+                            # outputs=[run_btn, loading_row, preview_row, example_input,  example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+    dataset_config_input.change(gate_validate_btn,
+                            inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
+                            outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+    dataset_split_input.change(gate_validate_btn,
+                            inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
+                            outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+    id2label_mapping_dataframe.input(gate_validate_btn,
+                            inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, id2label_mapping_dataframe, feature_mapping_dataframe],
+                            outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+    feature_mapping_dataframe.input(gate_validate_btn,
+                            inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, id2label_mapping_dataframe, feature_mapping_dataframe],
+                            outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+    scanners.change(write_scanners, inputs=scanners)
+    run_inference.change(
+        write_inference_type,
+        inputs=[run_inference]
+    )
+
+    run_btn.click(
+        try_submit,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+            id2label_mapping_dataframe,
+            feature_mapping_dataframe,
+            run_local,
+        ],
+        outputs=[
+            run_btn,
+        ],
+    )

app_text_classification.py

@@ -0,0 +1,232 @@
+import gradio as gr
+import datasets
+import os
+import time
+import subprocess
+import logging
+
+import json
+
+from transformers.pipelines import TextClassificationPipeline
+
+from text_classification import get_labels_and_features_from_dataset, check_model, get_example_prediction, check_column_mapping_keys_validity, text_classification_fix_column_mapping
+from utils import read_scanners, write_scanners, read_inference_type, read_column_mapping, write_column_mapping, write_inference_type, convert_column_mapping_to_json
+from wordings import CONFIRM_MAPPING_DETAILS_MD, CONFIRM_MAPPING_DETAILS_FAIL_MD, CONFIRM_MAPPING_DETAILS_FAIL_RAW
+
+HF_REPO_ID = 'HF_REPO_ID'
+HF_SPACE_ID = 'SPACE_ID'
+HF_WRITE_TOKEN = 'HF_WRITE_TOKEN'
+
+MAX_LABELS = 20
+MAX_FEATURES = 20
+
+EXAMPLE_MODEL_ID = 'cardiffnlp/twitter-roberta-base-sentiment-latest'
+EXAMPLE_DATA_ID = 'tweet_eval'
+CONFIG_PATH='./config.yaml'
+
+def try_submit(m_id, d_id, config, split, local):
+    all_mappings = read_column_mapping(CONFIG_PATH)
+
+    if "labels" not in all_mappings.keys():
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        return gr.update(interactive=True)
+    label_mapping = all_mappings["labels"]
+    
+    if "features" not in all_mappings.keys():
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        return gr.update(interactive=True)
+    feature_mapping = all_mappings["features"]
+
+    # TODO: Set column mapping for some dataset such as `amazon_polarity`
+    if local:
+        command = [
+            "python",
+            "cli.py",
+            "--loader", "huggingface",
+            "--model", m_id,
+            "--dataset", d_id,
+            "--dataset_config", config,
+            "--dataset_split", split,
+            "--hf_token", os.environ.get(HF_WRITE_TOKEN),
+            "--discussion_repo", os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
+            "--output_format", "markdown",
+            "--output_portal", "huggingface",
+            "--feature_mapping", json.dumps(feature_mapping),
+            "--label_mapping", json.dumps(label_mapping),
+            "--scan_config", "../config.yaml",
+        ]
+
+        eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+        start = time.time()
+        logging.info(f"Start local evaluation on {eval_str}")
+
+        evaluator = subprocess.Popen(
+            command,
+            cwd=os.path.join(os.path.dirname(os.path.realpath(__file__)), "cicd"),
+            stderr=subprocess.STDOUT,
+        )
+        result = evaluator.wait()
+
+        logging.info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")
+
+        gr.Info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")
+    else:
+        gr.Info("TODO: Submit task to an endpoint")
+    
+    return gr.update(interactive=True)  # Submit button
+
+
+def check_dataset_and_get_config(dataset_id):
+    try:
+        configs = datasets.get_dataset_config_names(dataset_id)
+        return gr.Dropdown(configs, value=configs[0], visible=True)
+    except Exception:
+        # Dataset may not exist
+        pass
+
+def check_dataset_and_get_split(dataset_id, dataset_config):
+    try:
+        splits = list(datasets.load_dataset(dataset_id, dataset_config).keys())
+        return gr.Dropdown(splits, value=splits[0], visible=True)
+    except Exception:
+        # Dataset may not exist
+        # gr.Warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
+        pass
+
+def get_demo():
+    with gr.Row():
+        gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
+    with gr.Row():
+        model_id_input = gr.Textbox(
+            label="Hugging Face model id",
+            placeholder=EXAMPLE_MODEL_ID + " (press enter to confirm)",
+        )
+
+        dataset_id_input = gr.Textbox(
+            label="Hugging Face Dataset id",
+            placeholder=EXAMPLE_DATA_ID + " (press enter to confirm)",
+        )
+    
+    with gr.Row():
+        dataset_config_input = gr.Dropdown(label='Dataset Config', visible=False)
+        dataset_split_input = gr.Dropdown(label='Dataset Split', visible=False)
+    
+    with gr.Row():
+        example_input = gr.Markdown('Example Input', visible=False)
+    with gr.Row():
+        example_prediction = gr.Label(label='Model Prediction Sample', visible=False)
+    
+    with gr.Row():
+        column_mappings = []
+        with gr.Column():
+            for _ in range(MAX_LABELS):
+                column_mappings.append(gr.Dropdown(visible=False))
+        with gr.Column():    
+            for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
+                column_mappings.append(gr.Dropdown(visible=False))
+    
+    with gr.Accordion(label='Model Wrap Advance Config (optional)', open=False):
+        run_local = gr.Checkbox(value=True, label="Run in this Space")
+        use_inference = read_inference_type('./config.yaml') == 'hf_inference_api'
+        run_inference = gr.Checkbox(value=use_inference, label="Run with Inference API")
+    
+    with gr.Accordion(label='Scanner Advance Config (optional)', open=False):
+        selected = read_scanners('./config.yaml')
+        scan_config = selected + ['data_leakage']
+        scanners = gr.CheckboxGroup(choices=scan_config, value=selected, label='Scan Settings', visible=True)
+
+    with gr.Row():
+        run_btn = gr.Button(
+            "Get Evaluation Result",
+            variant="primary",
+            interactive=True,
+            size="lg",
+        )
+        
+    @gr.on(triggers=[label.change for label in column_mappings], 
+           inputs=[dataset_id_input, dataset_config_input, dataset_split_input, *column_mappings])
+    def write_column_mapping_to_config(dataset_id, dataset_config, dataset_split, *labels):
+        ds_labels, ds_features = get_labels_and_features_from_dataset(dataset_id, dataset_config, dataset_split)
+        if labels is None:
+            return
+        labels = [*labels]
+        all_mappings = read_column_mapping(CONFIG_PATH)
+
+        if "labels" not in all_mappings.keys():
+            all_mappings["labels"] = dict()
+        for i, label in enumerate(labels[:MAX_LABELS]):
+            if label:
+                all_mappings["labels"][label] = ds_labels[i]
+
+        if "features" not in all_mappings.keys():
+            all_mappings["features"] = dict()
+        for i, feat in enumerate(labels[MAX_LABELS:(MAX_LABELS + MAX_FEATURES)]):
+            if feat:
+                all_mappings["features"][feat] = ds_features[i]
+        write_column_mapping(all_mappings)
+
+    def list_labels_and_features_from_dataset(dataset_id, dataset_config, dataset_split, model_id2label, model_features):
+        ds_labels, ds_features = get_labels_and_features_from_dataset(dataset_id, dataset_config, dataset_split)
+        if ds_labels is None or ds_features is None:
+            return [gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)]
+        model_labels = list(model_id2label.values())
+        lables = [gr.Dropdown(label=f"{label}", choices=model_labels, value=model_id2label[i], interactive=True, visible=True) for i, label in enumerate(ds_labels[:MAX_LABELS])]
+        lables += [gr.Dropdown(visible=False) for _ in range(MAX_LABELS - len(lables))]
+        features = [gr.Dropdown(label=f"{feature}", choices=ds_features, value=ds_features[0], interactive=True, visible=True) for feature in model_features]
+        features += [gr.Dropdown(visible=False) for _ in range(MAX_FEATURES - len(features))]
+        return lables + features
+    
+    @gr.on(triggers=[model_id_input.change, dataset_config_input.change])
+    def clear_column_mapping_config():
+        write_column_mapping(None)
+    
+    @gr.on(triggers=[model_id_input.change, dataset_config_input.change, dataset_split_input.change],
+        inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
+        outputs=[example_input, example_prediction, *column_mappings])
+    def check_model_and_show_prediction(model_id, dataset_id, dataset_config, dataset_split):
+        ppl = check_model(model_id)
+        if ppl is None or not isinstance(ppl, TextClassificationPipeline):
+            gr.Warning("Please check your model.")
+            return (
+                gr.update(visible=False),
+                gr.update(visible=False),
+                *[gr.update(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)]
+            )
+        model_id2label = ppl.model.config.id2label
+        model_features = ['text']
+        column_mappings = list_labels_and_features_from_dataset(
+            dataset_id, 
+            dataset_config, 
+            dataset_split, 
+            model_id2label, 
+            model_features
+        )
+
+        if ppl is None:
+            gr.Warning("Model not found")
+            return (
+                gr.update(visible=False),
+                gr.update(visible=False),
+                *column_mappings
+            )
+        prediction_input, prediction_output = get_example_prediction(ppl, dataset_id, dataset_config, dataset_split)
+        return (
+            gr.update(value=prediction_input, visible=True),
+            gr.update(value=prediction_output, visible=True),
+            *column_mappings
+        )
+
+    dataset_id_input.blur(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
+
+    dataset_config_input.change(
+        check_dataset_and_get_split, 
+        inputs=[dataset_id_input, dataset_config_input], 
+        outputs=[dataset_split_input])
+    
+    gr.on(
+        triggers=[
+            run_btn.click,
+            ],
+        fn=try_submit,
+        inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, run_local],
+        outputs=[run_btn])

cicd

@@ -1 +0,0 @@
-Subproject commit 96913a4f713372d3325002e0ec97320bae55d323

config.yaml

@@ -1,3 +1,6 @@
+configuration:
+  ethical_bias:
+    threshold: 0.01
 detectors:
   - ethical_bias
   - text_perturbation
@@ -6,10 +9,4 @@ detectors:
   - underconfidence
   - overconfidence
   - spurious_correlation
-
-configuration:
-  ethical_bias:
-    threshold:
-      0.01
-
 inference_type: hf_pipeline

text_classification.py

@@ -2,7 +2,33 @@ import datasets
 import logging
 import json
 import pandas as pd
+import huggingface_hub
+from transformers import pipeline
 
+def get_labels_and_features_from_dataset(dataset_id, dataset_config, split):
+    try:
+        ds = datasets.load_dataset(dataset_id, dataset_config)[split]
+        dataset_features = ds.features
+        labels = dataset_features["label"].names
+        features = [f for f in dataset_features.keys() if f != "label"]
+        return labels, features
+    except Exception as e:
+        logging.warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
+        return None, None
+
+def check_model(model_id):
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+    except Exception:
+        return None
+
+    try:
+        ppl = pipeline(task=task, model=model_id)
+
+        return ppl
+    except Exception:
+        return None
+    
 
 def text_classificaiton_match_label_case_unsensative(id2label_mapping, label):
     for model_label in id2label_mapping.keys():
@@ -60,10 +86,20 @@ def check_column_mapping_keys_validity(column_mapping, ppl):
     
     return user_labels == model_labels == original_labels
 
+'''
+params:
+    column_mapping: dict
+    dataset_features: dict
+    example: {
+        'text': Value(dtype='string', id=None), 
+        'label': ClassLabel(names=['negative', 'neutral', 'positive'], id=None)
+    }
+'''
 def infer_text_input_column(column_mapping, dataset_features):
     # Check whether we need to infer the text input column
     infer_text_input_column = True
     feature_map_df = None
+
     if "text" in column_mapping.keys():
         dataset_text_column = column_mapping["text"]
         if dataset_text_column in dataset_features.keys():
@@ -82,33 +118,21 @@ def infer_text_input_column(column_mapping, dataset_features):
             logging.debug(f"Candidates are {candidates}")
             column_mapping["text"] = candidates[0]
     
-    return column_mapping, feature_map_df 
-
-def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split):
-    # We assume dataset is ok here
-    ds = datasets.load_dataset(d_id, config)[split]
-    try:
-        dataset_features = ds.features
-    except AttributeError:
-        # Dataset does not have features, need to provide everything
-        return None, None, None, None, None
-
-    column_mapping, feature_map_df = infer_text_input_column(column_mapping, dataset_features)
-
-    # Load dataset as DataFrame
-    df = ds.to_pandas()
-
-    # Retrieve all labels
-    id2label_mapping = {}
-    id2label = ppl.model.config.id2label
-    label2id = {v: k for k, v in id2label.items()}
-
-    # Infer labels
-    id2label_mapping, dataset_labels = text_classification_map_model_and_dataset_labels(id2label, dataset_features)
-    id2label_mapping_dataset_model = {
-        v: k for k, v in id2label_mapping.items()
-    }
+    return column_mapping, feature_map_df
 
+'''
+params:
+    column_mapping: dict
+    id2label_mapping: dict
+    example:
+    id2label_mapping: {
+        'negative': 'negative', 
+        'neutral': 'neutral', 
+        'positive': 'positive'
+        }
+'''
+def infer_output_label_column(column_mapping, id2label_mapping, id2label, dataset_labels):
+    # Check whether we need to infer the output label column
     if "data" in column_mapping.keys():
         if isinstance(column_mapping["data"], list):
             # Use the column mapping passed by user
@@ -118,13 +142,63 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
         column_mapping["label"] = {
             i: None for i in id2label.keys()
         }
-        return column_mapping, None, None, None, feature_map_df
+        return column_mapping, None
+    
+    if "data" not in column_mapping.keys():
+        # Column mapping should contain original model labels
+        column_mapping["label"] = {
+            str(i): id2label_mapping[label] for i, label in zip(id2label.keys(), dataset_labels)
+        }
+    # print('>>>>> column_mapping >>>>>', column_mapping)
 
     id2label_df = pd.DataFrame({
         "Dataset Labels": dataset_labels,
-        "Model Prediction Labels": [id2label_mapping_dataset_model[label] for label in dataset_labels],
+        "Model Prediction Labels": [id2label_mapping[label] for label in dataset_labels],
     })
+
+    return column_mapping, id2label_df
+
+def check_dataset_features_validity(d_id, config, split):
+    # We assume dataset is ok here
+    ds = datasets.load_dataset(d_id, config)[split]
+    try:
+        dataset_features = ds.features
+    except AttributeError:
+        # Dataset does not have features, need to provide everything
+        return None, None
+        # Load dataset as DataFrame
+    df = ds.to_pandas()
+
+    return df, dataset_features
+
+def get_example_prediction(ppl, dataset_id, dataset_config, dataset_split):
+    # get a sample prediction from the model on the dataset
+    prediction_input = None
+    prediction_result = None
+    try:
+        # Use the first item to test prediction
+        ds = datasets.load_dataset(dataset_id, dataset_config)[dataset_split]
+        if "text" not in ds.features.keys():
+            # Dataset does not have text column
+            prediction_input = ds[0][ds.features.keys()[0]]
+        else:
+            prediction_input = ds[0]["text"]
+
+        print('prediction_input', prediction_input)
+        results = ppl(prediction_input, top_k=None)
+        # Display results in original label and mapped label
+        prediction_result = {
+            f'{result["label"]}': result["score"] for result in results
+        }
+    except Exception:
+        # Pipeline prediction failed, need to provide labels
+        return prediction_input, None
     
+
+    return prediction_input, prediction_result
+
+
+def get_sample_prediction(ppl, df, column_mapping, id2label_mapping):
     # get a sample prediction from the model on the dataset
     prediction_input = None
     prediction_result = None
@@ -133,21 +207,42 @@ def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, sp
         prediction_input = df.head(1).at[0, column_mapping["text"]]
         results = ppl({"text": prediction_input}, top_k=None)
         prediction_result = {
-            f'{result["label"]}({label2id[result["label"]]})': result["score"] for result in results
+            f'{result["label"]}': result["score"] for result in results
         }
-    except Exception as e:
+    except Exception:
         # Pipeline prediction failed, need to provide labels
-        print(e, '>>>> error')
-        return column_mapping, prediction_input, None, id2label_df, feature_map_df
+        return prediction_input, None
     
+    # Display results in original label and mapped label
     prediction_result = {
-        f'[{label2id[result["label"]]}]{result["label"]}(original) - {id2label_mapping[result["label"]]}(mapped)': result["score"] for result in results
+        f'{result["label"]}(original) - {id2label_mapping[result["label"]]}(mapped)': result["score"] for result in results
     }
+    return prediction_input, prediction_result
 
-    if "data" not in column_mapping.keys():
-        # Column mapping should contain original model labels
-        column_mapping["label"] = {
-            str(i): id2label_mapping_dataset_model[label] for i, label in zip(id2label.keys(), dataset_labels)
-        }
+def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split):
+    # load dataset as pd DataFrame
+    # get features column from dataset
+    df, dataset_features = check_dataset_features_validity(d_id, config, split)
 
+    column_mapping, feature_map_df = infer_text_input_column(column_mapping, dataset_features)
+    if feature_map_df is None:
+        # dataset does not have any features
+        return None, None, None, None, None 
+
+    # Retrieve all labels
+    id2label = ppl.model.config.id2label
+
+    # Infer labels
+    id2label_mapping, dataset_labels = text_classification_map_model_and_dataset_labels(id2label, dataset_features)
+    column_mapping, id2label_df = infer_output_label_column(column_mapping, id2label_mapping, id2label, dataset_labels)
+    if id2label_df is None:
+        # does not able to infer output label column
+        return column_mapping, None, None, None, feature_map_df
+    
+    # Get a sample prediction
+    prediction_input, prediction_result = get_sample_prediction(ppl, df, column_mapping, id2label_mapping)
+    if prediction_result is None:
+        # does not able to get a sample prediction
+        return column_mapping, prediction_input, None, id2label_df, feature_map_df
+    
     return column_mapping, prediction_input, prediction_result, id2label_df, feature_map_df

utils.py

@@ -12,7 +12,7 @@ def read_scanners(path):
     scanners = []
     with open(path, "r") as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
-        scanners = config.get("detectors", None)
+        scanners = config.get("detectors", [])
     return scanners
 
 # convert a list of scanners to yaml file
@@ -30,7 +30,7 @@ def read_inference_type(path):
     inference_type = ""
     with open(path, "r") as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
-        inference_type = config.get("inference_type", None)
+        inference_type = config.get("inference_type", "")
     return inference_type
 
 # write model_type to yaml file
@@ -45,10 +45,30 @@ def write_inference_type(use_inference):
         # save inference_type to inference_type in yaml
         yaml.dump(config, f, Dumper=Dumper)
 
+# read column mapping from yaml file
+def read_column_mapping(path):
+    column_mapping = {}
+    with open(path, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        column_mapping = config.get("column_mapping", dict())
+    return column_mapping
+
+# write column mapping to yaml file
+def write_column_mapping(mapping):
+    with open(YAML_PATH, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+    if mapping is None:
+        del config["column_mapping"]
+    else:
+        config["column_mapping"] = mapping
+    with open(YAML_PATH, "w") as f:
+        # save column_mapping to column_mapping in yaml
+        yaml.dump(config, f, Dumper=Dumper)
+
 # convert column mapping dataframe to json
 def convert_column_mapping_to_json(df, label=""):
     column_mapping = {}
     column_mapping[label] = []
     for _, row in df.iterrows():
         column_mapping[label].append(row.tolist())
-    return column_mapping
+    return column_mapping

wordings.py

@@ -0,0 +1,17 @@
+CONFIRM_MAPPING_DETAILS_MD = '''
+                <h1 style="text-align: center;">
+                Giskard Evaluator
+                </h1>
+                Welcome to Giskard Evaluator Space! Get your report immediately by simply input your model id and dataset id below. Follow our leads and improve your model in no time.
+                '''
+
+CONFIRM_MAPPING_DETAILS_FAIL_MD = '''
+                            <h1 style="text-align: center;">
+                            Confirm Pre-processing Details
+                            </h1>
+                            Sorry, we cannot align the input/output of your dataset with the model. <b>Pleaser double check your model and dataset.</b>
+                            '''
+
+CONFIRM_MAPPING_DETAILS_FAIL_RAW= '''
+                            Sorry, we cannot align the input/output of your dataset with the model. Pleaser double check your model and dataset.
+                            '''