add login button

app.py

@@ -5,7 +5,7 @@ import gradio as gr
 from app_debug import get_demo as get_demo_debug
 from app_leaderboard import get_demo as get_demo_leaderboard
 from app_text_classification import get_demo as get_demo_text_classification
-from utils.run_jobs import start_process_run_job, stop_thread
+from run_jobs import start_process_run_job, stop_thread
 
 try:
     with gr.Blocks(theme=gr.themes.Soft(primary_hue="green")) as demo:

app_leaderboard.py

@@ -5,10 +5,10 @@ import gradio as gr
 import pandas as pd
 import datetime
 
-from utils.fetch_utils import (check_dataset_and_get_config,
+from fetch_utils import (check_dataset_and_get_config,
                          check_dataset_and_get_split)
 
-import utils.leaderboard as leaderboard
+import leaderboard
 logger = logging.getLogger(__name__)
 global update_time 
 update_time = datetime.datetime.fromtimestamp(0)

app_text_classification.py

@@ -2,13 +2,12 @@ import uuid
 
 import gradio as gr
 
-from utils.io_utils import read_scanners, write_scanners
-from utils.ui_helpers import (
+from io_utils import read_scanners, write_scanners
+from text_classification_ui_helpers import (
     get_related_datasets_from_leaderboard,
     align_columns_and_show_prediction,
     get_dataset_splits,
     check_dataset,
-    show_hf_token_info,
     precheck_model_ds_enable_example_btn,
     try_submit,
     empty_column_mapping,
@@ -161,12 +160,6 @@ def get_demo():
         inputs=[uid_label]
     )
 
-    gr.on(
-        triggers=[model_id_input.change, dataset_id_input.change, dataset_config_input.change],
-        fn=empty_column_mapping,
-        inputs=[uid_label]
-    )
-
     gr.on(
         triggers=[label.change for label in column_mappings],
         fn=write_column_mapping_to_config,

fetch_utils.py

@@ -0,0 +1,32 @@
+import logging
+
+import datasets
+
+
+def check_dataset_and_get_config(dataset_id):
+    try:
+        configs = datasets.get_dataset_config_names(dataset_id, trust_remote_code=True)
+        return configs
+    except Exception:
+        # Dataset may not exist
+        return None
+
+
+def check_dataset_and_get_split(dataset_id, dataset_config):
+    try:
+        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
+    except Exception as e:
+        # Dataset may not exist
+        logging.warning(
+            f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
+        )
+        return None
+    try:
+        splits = list(ds.keys())
+        return splits
+    except Exception as e:
+        # Dataset has no splits
+        logging.warning(
+            f"Dataset {dataset_id} with config {dataset_config} has no splits: {e}"
+        )
+        return None

io_utils.py

@@ -0,0 +1,134 @@
+import os
+import logging
+import yaml
+
+YAML_PATH = "./cicd/configs"
+LOG_FILE = "temp_log"
+
+logger = logging.getLogger(__name__)
+
+class Dumper(yaml.Dumper):
+    def increase_indent(self, flow=False, *args, **kwargs):
+        return super().increase_indent(flow=flow, indentless=False)
+
+def get_submitted_yaml_path(uid):
+    if not os.path.exists(f"{YAML_PATH}/submitted"):
+        os.makedirs(f"{YAML_PATH}/submitted")
+    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
+        logger.error(f"config.yaml does not exist for {uid}")
+        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
+    if not os.path.exists(f"{YAML_PATH}/submitted/{uid}_config.yaml"):
+        os.system(f"cp {YAML_PATH}/{uid}_config.yaml {YAML_PATH}/submitted/{uid}_config.yaml")
+    return f"{YAML_PATH}/submitted/{uid}_config.yaml"
+
+def get_yaml_path(uid):
+    if not os.path.exists(YAML_PATH):
+        os.makedirs(YAML_PATH)
+    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
+        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
+    return f"{YAML_PATH}/{uid}_config.yaml"
+
+
+# read scanners from yaml file
+# return a list of scanners
+def read_scanners(uid):
+    scanners = []
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        scanners = config.get("detectors", [])
+    return scanners
+
+
+# convert a list of scanners to yaml file
+def write_scanners(scanners, uid):
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        if config:
+            config["detectors"] = scanners
+    # save scanners to detectors in yaml
+    with open(get_yaml_path(uid), "w") as f:
+        yaml.dump(config, f, Dumper=Dumper)
+
+
+# read model_type from yaml file
+def read_inference_type(uid):
+    inference_type = ""
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        inference_type = config.get("inference_type", "")
+    return inference_type
+
+
+# write model_type to yaml file
+def write_inference_type(use_inference, inference_token, uid):
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        if use_inference:
+            config["inference_type"] = "hf_inference_api"
+            config["inference_token"] = inference_token
+        else:
+            config["inference_type"] = "hf_pipeline"
+            # FIXME: A quick and temp fix for missing token
+            config["inference_token"] = ""
+    # save inference_type to inference_type in yaml
+    with open(get_yaml_path(uid), "w") as f:
+        yaml.dump(config, f, Dumper=Dumper)
+
+
+# read column mapping from yaml file
+def read_column_mapping(uid):
+    column_mapping = {}
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        if config:
+            column_mapping = config.get("column_mapping", dict())
+    if column_mapping is None:
+        column_mapping = {}
+    return column_mapping
+
+
+# write column mapping to yaml file
+def write_column_mapping(mapping, uid):
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+
+    if config is None:
+        return
+    if mapping is None and "column_mapping" in config.keys():
+        del config["column_mapping"]
+    else:
+        config["column_mapping"] = mapping
+    with open(get_yaml_path(uid), "w") as f:
+        # yaml Dumper will by default sort the keys
+        yaml.dump(config, f, Dumper=Dumper, sort_keys=False)
+
+
+# convert column mapping dataframe to json
+def convert_column_mapping_to_json(df, label=""):
+    column_mapping = {}
+    column_mapping[label] = []
+    for _, row in df.iterrows():
+        column_mapping[label].append(row.tolist())
+    return column_mapping
+
+
+def get_log_file_with_uid(uid):
+    try:
+        print(f"Loading {uid}.log")
+        with open(f"./tmp/{uid}.log", "a") as file:
+            return file.read()
+    except Exception:
+        return "Log file does not exist"
+
+
+def get_logs_file():
+    try:
+        with open(LOG_FILE, "r") as file:
+            return file.read()
+    except Exception:
+        return "Log file does not exist"
+
+
+def write_log_to_user_file(task_id, log):
+    with open(f"./tmp/{task_id}.log", "a") as f:
+        f.write(log)

isolated_env.py

@@ -1,7 +1,7 @@
 import os
 import subprocess
 
-from utils.io_utils import write_log_to_user_file
+from io_utils import write_log_to_user_file
 
 
 def prepare_venv(execution_id, deps):

leaderboard.py

@@ -0,0 +1,5 @@
+import pandas as pd
+
+records = pd.DataFrame()
+
+LEADERBOARD = "giskard-bot/evaluator-leaderboard"

pipe.py

@@ -0,0 +1,3 @@
+
+jobs = list()
+current = None

run_jobs.py

@@ -0,0 +1,181 @@
+import json
+import logging
+import os
+import subprocess
+import threading
+import time
+from pathlib import Path
+
+import pipe
+from app_env import (
+    HF_GSK_HUB_HF_TOKEN,
+    HF_GSK_HUB_KEY,
+    HF_GSK_HUB_PROJECT_KEY,
+    HF_GSK_HUB_UNLOCK_TOKEN,
+    HF_GSK_HUB_URL,
+    HF_REPO_ID,
+    HF_SPACE_ID,
+    HF_WRITE_TOKEN,
+)
+from io_utils import LOG_FILE, get_submitted_yaml_path, write_log_to_user_file
+from isolated_env import prepare_venv
+from leaderboard import LEADERBOARD
+
+is_running = False
+
+logger = logging.getLogger(__file__)
+
+
+def start_process_run_job():
+    try:
+        logging.debug("Running jobs in thread")
+        global thread, is_running
+        thread = threading.Thread(target=run_job)
+        thread.daemon = True
+        is_running = True
+        thread.start()
+
+    except Exception as e:
+        print("Failed to start thread: ", e)
+
+
+def stop_thread():
+    logging.debug("Stop thread")
+    global is_running
+    is_running = False
+
+
+def prepare_env_and_get_command(
+    m_id,
+    d_id,
+    config,
+    split,
+    inference_token,
+    uid,
+    label_mapping,
+    feature_mapping,
+):
+    leaderboard_dataset = None
+    if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
+        leaderboard_dataset = LEADERBOARD
+
+    executable = "giskard_scanner"
+    try:
+        # Copy the current requirements (might be changed)
+        with open("requirements.txt", "r") as f:
+            executable = prepare_venv(
+                uid,
+                "\n".join(f.readlines()),
+            )
+        logger.info(f"Using {executable} as executable")
+    except Exception as e:
+        logger.warn(f"Create env failed due to {e}, using the current env as fallback.")
+        executable = "giskard_scanner"
+
+    command = [
+        executable,
+        "--loader",
+        "huggingface",
+        "--model",
+        m_id,
+        "--dataset",
+        d_id,
+        "--dataset_config",
+        config,
+        "--dataset_split",
+        split,
+        "--output_format",
+        "markdown",
+        "--output_portal",
+        "huggingface",
+        "--feature_mapping",
+        json.dumps(feature_mapping),
+        "--label_mapping",
+        json.dumps(label_mapping),
+        "--scan_config",
+        get_submitted_yaml_path(uid),
+        "--inference_type",
+        "hf_inference_api",
+        "--inference_api_token",
+        inference_token,
+    ]
+    # The token to publish post
+    if os.environ.get(HF_WRITE_TOKEN):
+        command.append("--hf_token")
+        command.append(os.environ.get(HF_WRITE_TOKEN))
+
+    # The repo to publish post
+    if os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID):
+        command.append("--discussion_repo")
+        # TODO: Replace by the model id
+        command.append(os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID))
+
+    # The repo to publish for ranking
+    if leaderboard_dataset:
+        command.append("--leaderboard_dataset")
+        command.append(leaderboard_dataset)
+
+    # The info to upload to Giskard hub
+    if os.environ.get(HF_GSK_HUB_KEY):
+        command.append("--giskard_hub_api_key")
+        command.append(os.environ.get(HF_GSK_HUB_KEY))
+        if os.environ.get(HF_GSK_HUB_URL):
+            command.append("--giskard_hub_url")
+            command.append(os.environ.get(HF_GSK_HUB_URL))
+        if os.environ.get(HF_GSK_HUB_PROJECT_KEY):
+            command.append("--giskard_hub_project_key")
+            command.append(os.environ.get(HF_GSK_HUB_PROJECT_KEY))
+        if os.environ.get(HF_GSK_HUB_HF_TOKEN):
+            command.append("--giskard_hub_hf_token")
+            command.append(os.environ.get(HF_GSK_HUB_HF_TOKEN))
+        if os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN):
+            command.append("--giskard_hub_unlock_token")
+            command.append(os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN))
+
+    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+
+    write_log_to_user_file(
+        uid,
+        f"Start local evaluation on {eval_str}. Please wait for your job to start...\n",
+    )
+
+    return command
+
+
+def save_job_to_pipe(task_id, job, description, lock):
+    with lock:
+        pipe.jobs.append((task_id, job, description))
+
+
+def pop_job_from_pipe():
+    if len(pipe.jobs) == 0:
+        return
+    job_info = pipe.jobs.pop()
+    pipe.current = job_info[2]
+    task_id = job_info[0]
+
+    # Link to LOG_FILE
+    log_file_path = Path(LOG_FILE)
+    if log_file_path.exists():
+        log_file_path.unlink()
+    os.symlink(f"./tmp/{task_id}.log", LOG_FILE)
+
+    write_log_to_user_file(task_id, f"Running job id {task_id}\n")
+    command = prepare_env_and_get_command(*job_info[1])
+
+    with open(f"./tmp/{task_id}.log", "a") as log_file:
+        p = subprocess.Popen(command, stdout=log_file, stderr=subprocess.STDOUT)
+        p.wait()
+    pipe.current = None
+
+
+def run_job():
+    global is_running
+    while is_running:
+        try:
+            pop_job_from_pipe()
+            time.sleep(10)
+        except KeyboardInterrupt:
+            logging.debug("KeyboardInterrupt stop background thread")
+            is_running = False
+            break

text_classification_ui_helpers.py

@@ -0,0 +1,460 @@
+import collections
+import logging
+import threading
+import uuid
+
+import datasets
+import gradio as gr
+import pandas as pd
+
+import leaderboard
+from io_utils import (
+  read_column_mapping, 
+  write_column_mapping, 
+  read_scanners, 
+  write_scanners,
+)
+from run_jobs import save_job_to_pipe
+from text_classification import (
+    strip_model_id_from_url,
+    check_model_task,
+    preload_hf_inference_api,
+    get_example_prediction,
+    get_labels_and_features_from_dataset,
+    check_hf_token_validity,
+    HuggingFaceInferenceAPIResponse,
+)
+from wordings import (
+    CHECK_CONFIG_OR_SPLIT_RAW,
+    CONFIRM_MAPPING_DETAILS_FAIL_RAW,
+    MAPPING_STYLED_ERROR_WARNING,
+    NOT_TEXT_CLASSIFICATION_MODEL_RAW,
+    UNMATCHED_MODEL_DATASET_STYLED_ERROR,
+    CHECK_LOG_SECTION_RAW,
+    VALIDATED_MODEL_DATASET_STYLED,
+    get_dataset_fetch_error_raw,
+)
+import os
+from app_env import HF_WRITE_TOKEN
+
+MAX_LABELS = 40
+MAX_FEATURES = 20
+
+ds_dict = None
+ds_config = None
+
+def get_related_datasets_from_leaderboard(model_id):
+    records = leaderboard.records
+    model_id = strip_model_id_from_url(model_id)
+    model_records = records[records["model_id"] == model_id]
+    datasets_unique = list(model_records["dataset_id"].unique())
+
+    if len(datasets_unique) == 0:
+        return gr.update(choices=[])
+    
+    return gr.update(choices=datasets_unique)
+
+
+logger = logging.getLogger(__file__)
+
+def get_dataset_splits(dataset_id, dataset_config):
+    try:
+        splits = datasets.get_dataset_split_names(dataset_id, dataset_config, trust_remote_code=True)
+        return gr.update(choices=splits, value=splits[0], visible=True)
+    except Exception as e:
+        logger.warn(f"Check your dataset {dataset_id} and config {dataset_config}: {e}")
+        return gr.update(visible=False)
+
+def check_dataset(dataset_id):
+    logger.info(f"Loading {dataset_id}")
+    try:
+        configs = datasets.get_dataset_config_names(dataset_id, trust_remote_code=True)
+        if len(configs) == 0:
+            return (
+                gr.update(visible=False),
+                gr.update(visible=False),
+                ""
+            )
+        splits = datasets.get_dataset_split_names(dataset_id, configs[0], trust_remote_code=True)
+        return (
+            gr.update(choices=configs, value=configs[0], visible=True),
+            gr.update(choices=splits, value=splits[0], visible=True),
+            ""
+        )
+    except Exception as e:
+        logger.warn(f"Check your dataset {dataset_id}: {e}")
+        if "doesn't exist" in str(e):
+            gr.Warning(get_dataset_fetch_error_raw(e))
+        if "forbidden" in str(e).lower(): # GSK-2770
+            gr.Warning(get_dataset_fetch_error_raw(e))
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            ""
+        )
+
+def empty_column_mapping(uid):
+    write_column_mapping(None, uid)
+
+def write_column_mapping_to_config(uid, *labels):
+    # TODO: Substitute 'text' with more features for zero-shot
+    # we are not using ds features because we only support "text" for now
+    all_mappings = read_column_mapping(uid)
+
+    if labels is None:
+        return
+    all_mappings = export_mappings(all_mappings, "labels", None, labels[:MAX_LABELS])
+    all_mappings = export_mappings(
+        all_mappings,
+        "features",
+        ["text"],
+        labels[MAX_LABELS : (MAX_LABELS + MAX_FEATURES)],
+    )
+
+    write_column_mapping(all_mappings, uid)
+
+def export_mappings(all_mappings, key, subkeys, values):
+    if key not in all_mappings.keys():
+        all_mappings[key] = dict()
+    if subkeys is None:
+        subkeys = list(all_mappings[key].keys())
+
+    if not subkeys: 
+        logging.debug(f"subkeys is empty for {key}")
+        return all_mappings
+
+    for i, subkey in enumerate(subkeys):
+        if subkey:
+            all_mappings[key][subkey] = values[i % len(values)]
+    return all_mappings
+
+
+def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels, uid):
+    all_mappings = read_column_mapping(uid)
+    # For flattened raw datasets with no labels
+    # check if there are shared labels between model and dataset
+    shared_labels = set(model_labels).intersection(set(ds_labels))
+    if shared_labels:
+        ds_labels = list(shared_labels)
+    if len(ds_labels) > MAX_LABELS:
+        ds_labels = ds_labels[:MAX_LABELS]
+        gr.Warning(f"Too many labels to display for this spcae. We do not support more than {MAX_LABELS} in this space. You can use cli tool at https://github.com/Giskard-AI/cicd.")
+
+    # sort labels to make sure the order is consistent
+    # prediction gives the order based on probability
+    ds_labels.sort()
+    model_labels.sort()
+
+    lables = [
+        gr.Dropdown(
+            label=f"{label}",
+            choices=model_labels,
+            value=model_labels[i % len(model_labels)],
+            interactive=True,
+            visible=True,
+        )
+        for i, label in enumerate(ds_labels)
+    ]
+    lables += [gr.Dropdown(visible=False) for _ in range(MAX_LABELS - len(lables))]
+    all_mappings = export_mappings(all_mappings, "labels", ds_labels, model_labels)
+
+    # TODO: Substitute 'text' with more features for zero-shot
+    features = [
+        gr.Dropdown(
+            label=f"{feature}",
+            choices=ds_features,
+            value=ds_features[0],
+            interactive=True,
+            visible=True,
+        )
+        for feature in ["text"]
+    ]
+    features += [
+        gr.Dropdown(visible=False) for _ in range(MAX_FEATURES - len(features))
+    ]
+    all_mappings = export_mappings(all_mappings, "features", ["text"], ds_features)
+    write_column_mapping(all_mappings, uid)
+
+    return lables + features
+
+
+def precheck_model_ds_enable_example_btn(
+    model_id, dataset_id, dataset_config, dataset_split
+):  
+    model_id = strip_model_id_from_url(model_id)
+    model_task = check_model_task(model_id)
+    preload_hf_inference_api(model_id)
+
+    if dataset_config is None or dataset_split is None or len(dataset_config) == 0:
+        return (
+            gr.update(interactive=False), 
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
+
+    try:
+        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
+        df: pd.DataFrame = ds[dataset_split].to_pandas().head(5)
+        ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds[dataset_split])
+
+        if model_task is None or model_task != "text-classification":
+          gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
+          return (
+              gr.update(interactive=False), 
+              gr.update(value=df, visible=True),
+              gr.update(visible=False),
+              gr.update(visible=False),
+              gr.update(visible=False),
+              gr.update(visible=False),
+          )
+
+        if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
+            gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
+            return (
+                gr.update(interactive=False), 
+                gr.update(value=df, visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+            )
+
+        return (
+                gr.update(interactive=True), 
+                gr.update(value=df, visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+            )
+    except Exception as e:
+        # Config or split wrong
+        logger.warn(f"Check your dataset {dataset_id} and config {dataset_config} on split {dataset_split}: {e}")
+        return (
+            gr.update(interactive=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
+
+
+def align_columns_and_show_prediction(
+    model_id,
+    dataset_id,
+    dataset_config,
+    dataset_split,
+    uid,
+    profile: gr.OAuthProfile | None,
+    oauth_token: gr.OAuthToken | None,
+):
+    model_id = strip_model_id_from_url(model_id)
+    model_task = check_model_task(model_id)
+    if model_task is None or model_task != "text-classification":
+        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            "",
+            *[gr.update(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)],
+        )
+
+    dropdown_placement = [
+        gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)
+    ]
+    
+    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
+
+    prediction_input, prediction_response = get_example_prediction(
+        model_id, dataset_id, dataset_config, dataset_split, hf_token
+    )
+
+    if prediction_input is None or prediction_response is None:
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            "",
+            *dropdown_placement,
+        )
+
+    if isinstance(prediction_response, HuggingFaceInferenceAPIResponse):
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            f"Hugging Face Inference API is loading your model. {prediction_response.message}",
+            *dropdown_placement,
+        )
+
+    model_labels = list(prediction_response.keys())
+    
+    ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
+    ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds)
+
+    # when dataset does not have labels or features
+    if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
+        gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            "",
+            *dropdown_placement,
+        )
+    
+    if len(ds_labels) != len(model_labels):
+        return (
+            gr.update(value=UNMATCHED_MODEL_DATASET_STYLED_ERROR, visible=True),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            "",
+            *dropdown_placement,
+        )
+
+    column_mappings = list_labels_and_features_from_dataset(
+        ds_labels,
+        ds_features,
+        model_labels,
+        uid,
+    )
+
+    # when labels or features are not aligned
+    # show manually column mapping
+    if (
+        collections.Counter(model_labels) != collections.Counter(ds_labels)
+        or ds_features[0] != "text"
+    ):
+        return (
+            gr.update(value=MAPPING_STYLED_ERROR_WARNING, visible=True),
+            gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
+            gr.update(value=prediction_response, visible=True),
+            gr.update(visible=True, open=True),
+            gr.update(interactive=(profile is not None and oauth_token is not None)),
+            "",
+            *column_mappings,
+        )
+
+    return (
+        gr.update(value=VALIDATED_MODEL_DATASET_STYLED, visible=True),
+        gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
+        gr.update(value=prediction_response, visible=True),
+        gr.update(visible=True, open=False),
+        gr.update(interactive=(profile is not None and oauth_token is not None)),
+        "",
+        *column_mappings,
+    )
+
+
+def check_column_mapping_keys_validity(all_mappings):
+    if all_mappings is None:
+        logger.warning("all_mapping is None")
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        return False
+
+    if "labels" not in all_mappings.keys():
+        logger.warning(f"Label mapping is not valid, all_mappings: {all_mappings}")
+        return False
+
+    return True
+
+def enable_run_btn(uid, model_id, dataset_id, dataset_config, dataset_split, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
+    if profile is None:
+        return gr.update(interactive=False)
+    if oath_token is None:
+        return gr.update(interactive=False)
+    if model_id == "" or dataset_id == "" or dataset_config == "" or dataset_split == "":
+        logger.warn("Model id or dataset id is not selected")
+        return gr.update(interactive=False)
+    
+    all_mappings = read_column_mapping(uid)
+    if not check_column_mapping_keys_validity(all_mappings):
+        logger.warn("Column mapping is not valid")
+        return gr.update(interactive=False)
+
+def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys=None):
+    label_mapping = {}
+    if len(all_mappings["labels"].keys()) != len(ds_labels):
+        logger.warn(f"""Label mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
+                    \nall_mappings: {all_mappings}\nds_labels: {ds_labels}""")
+      
+    if len(all_mappings["features"].keys()) != len(ds_features):
+        logger.warn(f"""Feature mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
+                    \nall_mappings: {all_mappings}\nds_features: {ds_features}""")
+
+    for i, label in zip(range(len(ds_labels)),  ds_labels):
+        # align the saved labels with dataset labels order
+        label_mapping.update({str(i): all_mappings["labels"][label]})
+
+    if "features" not in all_mappings.keys():
+        logger.warning("features not in all_mappings")
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+
+    feature_mapping = all_mappings["features"]
+    if len(label_keys) > 0:
+      feature_mapping.update({"label": label_keys[0]})
+    return label_mapping, feature_mapping
+
+def show_hf_token_info(token):
+    valid = check_hf_token_validity(token)
+    if not valid:
+        return gr.update(visible=True)
+    return gr.update(visible=False)
+
+def try_submit(m_id, d_id, config, split, uid, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
+    print(oath_token.token)
+    print(".>>>>>>>>>>>>>>>>>>>>>>")
+    all_mappings = read_column_mapping(uid)
+    if not check_column_mapping_keys_validity(all_mappings):
+        return (gr.update(interactive=True), gr.update(visible=False))
+
+    # get ds labels and features again for alignment
+    ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
+    ds_labels, ds_features, label_keys = get_labels_and_features_from_dataset(ds)
+    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys)
+    
+    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+    save_job_to_pipe(
+        uid,
+        (
+            m_id,
+            d_id,
+            config,
+            split,
+            oath_token.token,
+            uid,
+            label_mapping,
+            feature_mapping,
+        ),
+        eval_str,
+        threading.Lock(),
+    )
+    gr.Info("Your evaluation has been submitted")
+
+    new_uid = uuid.uuid4()
+    scanners = read_scanners(uid)
+    write_scanners(scanners, new_uid)
+
+    return (
+        gr.update(interactive=False),  # Submit button
+        gr.update(value=f"{CHECK_LOG_SECTION_RAW}Your job id is: {uid}. ", lines=5, visible=True, interactive=False),
+        new_uid,  # Allocate a new uuid
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+    )

utils.py

@@ -0,0 +1,29 @@
+import sys
+
+import yaml
+
+
+# read scanners from yaml file
+# return a list of scanners
+def read_scanners(path):
+    scanners = []
+    with open(path, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        scanners = config.get("detectors", None)
+    return scanners
+
+
+# convert a list of scanners to yaml file
+def write_scanners(scanners):
+    with open("./scan_config.yaml", "w") as f:
+        # save scanners to detectors in yaml
+        yaml.dump({"detectors": scanners}, f)
+
+
+# convert column mapping dataframe to json
+def convert_column_mapping_to_json(df, label=""):
+    column_mapping = {}
+    column_mapping[label] = []
+    for _, row in df.iterrows():
+        column_mapping[label].append(row.tolist())
+    return column_mapping

wordings.py

@@ -0,0 +1,80 @@
+INTRODUCTION_MD = """
+                <h1 style="text-align: center;">
+                🐢Giskard Evaluator - Text Classification
+                </h1>
+                Welcome to the Giskard Evaluator Space! Get a model vulnerability report immediately by simply sharing your model and dataset id below. 
+                You can also checkout our library documentation <a href="https://docs.giskard.ai/en/latest/getting_started/quickstart/index.html">here</a>.
+                """
+CONFIRM_MAPPING_DETAILS_MD = """
+                            <h1 style="text-align: center;">
+                            Confirm Pre-processing Details
+                            </h1>
+                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model. You can select the output variable's labels from the dropdowns below.
+                            """
+CONFIRM_MAPPING_DETAILS_FAIL_MD = """
+                            <h1 style="text-align: center;">
+                            Confirm Pre-processing Details
+                            </h1>
+                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                            """
+
+CONFIRM_MAPPING_DETAILS_FAIL_RAW = """
+                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                            """
+
+CHECK_CONFIG_OR_SPLIT_RAW = """
+                            Please check your dataset config or split.
+                            """
+
+CHECK_LOG_SECTION_RAW = """
+                          Your have successfully submitted a Giskard evaluation. Further details are available in the Logs tab. You can find your report will be posted to your model's community discussion.
+                        """
+
+PREDICTION_SAMPLE_MD = """
+                        <h1 style="text-align: center;">
+                        Model Prediction Sample
+                        </h1>
+                        Here's a sample of your model's prediction on an example from the dataset.
+                        """
+
+MAPPING_STYLED_ERROR_WARNING = """
+                        <h3 style="text-align: center;color: orange; background-color: #fff0f3; border-radius: 8px; padding: 10px; ">
+                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                        </h3>
+                        """
+
+UNMATCHED_MODEL_DATASET_STYLED_ERROR = """
+                        <h3 style="text-align: center;color: #fa5f5f; background-color: #fbe2e2; border-radius: 8px; padding: 10px; ">
+                        ❌ Your model and dataset have different numbers of labels. Please double check your model and dataset.
+                        </h3>
+                        """
+
+NOT_TEXT_CLASSIFICATION_MODEL_RAW = """
+                        Your model does not fall under the category of text classification. This page is specifically designated for the evaluation of text classification models.  
+                      """
+
+USE_INFERENCE_API_TIP = """
+                To speed up the evaluation, we recommend using the
+                <a href="https://huggingface.co/docs/api-inference/detailed_parameters#text-classification-task">
+                    Hugging Face Inference API
+                </a>
+                . Please input your <a href="https://huggingface.co/settings/tokens">Hugging Face token</a> to do so. You can find it <a href="https://huggingface.co/settings/tokens">here</a>.
+            """
+
+LOG_IN_TIPS = """
+            To use the Hugging Face Inference API, you need to log in to your Hugging Face account.
+            """
+
+HF_TOKEN_INVALID_STYLED= """
+                <p style="text-align: left;color: red; ">
+                Your Hugging Face token is invalid. Please double check your token.
+                </p>
+                """
+
+VALIDATED_MODEL_DATASET_STYLED = """
+            <h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
+            Your model and dataset have been validated!
+            </h3>"""
+
+def get_dataset_fetch_error_raw(error):
+    return f"""Sorry you cannot use this dataset because {error}. Contact HF team to support this dataset."""