import uuid import gradio as gr from io_utils import read_scanners, write_scanners from text_classification_ui_helpers import ( get_related_datasets_from_leaderboard, align_columns_and_show_prediction, get_dataset_splits, check_dataset, show_hf_token_info, precheck_model_ds_enable_example_btn, try_submit, empty_column_mapping, write_column_mapping_to_config, enable_run_btn, ) import logging from wordings import ( EXAMPLE_MODEL_ID, CONFIRM_MAPPING_DETAILS_MD, INTRODUCTION_MD, USE_INFERENCE_API_TIP, USE_INFERENCE_API_NOTICE, CHECK_LOG_SECTION_RAW, HF_TOKEN_INVALID_STYLED, ) MAX_LABELS = 40 MAX_FEATURES = 20 CONFIG_PATH = "./config.yaml" logger = logging.getLogger(__name__) def get_demo(): with gr.Row(): gr.Markdown(INTRODUCTION_MD) uid_label = gr.Textbox( label="Evaluation ID:", value=uuid.uuid4, visible=False, interactive=False ) with gr.Row(): with gr.Column(): with gr.Row(): model_id_input = gr.Textbox( label="Hugging Face Model id", placeholder=f"e.g. {EXAMPLE_MODEL_ID}", ) with gr.Row(): model_id_error_info = gr.HTML(visible=False) with gr.Column(): dataset_id_input = gr.Dropdown( choices=[], value="", allow_custom_value=True, label="Hugging Face Dataset id", ) with gr.Row(): dataset_config_input = gr.Dropdown( label="Dataset Config", visible=False, allow_custom_value=True ) dataset_split_input = gr.Dropdown( label="Dataset Split", visible=False, allow_custom_value=True ) with gr.Row(): first_line_ds = gr.DataFrame(label="Dataset Preview", visible=False) with gr.Row(): loading_dataset_info = gr.HTML(visible=True) with gr.Row(): example_btn = gr.Button( "Validate Model & Dataset", visible=True, variant="primary", interactive=False, ) with gr.Row(): loading_validation = gr.HTML(visible=True) with gr.Row(): validation_result = gr.HTML(visible=False) with gr.Row(): example_input = gr.Textbox( label="Example Input", visible=False, interactive=False ) example_prediction = gr.Label(label="Model Sample Prediction", visible=False) with gr.Row(): with gr.Accordion( label="Label and Feature Mapping", visible=False, open=False ) as column_mapping_accordion: with gr.Row(): gr.Markdown(CONFIRM_MAPPING_DETAILS_MD) column_mappings = [] with gr.Row(): with gr.Column(): gr.Markdown("# Label Mapping") for _ in range(MAX_LABELS): column_mappings.append(gr.Dropdown(visible=False)) with gr.Column(): gr.Markdown("# Feature Mapping") for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES): column_mappings.append(gr.Dropdown(visible=False)) with gr.Accordion(label="Model Wrap Advanced Config", open=True): gr.HTML(USE_INFERENCE_API_TIP) inference_token = gr.Textbox( placeholder="hf_xxxxxxxxxxxxxxxxxxxx", value="", label="HF Token for Inference API", visible=True, interactive=True, ) inference_token_info = gr.HTML(value=HF_TOKEN_INVALID_STYLED, visible=False) gr.HTML(USE_INFERENCE_API_NOTICE) inference_token.change( fn=show_hf_token_info, inputs=[inference_token], outputs=[inference_token_info], ) with gr.Accordion(label="Scanner Advanced Config (optional)", open=False): with gr.Group(): verbose_mode_checkbox = gr.Checkbox(label="Verbose mode") scanners = gr.CheckboxGroup(visible=True) @gr.on(triggers=[uid_label.change], inputs=[uid_label], outputs=[scanners]) def get_scanners(uid): selected = read_scanners(uid) # we remove data_leakage from the default scanners # Reason: data_leakage barely raises any issues and takes too many requests # when using inference API, causing rate limit error scan_config = [ "ethical_bias", "text_perturbation", "robustness", "performance", "underconfidence", "overconfidence", "spurious_correlation", "data_leakage", ] return gr.update( choices=scan_config, value=selected, label="Scan Settings", visible=True ) with gr.Row(): run_btn = gr.Button( "Get Evaluation Result", variant="primary", interactive=False, size="lg", ) with gr.Row(): logs = gr.Textbox( value=CHECK_LOG_SECTION_RAW, label="Log", visible=False, every=0.5, ) scanners.change(write_scanners, inputs=[scanners, uid_label]) gr.on( triggers=[model_id_input.change], fn=get_related_datasets_from_leaderboard, inputs=[model_id_input, dataset_id_input], outputs=[dataset_id_input], ).then( fn=check_dataset, inputs=[dataset_id_input], outputs=[dataset_config_input, dataset_split_input, loading_dataset_info], ) gr.on( triggers=[dataset_id_input.input, dataset_id_input.select], fn=check_dataset, inputs=[dataset_id_input], outputs=[dataset_config_input, dataset_split_input, loading_dataset_info], ) dataset_config_input.change( fn=get_dataset_splits, inputs=[dataset_id_input, dataset_config_input], outputs=[dataset_split_input], ) gr.on( triggers=[ model_id_input.change, dataset_id_input.change, dataset_config_input.change, ], fn=empty_column_mapping, inputs=[uid_label], ) gr.on( triggers=[label.change for label in column_mappings], fn=write_column_mapping_to_config, inputs=[ uid_label, *column_mappings, ], ) # label.change sometimes does not pass the changed value gr.on( triggers=[label.input for label in column_mappings], fn=write_column_mapping_to_config, inputs=[ uid_label, *column_mappings, ], ) gr.on( triggers=[ model_id_input.change, dataset_id_input.change, dataset_config_input.change, dataset_split_input.change, ], fn=precheck_model_ds_enable_example_btn, inputs=[ model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, ], outputs=[ example_btn, first_line_ds, validation_result, example_input, example_prediction, column_mapping_accordion, model_id_error_info, ], ) gr.on( triggers=[ example_btn.click, ], fn=align_columns_and_show_prediction, inputs=[ model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, uid_label, inference_token, ], outputs=[ validation_result, example_input, example_prediction, column_mapping_accordion, run_btn, loading_validation, *column_mappings, ], ) gr.on( triggers=[ run_btn.click, ], fn=try_submit, inputs=[ model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, inference_token, uid_label, verbose_mode_checkbox, ], outputs=[ run_btn, logs, uid_label, validation_result, example_input, example_prediction, column_mapping_accordion, ], ) gr.on( triggers=[ inference_token.input, scanners.input, ], fn=enable_run_btn, inputs=[ uid_label, inference_token, model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, ], outputs=[run_btn], ) gr.on( triggers=[label.input for label in column_mappings], fn=enable_run_btn, inputs=[ uid_label, inference_token, model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, ], # FIXME outputs=[run_btn], )