Spaces:
Running
Running
| import uuid | |
| import gradio as gr | |
| from io_utils import read_scanners, write_scanners | |
| from text_classification_ui_helpers import ( | |
| get_related_datasets_from_leaderboard, | |
| align_columns_and_show_prediction, | |
| get_dataset_splits, | |
| check_dataset, | |
| show_hf_token_info, | |
| precheck_model_ds_enable_example_btn, | |
| try_submit, | |
| empty_column_mapping, | |
| write_column_mapping_to_config, | |
| enable_run_btn, | |
| ) | |
| import logging | |
| from wordings import ( | |
| EXAMPLE_MODEL_ID, | |
| CONFIRM_MAPPING_DETAILS_MD, | |
| INTRODUCTION_MD, | |
| USE_INFERENCE_API_TIP, | |
| USE_INFERENCE_API_NOTICE, | |
| CHECK_LOG_SECTION_RAW, | |
| HF_TOKEN_INVALID_STYLED, | |
| ) | |
| MAX_LABELS = 40 | |
| MAX_FEATURES = 20 | |
| CONFIG_PATH = "./config.yaml" | |
| logger = logging.getLogger(__name__) | |
| def get_demo(): | |
| with gr.Row(): | |
| gr.Markdown(INTRODUCTION_MD) | |
| uid_label = gr.Textbox( | |
| label="Evaluation ID:", value=uuid.uuid4, visible=False, interactive=False | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Row(): | |
| model_id_input = gr.Textbox( | |
| label="Hugging Face Model id", | |
| placeholder=f"e.g. {EXAMPLE_MODEL_ID}", | |
| ) | |
| with gr.Row(): | |
| model_id_error_info = gr.HTML(visible=False) | |
| with gr.Column(): | |
| dataset_id_input = gr.Dropdown( | |
| choices=[], | |
| value="", | |
| allow_custom_value=True, | |
| label="Hugging Face Dataset id", | |
| ) | |
| with gr.Row(): | |
| dataset_config_input = gr.Dropdown( | |
| label="Dataset Config", visible=False, allow_custom_value=True | |
| ) | |
| dataset_split_input = gr.Dropdown( | |
| label="Dataset Split", visible=False, allow_custom_value=True | |
| ) | |
| with gr.Row(): | |
| first_line_ds = gr.DataFrame(label="Dataset Preview", visible=False) | |
| with gr.Row(): | |
| loading_dataset_info = gr.HTML(visible=True) | |
| with gr.Row(): | |
| example_btn = gr.Button( | |
| "Validate Model & Dataset", | |
| visible=True, | |
| variant="primary", | |
| interactive=False, | |
| ) | |
| with gr.Row(): | |
| loading_validation = gr.HTML(visible=True) | |
| with gr.Row(): | |
| validation_result = gr.HTML(visible=False) | |
| with gr.Row(): | |
| example_input = gr.Textbox( | |
| label="Example Input", visible=False, interactive=False | |
| ) | |
| example_prediction = gr.Label(label="Model Sample Prediction", visible=False) | |
| with gr.Row(): | |
| with gr.Accordion( | |
| label="Label and Feature Mapping", visible=False, open=False | |
| ) as column_mapping_accordion: | |
| with gr.Row(): | |
| gr.Markdown(CONFIRM_MAPPING_DETAILS_MD) | |
| column_mappings = [] | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("# Label Mapping") | |
| for _ in range(MAX_LABELS): | |
| column_mappings.append(gr.Dropdown(visible=False)) | |
| with gr.Column(): | |
| gr.Markdown("# Feature Mapping") | |
| for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES): | |
| column_mappings.append(gr.Dropdown(visible=False)) | |
| with gr.Accordion(label="Model Wrap Advanced Config", open=True): | |
| gr.HTML(USE_INFERENCE_API_TIP) | |
| inference_token = gr.Textbox( | |
| placeholder="hf_xxxxxxxxxxxxxxxxxxxx", | |
| value="", | |
| label="HF Token for Inference API", | |
| visible=True, | |
| interactive=True, | |
| ) | |
| inference_token_info = gr.HTML(value=HF_TOKEN_INVALID_STYLED, visible=False) | |
| gr.HTML(USE_INFERENCE_API_NOTICE) | |
| inference_token.change( | |
| fn=show_hf_token_info, | |
| inputs=[inference_token], | |
| outputs=[inference_token_info], | |
| ) | |
| with gr.Accordion(label="Scanner Advanced Config (optional)", open=False): | |
| with gr.Group(): | |
| verbose_mode_checkbox = gr.Checkbox(label="Verbose mode") | |
| scanners = gr.CheckboxGroup(visible=True) | |
| def get_scanners(uid): | |
| selected = read_scanners(uid) | |
| # we remove data_leakage from the default scanners | |
| # Reason: data_leakage barely raises any issues and takes too many requests | |
| # when using inference API, causing rate limit error | |
| scan_config = [ | |
| "ethical_bias", | |
| "text_perturbation", | |
| "robustness", | |
| "performance", | |
| "underconfidence", | |
| "overconfidence", | |
| "spurious_correlation", | |
| "data_leakage", | |
| ] | |
| return gr.update( | |
| choices=scan_config, value=selected, label="Scan Settings", visible=True | |
| ) | |
| with gr.Row(): | |
| run_btn = gr.Button( | |
| "Get Evaluation Result", | |
| variant="primary", | |
| interactive=False, | |
| size="lg", | |
| ) | |
| with gr.Row(): | |
| logs = gr.Textbox( | |
| value=CHECK_LOG_SECTION_RAW, | |
| label="Log", | |
| visible=False, | |
| every=0.5, | |
| ) | |
| scanners.change(write_scanners, inputs=[scanners, uid_label]) | |
| gr.on( | |
| triggers=[model_id_input.change], | |
| fn=get_related_datasets_from_leaderboard, | |
| inputs=[model_id_input, dataset_id_input], | |
| outputs=[dataset_id_input], | |
| ).then( | |
| fn=check_dataset, | |
| inputs=[dataset_id_input], | |
| outputs=[dataset_config_input, dataset_split_input, loading_dataset_info], | |
| ) | |
| gr.on( | |
| triggers=[dataset_id_input.input, dataset_id_input.select], | |
| fn=check_dataset, | |
| inputs=[dataset_id_input], | |
| outputs=[dataset_config_input, dataset_split_input, loading_dataset_info], | |
| ) | |
| dataset_config_input.change( | |
| fn=get_dataset_splits, | |
| inputs=[dataset_id_input, dataset_config_input], | |
| outputs=[dataset_split_input], | |
| ) | |
| gr.on( | |
| triggers=[ | |
| model_id_input.change, | |
| dataset_id_input.change, | |
| dataset_config_input.change, | |
| ], | |
| fn=empty_column_mapping, | |
| inputs=[uid_label], | |
| ) | |
| gr.on( | |
| triggers=[label.change for label in column_mappings], | |
| fn=write_column_mapping_to_config, | |
| inputs=[ | |
| uid_label, | |
| *column_mappings, | |
| ], | |
| ) | |
| # label.change sometimes does not pass the changed value | |
| gr.on( | |
| triggers=[label.input for label in column_mappings], | |
| fn=write_column_mapping_to_config, | |
| inputs=[ | |
| uid_label, | |
| *column_mappings, | |
| ], | |
| ) | |
| gr.on( | |
| triggers=[ | |
| model_id_input.change, | |
| dataset_id_input.change, | |
| dataset_config_input.change, | |
| dataset_split_input.change, | |
| ], | |
| fn=precheck_model_ds_enable_example_btn, | |
| inputs=[ | |
| model_id_input, | |
| dataset_id_input, | |
| dataset_config_input, | |
| dataset_split_input, | |
| ], | |
| outputs=[ | |
| example_btn, | |
| first_line_ds, | |
| validation_result, | |
| example_input, | |
| example_prediction, | |
| column_mapping_accordion, | |
| model_id_error_info, | |
| ], | |
| ) | |
| gr.on( | |
| triggers=[ | |
| example_btn.click, | |
| ], | |
| fn=align_columns_and_show_prediction, | |
| inputs=[ | |
| model_id_input, | |
| dataset_id_input, | |
| dataset_config_input, | |
| dataset_split_input, | |
| uid_label, | |
| inference_token, | |
| ], | |
| outputs=[ | |
| validation_result, | |
| example_input, | |
| example_prediction, | |
| column_mapping_accordion, | |
| run_btn, | |
| loading_validation, | |
| *column_mappings, | |
| ], | |
| ) | |
| gr.on( | |
| triggers=[ | |
| run_btn.click, | |
| ], | |
| fn=try_submit, | |
| inputs=[ | |
| model_id_input, | |
| dataset_id_input, | |
| dataset_config_input, | |
| dataset_split_input, | |
| inference_token, | |
| uid_label, | |
| verbose_mode_checkbox, | |
| ], | |
| outputs=[ | |
| run_btn, | |
| logs, | |
| uid_label, | |
| validation_result, | |
| example_input, | |
| example_prediction, | |
| column_mapping_accordion, | |
| ], | |
| ) | |
| gr.on( | |
| triggers=[ | |
| inference_token.input, | |
| scanners.input, | |
| ], | |
| fn=enable_run_btn, | |
| inputs=[ | |
| uid_label, | |
| inference_token, | |
| model_id_input, | |
| dataset_id_input, | |
| dataset_config_input, | |
| dataset_split_input, | |
| ], | |
| outputs=[run_btn], | |
| ) | |
| gr.on( | |
| triggers=[label.input for label in column_mappings], | |
| fn=enable_run_btn, | |
| inputs=[ | |
| uid_label, | |
| inference_token, | |
| model_id_input, | |
| dataset_id_input, | |
| dataset_config_input, | |
| dataset_split_input, | |
| ], # FIXME | |
| outputs=[run_btn], | |
| ) | |