Spaces:
Running
Running
| __all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions'] | |
| import os | |
| import gradio as gr | |
| import pandas as pd | |
| import json | |
| import tempfile | |
| from constants import * | |
| from huggingface_hub import Repository | |
| HF_TOKEN = os.environ.get("HF_TOKEN") | |
| global data_component, filter_component | |
| def upload_file(files): | |
| file_paths = [file.name for file in files] | |
| return file_paths | |
| def add_new_eval( | |
| input_file, | |
| model_name_textbox: str, | |
| revision_name_textbox: str, | |
| model_link: str, | |
| ): | |
| if input_file is None: | |
| return "Error! Empty file!" | |
| upload_data=json.loads(input_file) | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}")) | |
| csv_data = pd.read_csv(CSV_DIR) | |
| if revision_name_textbox == '': | |
| col = csv_data.shape[0] | |
| model_name = model_name_textbox | |
| else: | |
| model_name = revision_name_textbox | |
| model_name_list = csv_data['Model Name (clickable)'] | |
| name_list = [name.split(']')[0][1:] for name in model_name_list] | |
| if revision_name_textbox not in name_list: | |
| col = csv_data.shape[0] | |
| else: | |
| col = name_list.index(revision_name_textbox) | |
| if model_link == '': | |
| model_name = model_name # no url | |
| else: | |
| model_name = '[' + model_name + '](' + model_link + ')' | |
| # add new data | |
| new_data = [ | |
| model_name | |
| ] | |
| for key in TASK_INFO: | |
| if key in upload_data: | |
| new_data.append(upload_data[key][0]) | |
| else: | |
| new_data.append(0) | |
| csv_data.loc[col] = new_data | |
| csv_data = csv_data.to_csv(CSV_DIR, index=False) | |
| submission_repo.push_to_hub() | |
| return 0 | |
| def get_normalized_df(df): | |
| # final_score = df.drop('name', axis=1).sum(axis=1) | |
| # df.insert(1, 'Overall Score', final_score) | |
| normalize_df = df.copy().fillna(0.0) | |
| for column in normalize_df.columns[1:]: | |
| min_val = NORMALIZE_DIC[column]['Min'] | |
| max_val = NORMALIZE_DIC[column]['Max'] | |
| normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val) | |
| return normalize_df | |
| def get_normalized_i2v_df(df): | |
| normalize_df = df.copy().fillna(0.0) | |
| for column in normalize_df.columns[1:]: | |
| min_val = NORMALIZE_DIC_I2V[column]['Min'] | |
| max_val = NORMALIZE_DIC_I2V[column]['Max'] | |
| normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val) | |
| return normalize_df | |
| def calculate_selected_score(df, selected_columns): | |
| # selected_score = df[selected_columns].sum(axis=1) | |
| selected_QUALITY = [i for i in selected_columns if i in QUALITY_LIST] | |
| selected_SEMANTIC = [i for i in selected_columns if i in SEMANTIC_LIST] | |
| selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_QUALITY]) | |
| selected_semantic_score = df[selected_SEMANTIC].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_SEMANTIC ]) | |
| if selected_quality_score.isna().any().any() and selected_semantic_score.isna().any().any(): | |
| selected_score = (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) | |
| return selected_score.fillna(0.0) | |
| if selected_quality_score.isna().any().any(): | |
| return selected_semantic_score | |
| if selected_semantic_score.isna().any().any(): | |
| return selected_quality_score | |
| # print(selected_semantic_score,selected_quality_score ) | |
| selected_score = (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) | |
| return selected_score.fillna(0.0) | |
| def calculate_selected_score_i2v(df, selected_columns): | |
| # selected_score = df[selected_columns].sum(axis=1) | |
| selected_QUALITY = [i for i in selected_columns if i in I2V_QUALITY_LIST] | |
| selected_I2V = [i for i in selected_columns if i in I2V_LIST] | |
| selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_QUALITY]) | |
| selected_i2v_score = df[selected_I2V].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_I2V ]) | |
| if selected_quality_score.isna().any().any() and selected_i2v_score.isna().any().any(): | |
| selected_score = (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) | |
| return selected_score.fillna(0.0) | |
| if selected_quality_score.isna().any().any(): | |
| return selected_i2v_score | |
| if selected_i2v_score.isna().any().any(): | |
| return selected_quality_score | |
| print(selected_i2v_score,selected_quality_score ) | |
| selected_score = (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) | |
| return selected_score.fillna(0.0) | |
| def get_final_score(df, selected_columns): | |
| normalize_df = get_normalized_df(df) | |
| #final_score = normalize_df.drop('name', axis=1).sum(axis=1) | |
| for name in normalize_df.drop('Model Name (clickable)', axis=1): | |
| normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name] | |
| quality_score = normalize_df[QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in QUALITY_LIST]) | |
| semantic_score = normalize_df[SEMANTIC_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in SEMANTIC_LIST ]) | |
| final_score = (quality_score * QUALITY_WEIGHT + semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) | |
| if 'Total Score' in df: | |
| df['Total Score'] = final_score | |
| else: | |
| df.insert(1, 'Total Score', final_score) | |
| if 'Semantic Score' in df: | |
| df['Semantic Score'] = semantic_score | |
| else: | |
| df.insert(2, 'Semantic Score', semantic_score) | |
| if 'Quality Score' in df: | |
| df['Quality Score'] = quality_score | |
| else: | |
| df.insert(3, 'Quality Score', quality_score) | |
| selected_score = calculate_selected_score(normalize_df, selected_columns) | |
| if 'Selected Score' in df: | |
| df['Selected Score'] = selected_score | |
| else: | |
| df.insert(1, 'Selected Score', selected_score) | |
| return df | |
| def get_final_score_i2v(df, selected_columns): | |
| normalize_df = get_normalized_i2v_df(df) | |
| #final_score = normalize_df.drop('name', axis=1).sum(axis=1) | |
| for name in normalize_df.drop('Model Name (clickable)', axis=1).drop('Video-Text Camera Motion', axis=1): | |
| normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name] | |
| quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST]) | |
| i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ]) | |
| final_score = (quality_score * I2V_QUALITY_WEIGHT + i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) | |
| if 'Total Score' in df: | |
| df['Total Score'] = final_score | |
| else: | |
| df.insert(1, 'Total Score', final_score) | |
| if 'I2V Score' in df: | |
| df['I2V Score'] = i2v_score | |
| else: | |
| df.insert(2, 'I2V Score', i2v_score) | |
| if 'Quality Score' in df: | |
| df['Quality Score'] = quality_score | |
| else: | |
| df.insert(3, 'Quality Score', quality_score) | |
| selected_score = calculate_selected_score_i2v(normalize_df, selected_columns) | |
| if 'Selected Score' in df: | |
| df['Selected Score'] = selected_score | |
| else: | |
| df.insert(1, 'Selected Score', selected_score) | |
| return df | |
| def get_final_score_quality(df, selected_columns): | |
| normalize_df = get_normalized_df(df) | |
| for name in normalize_df.drop('Model Name (clickable)', axis=1): | |
| normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name] | |
| quality_score = normalize_df[QUALITY_TAB].sum(axis=1) / sum([DIM_WEIGHT[i] for i in QUALITY_TAB]) | |
| if 'Quality Score' in df: | |
| df['Quality Score'] = quality_score | |
| else: | |
| df.insert(1, 'Quality Score', quality_score) | |
| # selected_score = normalize_df[selected_columns].sum(axis=1) / len(selected_columns) | |
| selected_score = normalize_df[selected_columns].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_columns]) | |
| if 'Selected Score' in df: | |
| df['Selected Score'] = selected_score | |
| else: | |
| df.insert(1, 'Selected Score', selected_score) | |
| return df | |
| def get_baseline_df(): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(CSV_DIR) | |
| df = get_final_score(df, checkbox_group.value) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| present_columns = MODEL_INFO + checkbox_group.value | |
| df = df[present_columns] | |
| df = convert_scores_to_percentage(df) | |
| return df | |
| def get_baseline_df_quality(): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(QUALITY_DIR) | |
| df = get_final_score_quality(df, checkbox_group_quality.value) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| present_columns = MODEL_INFO_TAB_QUALITY + checkbox_group_quality.value | |
| df = df[present_columns] | |
| df = convert_scores_to_percentage(df) | |
| return df | |
| def get_baseline_df_i2v(): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(I2V_DIR) | |
| df = get_final_score_i2v(df, checkbox_group_i2v.value) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| present_columns = MODEL_INFO_TAB_I2V + checkbox_group_i2v.value | |
| df = df[present_columns] | |
| df = convert_scores_to_percentage(df) | |
| return df | |
| def get_all_df(selected_columns, dir=CSV_DIR): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(dir) | |
| df = get_final_score(df, selected_columns) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| return df | |
| def get_all_df_quality(selected_columns, dir=QUALITY_DIR): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(dir) | |
| df = get_final_score_quality(df, selected_columns) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| return df | |
| def get_all_df_i2v(selected_columns, dir=I2V_DIR): | |
| # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| # submission_repo.git_pull() | |
| df = pd.read_csv(dir) | |
| df = get_final_score_i2v(df, selected_columns) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| return df | |
| def convert_scores_to_percentage(df): | |
| # 对DataFrame中的每一列（除了'name'列）进行操作 | |
| for column in df.columns[1:]: # 假设第一列是'name' | |
| df[column] = round(df[column] * 100,2) # 将分数转换为百分数 | |
| df[column] = df[column].astype(str) + '%' | |
| return df | |
| def choose_all_quailty(): | |
| return gr.update(value=QUALITY_LIST) | |
| def choose_all_semantic(): | |
| return gr.update(value=SEMANTIC_LIST) | |
| def disable_all(): | |
| return gr.update(value=[]) | |
| def enable_all(): | |
| return gr.update(value=TASK_INFO) | |
| def on_filter_model_size_method_change(selected_columns): | |
| updated_data = get_all_df(selected_columns, CSV_DIR) | |
| #print(updated_data) | |
| # columns: | |
| selected_columns = [item for item in TASK_INFO if item in selected_columns] | |
| present_columns = MODEL_INFO + selected_columns | |
| updated_data = updated_data[present_columns] | |
| updated_data = updated_data.sort_values(by="Selected Score", ascending=False) | |
| updated_data = convert_scores_to_percentage(updated_data) | |
| updated_headers = present_columns | |
| update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] | |
| # print(updated_data,present_columns,update_datatype) | |
| filter_component = gr.components.Dataframe( | |
| value=updated_data, | |
| headers=updated_headers, | |
| type="pandas", | |
| datatype=update_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| return filter_component#.value | |
| def on_filter_model_size_method_change_quality(selected_columns): | |
| updated_data = get_all_df_quality(selected_columns, QUALITY_DIR) | |
| #print(updated_data) | |
| # columns: | |
| selected_columns = [item for item in QUALITY_TAB if item in selected_columns] | |
| present_columns = MODEL_INFO_TAB_QUALITY + selected_columns | |
| updated_data = updated_data[present_columns] | |
| updated_data = updated_data.sort_values(by="Selected Score", ascending=False) | |
| updated_data = convert_scores_to_percentage(updated_data) | |
| updated_headers = present_columns | |
| update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] | |
| # print(updated_data,present_columns,update_datatype) | |
| filter_component = gr.components.Dataframe( | |
| value=updated_data, | |
| headers=updated_headers, | |
| type="pandas", | |
| datatype=update_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| return filter_component#.value | |
| def on_filter_model_size_method_change_i2v(selected_columns): | |
| updated_data = get_all_df_i2v(selected_columns, I2V_DIR) | |
| selected_columns = [item for item in I2V_TAB if item in selected_columns] | |
| present_columns = MODEL_INFO_TAB_I2V + selected_columns | |
| updated_data = updated_data[present_columns] | |
| updated_data = updated_data.sort_values(by="Selected Score", ascending=False) | |
| updated_data = convert_scores_to_percentage(updated_data) | |
| updated_headers = present_columns | |
| update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES_I2V.index(x)] for x in updated_headers] | |
| # print(updated_data,present_columns,update_datatype) | |
| filter_component = gr.components.Dataframe( | |
| value=updated_data, | |
| headers=updated_headers, | |
| type="pandas", | |
| datatype=update_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| return filter_component#.value | |
| block = gr.Blocks() | |
| with block: | |
| gr.Markdown( | |
| LEADERBORAD_INTRODUCTION | |
| ) | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| # Table 0 | |
| with gr.TabItem("📊 VBench", elem_id="vbench-tab-table", id=1): | |
| with gr.Row(): | |
| with gr.Accordion("Citation", open=False): | |
| citation_button = gr.Textbox( | |
| value=CITATION_BUTTON_TEXT, | |
| label=CITATION_BUTTON_LABEL, | |
| elem_id="citation-button", | |
| lines=10, | |
| ) | |
| gr.Markdown( | |
| TABLE_INTRODUCTION | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=0.2): | |
| choosen_q = gr.Button("Select Quality Dimensions") | |
| choosen_s = gr.Button("Select Semantic Dimensions") | |
| # enable_b = gr.Button("Select All") | |
| disable_b = gr.Button("Deselect All") | |
| with gr.Column(scale=0.8): | |
| # selection for column part: | |
| checkbox_group = gr.CheckboxGroup( | |
| choices=TASK_INFO, | |
| value=DEFAULT_INFO, | |
| label="Evaluation Dimension", | |
| interactive=True, | |
| ) | |
| data_component = gr.components.Dataframe( | |
| value=get_baseline_df, | |
| headers=COLUMN_NAMES, | |
| type="pandas", | |
| datatype=DATA_TITILE_TYPE, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| choosen_q.click(choose_all_quailty, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group], outputs=data_component) | |
| choosen_s.click(choose_all_semantic, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group], outputs=data_component) | |
| # enable_b.click(enable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group], outputs=data_component) | |
| disable_b.click(disable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group], outputs=data_component) | |
| checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[ checkbox_group], outputs=data_component) | |
| with gr.TabItem("Video Quaity", elem_id="vbench-tab-table", id=2): | |
| with gr.Accordion("INSTRUCTION", open=False): | |
| citation_button = gr.Textbox( | |
| value=QUALITY_CLAIM_TEXT, | |
| label="", | |
| elem_id="quality-button", | |
| lines=2, | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1.0): | |
| # selection for column part: | |
| checkbox_group_quality = gr.CheckboxGroup( | |
| choices=QUALITY_TAB, | |
| value=QUALITY_TAB, | |
| label="Evaluation Quality Dimension", | |
| interactive=True, | |
| ) | |
| data_component_quality = gr.components.Dataframe( | |
| value=get_baseline_df_quality, | |
| headers=COLUMN_NAMES_QUALITY, | |
| type="pandas", | |
| datatype=DATA_TITILE_TYPE, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| checkbox_group_quality.change(fn=on_filter_model_size_method_change_quality, inputs=[checkbox_group_quality], outputs=data_component_quality) | |
| with gr.TabItem("VBench-I2V", elem_id="vbench-tab-table", id=3): | |
| with gr.Accordion("NOTE", open=False): | |
| i2v_note_button = gr.Textbox( | |
| value=I2V_CLAIM_TEXT, | |
| label="", | |
| elem_id="quality-button", | |
| lines=3, | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1.0): | |
| # selection for column part: | |
| checkbox_group_i2v = gr.CheckboxGroup( | |
| choices=I2V_TAB, | |
| value=I2V_TAB, | |
| label="Evaluation Quality Dimension", | |
| interactive=True, | |
| ) | |
| data_component_i2v = gr.components.Dataframe( | |
| value=get_baseline_df_i2v, | |
| headers=COLUMN_NAMES_I2V, | |
| type="pandas", | |
| datatype=I2V_TITILE_TYPE, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v], outputs=data_component_i2v) | |
| # table 2 | |
| with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=4): | |
| gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text") | |
| # table 3 | |
| with gr.TabItem("🚀 Submit here! ", elem_id="mvbench-tab-table", id=4): | |
| gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text") | |
| with gr.Row(): | |
| gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text") | |
| with gr.Row(): | |
| gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text") | |
| with gr.Row(): | |
| with gr.Column(): | |
| model_name_textbox = gr.Textbox( | |
| label="Model name", placeholder="LaVie" | |
| ) | |
| revision_name_textbox = gr.Textbox( | |
| label="Revision Model Name", placeholder="LaVie" | |
| ) | |
| with gr.Column(): | |
| model_link = gr.Textbox( | |
| label="Model Link", placeholder="https://huggingface.co/decapoda-research/llama-7b-hf" | |
| ) | |
| with gr.Column(): | |
| input_file = gr.components.File(label = "Click to Upload a json File", file_count="single", type='binary') | |
| submit_button = gr.Button("Submit Eval") | |
| submission_result = gr.Markdown() | |
| submit_button.click( | |
| add_new_eval, | |
| inputs = [ | |
| input_file, | |
| model_name_textbox, | |
| revision_name_textbox, | |
| model_link, | |
| ], | |
| ) | |
| def refresh_data(): | |
| value1 = get_baseline_df() | |
| return value1 | |
| with gr.Row(): | |
| data_run = gr.Button("Refresh") | |
| data_run.click(on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component) | |
| block.launch() |