| from vlmeval.smp import * | |
| from vlmeval.tools import EVAL | |
| import gradio as gr | |
| HEADER = """ | |
| # Welcome to MMBench👏👏 | |
| We are delighted that you are willing to submit the evaluation results to the MMBench official website! The evaluation service currently can handle submissions of MMBench, MMBench-CN, and CCBench. We use `gpt-3.5-turbo-0125` to help answer matching. Evaluation Codes in VLMEvalKit: https://github.com/open-compass/VLMEvalKit. Please adopt / follow the implementation of VLMEvalKit to generate the submission files. | |
| The evaluation script is available at https://github.com/open-compass/VLMEvalKit/tree/main/scripts/mmb_eval_gradio.py | |
| Please contact `opencompass@pjlab.org.cn` for any inquirys about this script. | |
| """ | |
| def upload_file(file): | |
| file_path = file.name | |
| return file_path | |
| def prepare_file(file_name): | |
| file_md5 = md5(file_name) | |
| root = LMUDataRoot() | |
| root = osp.join(root, 'eval_server') | |
| os.makedirs(root, exist_ok=True) | |
| suffix = file_name.split('.')[-1] | |
| if suffix not in ['xlsx', 'tsv', 'csv']: | |
| return False, "Please submit a file that ends with `.xlsx`, `.tsv`, or `.csv`" | |
| new_file_name = osp.join(root, f'{file_md5}.{suffix}') | |
| shutil.move(file_name, new_file_name) | |
| eval_file = new_file_name | |
| try: | |
| data = load(eval_file) | |
| except: | |
| return False, "Your excel file can not be successfully loaded by `pd.read_excel`, please double check and submit again. " | |
| for k in data.keys(): | |
| data[k.lower() if k not in 'ABCD' else k] = data.pop(k) | |
| if "index" not in data: | |
| return False, "Your excel file should have a column named `index`, please double check and submit again" , {} | |
| if "prediction" not in data: | |
| return False, "Your excel file should have a column named `prediction`, please double check and submit again" , {} | |
| for ch in 'ABCD': | |
| if ch not in data: | |
| return False, f"Your excel file should have a column named `{ch}`, please double check and submit again" , {} | |
| dump(data, eval_file) | |
| return True, eval_file | |
| def determine_dataset(eval_file): | |
| data = load(eval_file) | |
| def cn_ratio(data): | |
| iscn = [cn_string(x) for x in data['question']] | |
| return np.mean(iscn) | |
| max_ind = np.max([int(x) for x in data['index'] if int(x) < 1e5]) | |
| if max_ind < 1000 and 'l2-category' not in data: | |
| return 'CCBench' if cn_ratio(data) > 0.5 else "Unknown" | |
| elif max_ind < 3000 : | |
| return 'MMBench_CN' if cn_ratio(data) > 0.5 else "MMBench" | |
| else: | |
| return 'MMBench_CN_V11' if cn_ratio(data) > 0.5 else "MMBench_V11" | |
| def reformat_acc(acc): | |
| splits = set(acc['split']) | |
| keys = list(acc.keys()) | |
| keys.remove('split') | |
| nacc = {'Category': []} | |
| for sp in splits: | |
| nacc[sp.upper()] = [] | |
| for k in keys: | |
| nacc['Category'].append(k) | |
| for sp in splits: | |
| nacc[sp.upper()].append(acc[acc['split'] == sp].iloc[0][k] * 100) | |
| return pd.DataFrame(nacc) | |
| def evaluate(file): | |
| file_name = file.name | |
| flag, eval_file = prepare_file(file_name) | |
| if not flag: | |
| return "Error: " + eval_file | |
| dataset = determine_dataset(eval_file) | |
| if dataset == 'Unknown': | |
| return "Error: Cannot determine the dataset given your submitted file. " | |
| eval_id = eval_file.split('/')[-1].split('.')[0] | |
| ret = f"Evaluation ID: {eval_id}\n" | |
| timestamp = datetime.datetime.now().strftime('%Y.%m.%d %H:%M:%S') | |
| ret += f'Evaluation Timestamp: {timestamp}\n' | |
| acc = EVAL(dataset, eval_file) | |
| nacc = reformat_acc(acc).round(1) | |
| return ret, nacc | |
| with gr.Blocks() as demo: | |
| gr.Markdown(HEADER) | |
| file_output = gr.File() | |
| upload_button = gr.UploadButton("Click to upload you prediction files for a supported benchmark") | |
| upload_button.upload(upload_file, upload_button, file_output) | |
| btn = gr.Button("🚀 Evaluate") | |
| eval_log = gr.Textbox(label="Evaluation Log", placeholder="Your evaluation log will be displayed here") | |
| df_empty = pd.DataFrame([], columns=['Evaluation Result']) | |
| eval_result = gr.components.DataFrame(value=df_empty) | |
| btn.click(evaluate, inputs=[file_output], outputs=[eval_log, eval_result]) | |
| if __name__ == '__main__': | |
| demo.launch(server_name='0.0.0.0', debug=True, show_error=True) |