# %% # TODOS: Plots with plotly import json import pandas as pd import streamlit as st import plotly.express as px from config import other_info_dict # %% st.title("Microsoft Phi-2 LLM assessment") # st.image('model_card.png', caption='Hugging face description', use_column_width=True) st.write(""" Microsoft Phi-2 (https://huggingface.co/microsoft/phi-2) is a Transformer model with 2.7 billion parameters. Performance on benchmarks for common sense, language understanding, and logical reasoning is nearly state-of-the-art among models with less than 13 billion parameters. Unlike typical Large Language Models (LLM), Phi-2 has not been fine-tuned through reinforcement learning from human feedback.""") import urllib.request import os prefix_post_processing = os.environ["POST_PROCESSING_JSON"] st.header('Evaluation dataset') st.write(other_info_dict['data_description']) # %% st.header("Prompt") st.write("For each data point in the evaluation dataset, we create a prompt for LLM by adding the context and the question to the below prompt template, while following the same structure of the prompt template.") prompt_options_dict = { 'Prompt option 0 (with typos and grammatical errors, two shot prompting)': 'prompt_option_0.json', 'Prompt option 1 (Zero shot prompting)': 'prompt_option_1.json', 'Prompt option 2 (Prompt option 2 with two shot prompting)': 'prompt_option_2.json', 'Prompt option 3 (Prompt option 0 with minor fixes)': 'prompt_option_3.json' } t_result_file = st.selectbox( 'Select the prompt:', list(prompt_options_dict.keys())) result_file = prefix_post_processing + prompt_options_dict[t_result_file] prompt_option = int(prompt_options_dict[t_result_file].split('_')[-1].split('.')[0]) with urllib.request.urlopen(result_file) as url: data_dict = json.load(url) # File uploader with open(f'prompt_{prompt_option}.txt', "r") as file: file_contents = file.read() # st.write(file_contents) st.text_area("Prompt template:", value=file_contents, height=300) st.write("The answer to the question is obtained by post-processing the output of the LLM, wherein any additional content starting from the first 'Context: ' is disregarded.") st.write("In the case that the LLM answers , the output is set to an empty string.") # 'Context: ' + context + '\n\n' + 'Question: ' + t_question + '\n\n' + 'Answer:' # %% overall_performance = round(data_dict["Overall performance"]*100, 2) st.header('Performance metric') st.write("""The performance metric used is an estimation of the percentage of correctly answered questions, i.e. the output of the model coincides with one of the ground truth answers. The performance metric can also be interpreted as the probability that the model correctly answers a question. The performance of the model is evaluated with the exact match accuracy metric (see compute_exact function in SQuAD2.0 official evaluation script [here](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/)), taking values in [0,1], where 0 is worst (model always wrong), and 1 is best (model always correct). It is the number of correctly answered questions divided by the number of data points. An answer is considered to be correctly answered (by the model), if the predicted answer after normalization (text is converted to lowercase, and punctuation, articles and extra whitespace are removed) matches exactly with any of the normalized ground truth answers. In the case of unanswerable questions, the empty string is considered to be the only ground truth answer.""") with st.container(): st.write(f"**Overall performance: {overall_performance}%**") # %% st.header("Bias ratios") st.write('Bias ratio is defined as the ratio of the lowest performance to the highest performance among categories that have sufficient data (with more than 50 data points) for a characteristic. The following table shows the bias ratio for each of the considered characteristic.') fairness_results = data_dict['Fairness results'] characteristic_list = [] fairness_ratio_list = [] for key, val in fairness_results.items(): characteristic_list += [key] fairness_ratio_list += [val['OverallFairness']] ch_df = pd.DataFrame({ 'Characteristic': characteristic_list, 'Bias ratio': fairness_ratio_list }) st.dataframe(ch_df) # %% st.header("Robustness") st.write(f"""We evaluate the robustness of the LLM by assessing the variation in performance when perturbations are introduced to the content outside of the prompt template. The following plot shows the performance across different levels of perturbation within a perturbation family that consists of a series of perturbation methods. We consider the following perturbation families. - ProbTypos: {other_info_dict['ProbTypos_description']} - MaxTypo: {other_info_dict['MaxTypo_description']} """) # st.write(f"ProbTypos: {other_info_dict['ProbTypos_description']}") # st.write(f"MaxTypo: {other_info_dict['MaxTypo_description']}") global_perturber_families = data_dict['Perturber Families'] t_pert_fig = None perf_pert_values = [] normalized_perf_pert_values = [] family_levels = [] family_names_list = [] levels_index_list = [] for item in global_perturber_families: family_name = item['family name'] family_results = data_dict['Performance Robustness']['Perturber family wise results'][family_name]["PerformancePerturbers"]# TODO: change the structuer of post processing here family_levels += item['levels'] original_perf = family_results[item['levels'][0]] count = 0 for t_item in item['levels']: perf_pert_values += [family_results[t_item]] normalized_perf_pert_values += [family_results[t_item]/original_perf] family_names_list += [family_name] levels_index_list += [count] count += 1 t_pert_df_global = pd.DataFrame({ 'Perturbation level': family_levels, 'Performance': perf_pert_values, 'normalized performance': normalized_perf_pert_values, 'Perturbation family': family_names_list, 'Levels' : levels_index_list }) t_pert_fig = px.line(t_pert_df_global, x="Levels", y="Performance", color='Perturbation family') t_pert_fig.update_xaxes(tickmode='linear', dtick=1) st.plotly_chart(t_pert_fig, theme="streamlit", use_container_width=True) # %% st.header("Characteristic results") embedder_categories = data_dict['Embedder categories'] option = st.selectbox( 'Select characteristic:', sorted(list(embedder_categories.keys()))) st.write('The following are the categories:') st.write(', '.join(embedder_categories[option])) if 'Length' in option: st.write("Note: Here, length denotes the number of characters. ") if 'gender' in option: st.write(other_info_dict['gender_categories_text']) if 'ethnicity' in option: st.write(other_info_dict['ethnicity_categories_text']) embedder_perf_ci_table = data_dict['Performance results'][option]['CI_Table'] n_points = data_dict['n points'] category_share_of_data = {} categories_list = [] share_of_data_list = [] n_points_list = [] for key, val in embedder_perf_ci_table.items(): categories_list += [val['category']] share_of_data_list += [val['Share of Data']] n_points_list += [int(val['Share of Data']*n_points/100)] st.markdown("---") st.write("The following plot illustrates the distribution of data points across different categories.") t_df = pd.DataFrame({ 'Category': categories_list, 'Share of data': share_of_data_list, 'Number of points': n_points_list }) fig = px.bar(t_df, x='Category', y='Number of points') st.plotly_chart(fig, theme="streamlit", use_container_width=True) st.markdown("---") st.write("The performance metric is shown together with 95% confidence intervals for each of the categories.") embedder_fair_ci_table = data_dict['Fairness results'][option]['CI_Table'] categories_list = [] estimates_list = [] uppers_list = [] lowers_list = [] for key, val in embedder_fair_ci_table.items(): categories_list += [val['category']] estimates_list += [val['Estimate']] uppers_list += [val['Upper']] lowers_list += [val['Lower']] t_fair_df = pd.DataFrame({ 'Category': categories_list, 'Estimate': estimates_list, 'Upper': uppers_list, 'Lower': lowers_list }) t_fair_df['Diff upper'] = t_fair_df['Upper'] - t_fair_df['Estimate'] t_fair_df['Diff lower'] = t_fair_df['Estimate'] - t_fair_df['Lower'] fig_fair = px.scatter(t_fair_df, x='Category', y='Estimate', error_y='Diff upper', error_y_minus='Diff lower') fig_fair.update_layout(yaxis_title="Performance in %") st.plotly_chart(fig_fair, theme="streamlit", use_container_width=True) st.markdown("---") st.write('The following plots show the normalized average performance for each category of a characteristic, for each level of perturbation, starting with no perturbation. Each curve represents the normalized average performance on a category, by which we mean that we divide the average performance at every level of perturbation by the average performance without perturbation. ') t_result = data_dict['Performance Robustness']['Embedder wise results'][option] # Embedder categories for item in global_perturber_families: family_name = item['family name'] dfs_list = [] count = 0 for t_item in item['levels']: df = pd.DataFrame(t_result[t_item]) df['Perturber'] = t_item df['Perturber family'] = family_name df['Levels'] = count dfs_list += [df] count += 1 merged_df = pd.concat(dfs_list, axis=0) temp_header = f'Perturber family: {family_name}' # st.markdown(f'##### {temp_header}') t_pert_fig = px.line(merged_df, x="Levels", y="normalized performance", color='category') t_pert_fig.update_layout(yaxis_title="Normalized performance") # px.line(t_pert_df_global, x="Levels", y="Performance", color='Perturbation family') t_pert_df_global_temp = t_pert_df_global[t_pert_df_global['Perturbation family'] == family_name].copy(deep=True) t_pert_df_global_temp['category'] = 'Overall' t_pert_fig.add_trace(px.line(t_pert_df_global_temp, x="Levels", y="normalized performance", color='category').data[0]) t_pert_fig.update_xaxes(tickmode='linear', dtick=1) st.write(f'The following plot illustrates the normalized performance of the model across different categories for the perturbation family: {family_name}.') st.plotly_chart(t_pert_fig, theme="streamlit", use_container_width=True) st.markdown("---")